@duckmind/dm-windows-x64 0.63.0 → 0.63.6

package/extensions/dm-web-access/xcrawl.js

@@ -0,0 +1,221 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const XCRAWL_API_URL = "https://run.xcrawl.com/v1/serp";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 60000;
+let cachedConfig = null;
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf-8");
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+  }
+  cachedConfig = parsed;
+  return cachedConfig;
+}
+async function getApiKey(signal) {
+  return resolveCredential({
+    provider: "XCrawl",
+    configuredValue: loadConfig().xcrawlApiKey,
+    environmentValue: process.env.XCRAWL_API_KEY,
+    signal
+  });
+}
+export function isXcrawlAvailable() {
+  return hasCredentialSource({ provider: "XCrawl", configuredValue: loadConfig().xcrawlApiKey, environmentValue: process.env.XCRAWL_API_KEY });
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function invalidResponse(message) {
+  return new Error(`XCrawl API returned invalid response: ${message}`);
+}
+function normalizeCount(value) {
+  if (typeof value !== "number" || !Number.isFinite(value))
+    return 5;
+  return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function hostnameOf(url) {
+  try {
+    return new URL(url).hostname.toLowerCase();
+  } catch {
+    return "";
+  }
+}
+function absolutizeLink(link) {
+  if (/^https?:\/\//i.test(link))
+    return link;
+  try {
+    return new URL(link, XCRAWL_API_URL).href;
+  } catch {
+    return link;
+  }
+}
+function normalizeDomain(value) {
+  let input = value.trim().toLowerCase();
+  if (!input)
+    return null;
+  if (input.startsWith("-"))
+    input = input.slice(1).trim();
+  if (!input)
+    return null;
+  try {
+    const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+    input = parsed.hostname;
+  } catch {
+    input = input.split("/")[0]?.split(":")[0] ?? "";
+  }
+  input = input.replace(/^\.+|\.+$/g, "");
+  return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function applyDomainFilter(results, domainFilter) {
+  const includes = [];
+  const excludes = [];
+  for (const raw of domainFilter) {
+    const normalized = normalizeDomain(raw);
+    if (!normalized)
+      continue;
+    if (raw.trim().startsWith("-"))
+      excludes.push(normalized);
+    else
+      includes.push(normalized);
+  }
+  if (!includes.length && !excludes.length)
+    return results;
+  return results.filter((result) => {
+    const host = hostnameOf(result.url);
+    if (!host)
+      return false;
+    const matches = (domain) => host === domain || host.endsWith(`.${domain}`);
+    if (excludes.some(matches))
+      return false;
+    if (includes.length && !includes.some(matches))
+      return false;
+    return true;
+  });
+}
+function parseResponse(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw invalidResponse("expected an object envelope");
+  }
+  const envelope = value;
+  const metadata = envelope.search_metadata;
+  if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) {
+    throw invalidResponse("expected search_metadata object");
+  }
+  const status = metadata.status;
+  if (status !== undefined && status !== "completed") {
+    throw invalidResponse(`expected search_metadata.status "completed", got ${JSON.stringify(status)}`);
+  }
+  if (!Array.isArray(envelope.organic_results))
+    throw invalidResponse("expected organic_results array");
+  const results = [];
+  for (const [index, value] of envelope.organic_results.entries()) {
+    if (!value || typeof value !== "object" || Array.isArray(value)) {
+      throw invalidResponse(`expected organic_results[${index}] object`);
+    }
+    const result = value;
+    const { title, link, snippet } = result;
+    if (typeof link !== "string" || !link.trim())
+      throw invalidResponse(`expected organic_results[${index}].link to be a non-empty string`);
+    if (title !== null && title !== undefined && typeof title !== "string") {
+      throw invalidResponse(`expected organic_results[${index}].title to be a string or null`);
+    }
+    if (snippet !== undefined && snippet !== null && typeof snippet !== "string") {
+      throw invalidResponse(`expected organic_results[${index}].snippet to be a string or null`);
+    }
+    const resolved = absolutizeLink(link.trim());
+    results.push({
+      title: typeof title === "string" && title.trim().length > 0 ? title : resolved,
+      url: resolved,
+      snippet: typeof snippet === "string" ? snippet : ""
+    });
+  }
+  return results;
+}
+function buildAnswer(results) {
+  return results.map((result) => result.snippet ? `${result.snippet}
+Source: ${result.title} (${result.url})` : `Source: ${result.title} (${result.url})`).join(`
+
+`);
+}
+export async function searchWithXCrawl(query, options = {}) {
+  const apiKey = await getApiKey(options.signal);
+  const numResults = normalizeCount(options.numResults);
+  if (!apiKey) {
+    throw new Error("XCrawl search requires an API key. Set xcrawlApiKey in " + CONFIG_PATH + " or export XCRAWL_API_KEY. Get one at https://dash.xcrawl.com/");
+  }
+  const body = { engine: "google_search", q: query };
+  const activityId = activityMonitor.logStart({ type: "api", query });
+  const timeoutSignal = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+  let response;
+  try {
+    response = await fetch(XCRAWL_API_URL, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify(body),
+      signal: options.signal ? AbortSignal.any([timeoutSignal, options.signal]) : timeoutSignal
+    });
+  } catch (err) {
+    const message = errorMessage(err);
+    if (options.signal?.aborted) {
+      activityMonitor.logComplete(activityId, 0);
+      throw new Error("Aborted");
+    }
+    const providerTimeout = timeoutSignal.aborted || err instanceof Error && err.name === "TimeoutError";
+    let outgoing;
+    if (providerTimeout) {
+      outgoing = new Error(`XCrawl search request timed out after ${Math.round(SEARCH_TIMEOUT_MS / 1000)}s`);
+    } else {
+      const redactedMessage = redactCredential(message, apiKey);
+      outgoing = redactedMessage === message && err instanceof Error ? err : new Error(redactedMessage);
+      if (err instanceof Error && outgoing.name === "Error")
+        outgoing.name = err.name;
+    }
+    activityMonitor.logError(activityId, redactCredential(errorMessage(outgoing), apiKey));
+    throw outgoing;
+  }
+  if (!response.ok) {
+    activityMonitor.logComplete(activityId, response.status);
+    const errorText = redactCredential(await response.text(), apiKey);
+    let detail = errorText.slice(0, 300);
+    try {
+      const parsed = JSON.parse(errorText);
+      if (typeof parsed.message === "string")
+        detail = parsed.message.slice(0, 300);
+      else if (typeof parsed.error === "string")
+        detail = parsed.error.slice(0, 300);
+    } catch {}
+    throw new Error(`XCrawl API error (${response.status}): ${detail}`);
+  }
+  activityMonitor.logComplete(activityId, response.status);
+  let payload;
+  try {
+    payload = await response.json();
+  } catch (err) {
+    throw invalidResponse(`response body is not valid JSON: ${errorMessage(err)}`);
+  }
+  const results = parseResponse(payload);
+  const filtered = ((options.domainFilter?.length) ? applyDomainFilter(results, options.domainFilter) : results).slice(0, numResults);
+  return {
+    answer: buildAnswer(filtered),
+    results: filtered
+  };
+}

package/extensions/dm-web-access/youtube-extract.js

@@ -0,0 +1,279 @@
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { canAttachImages } from "./feature-config.js";
+import { isGeminiWebAvailable, queryWithCookies } from "./gemini-web.js";
+import { isGeminiApiAvailableWithVideo, queryGeminiApiWithVideo } from "./gemini-api.js";
+import { isPerplexityAvailable, searchWithPerplexity } from "./perplexity.js";
+import { extractHeadingTitle } from "./extract.js";
+import { formatSeconds, readExecError, isTimeoutError, trimErrorText, mapFfmpegError, getWebSearchConfigPath } from "./utils.js";
+const CONFIG_PATH = getWebSearchConfigPath();
+const YOUTUBE_PROMPT = `Extract the complete content of this YouTube video. Include:
+1. Video title, channel name, and duration
+2. A brief summary (2-3 sentences)
+3. Full transcript with timestamps
+4. Descriptions of any code, terminal commands, diagrams, slides, or UI shown on screen
+
+Format as markdown.`;
+const YOUTUBE_REGEX = /(?:(?:www\.|m\.)?youtube\.com\/(?:watch\?.*v=|shorts\/|live\/|embed\/|v\/)|youtu\.be\/)([a-zA-Z0-9_-]{11})/;
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function shouldRethrow(err) {
+  return errorMessage(err).startsWith("Failed to parse ");
+}
+function addAttemptError(errors, label, err) {
+  const message = errorMessage(err).replace(/\s+/g, " ").trim();
+  if (message)
+    errors.push(`${label}: ${message}`);
+}
+function normalizePreferredModel(value, fallback) {
+  if (typeof value !== "string")
+    return fallback;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : fallback;
+}
+function normalizeEnabled(value, fallback) {
+  return typeof value === "boolean" ? value : fallback;
+}
+const defaults = { enabled: true, preferredModel: "gemini-3.6-flash" };
+let cachedConfig = null;
+function loadYouTubeConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = { ...defaults };
+    return cachedConfig;
+  }
+  const rawText = readFileSync(CONFIG_PATH, "utf-8");
+  let raw;
+  try {
+    raw = JSON.parse(rawText);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+  const yt = raw.youtube ?? {};
+  cachedConfig = {
+    enabled: normalizeEnabled(yt.enabled, defaults.enabled),
+    preferredModel: normalizePreferredModel(yt.preferredModel, defaults.preferredModel)
+  };
+  return cachedConfig;
+}
+export function isYouTubeURL(url) {
+  try {
+    const parsed = new URL(url);
+    if (parsed.pathname === "/playlist") {
+      return { isYouTube: false, videoId: null };
+    }
+  } catch {}
+  const match = url.match(YOUTUBE_REGEX);
+  if (!match)
+    return { isYouTube: false, videoId: null };
+  return { isYouTube: true, videoId: match[1] };
+}
+export function isYouTubeEnabled() {
+  return loadYouTubeConfig().enabled;
+}
+export async function extractYouTube(url, signal, prompt, model) {
+  const config = loadYouTubeConfig();
+  const { videoId } = isYouTubeURL(url);
+  const canonicalUrl = videoId ? `https://www.youtube.com/watch?v=${videoId}` : url;
+  const effectivePrompt = prompt ?? YOUTUBE_PROMPT;
+  const effectiveModel = model ?? config.preferredModel;
+  const activityId = activityMonitor.logStart({ type: "fetch", url: `youtube.com/${videoId ?? "video"}` });
+  const attemptErrors = [];
+  const result = await tryGeminiWeb(canonicalUrl, effectivePrompt, effectiveModel, signal, attemptErrors) ?? await tryGeminiApi(canonicalUrl, effectivePrompt, effectiveModel, signal, attemptErrors) ?? await tryPerplexity(url, effectivePrompt, signal, attemptErrors);
+  if (result) {
+    result.url = url;
+    if (!result.error && videoId && canAttachImages()) {
+      const thumb = await fetchYouTubeThumbnail(videoId);
+      if (thumb)
+        result.thumbnail = thumb;
+    }
+    activityMonitor.logComplete(activityId, result.error ? 0 : 200);
+    return result;
+  }
+  if (signal?.aborted) {
+    activityMonitor.logComplete(activityId, 0);
+    return null;
+  }
+  const error = attemptErrors.length > 0 ? ["Could not extract YouTube video content.", "", ...attemptErrors.map((message) => `- ${message}`)].join(`
+`) : "Could not extract YouTube video content. Sign into Google in a supported Chromium browser for automatic access, or set GEMINI_API_KEY.";
+  activityMonitor.logError(activityId, error);
+  return { url, title: "", content: "", error };
+}
+function mapYtDlpError(err) {
+  const { code, stderr, message } = readExecError(err);
+  if (code === "ENOENT")
+    return "yt-dlp is not installed. Install with: brew install yt-dlp";
+  if (isTimeoutError(err))
+    return "yt-dlp timed out fetching video info";
+  const lower = stderr.toLowerCase();
+  if (lower.includes("private"))
+    return "Video is private or unavailable";
+  if (lower.includes("sign in"))
+    return "Video is age-restricted and requires authentication";
+  if (lower.includes("not available"))
+    return "Video is unavailable in your region or has been removed";
+  if (lower.includes("live"))
+    return "Cannot extract frames from a live stream";
+  const snippet = trimErrorText(stderr || message);
+  return snippet ? `yt-dlp failed: ${snippet}` : "yt-dlp failed";
+}
+export async function getYouTubeStreamInfo(videoId) {
+  try {
+    const output = execFileSync("yt-dlp", [
+      "--print",
+      "duration",
+      "-g",
+      `https://www.youtube.com/watch?v=${videoId}`
+    ], { timeout: 15000, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
+    const lines = output.split(/\r?\n/);
+    const rawDuration = lines[0]?.trim();
+    const streamUrl = lines[1]?.trim();
+    if (!streamUrl)
+      return { error: "yt-dlp failed: missing stream URL" };
+    const parsedDuration = rawDuration && rawDuration !== "NA" ? Number.parseFloat(rawDuration) : NaN;
+    const duration = Number.isFinite(parsedDuration) ? parsedDuration : null;
+    return { streamUrl, duration };
+  } catch (err) {
+    return { error: mapYtDlpError(err) };
+  }
+}
+async function extractFrameFromStream(streamUrl, seconds) {
+  try {
+    const buffer = execFileSync("ffmpeg", [
+      "-ss",
+      String(seconds),
+      "-i",
+      streamUrl,
+      "-frames:v",
+      "1",
+      "-f",
+      "image2pipe",
+      "-vcodec",
+      "mjpeg",
+      "pipe:1"
+    ], { maxBuffer: 5 * 1024 * 1024, timeout: 30000, stdio: ["pipe", "pipe", "pipe"] });
+    if (buffer.length === 0)
+      return { error: "ffmpeg failed: empty output" };
+    return { data: buffer.toString("base64"), mimeType: "image/jpeg" };
+  } catch (err) {
+    return { error: mapFfmpegError(err) };
+  }
+}
+export async function extractYouTubeFrame(videoId, seconds, streamInfo) {
+  const info = streamInfo ?? await getYouTubeStreamInfo(videoId);
+  if ("error" in info)
+    return info;
+  return extractFrameFromStream(info.streamUrl, seconds);
+}
+export async function extractYouTubeFrames(videoId, timestamps, streamInfo) {
+  const info = streamInfo ?? await getYouTubeStreamInfo(videoId);
+  if ("error" in info)
+    return { frames: [], duration: null, error: info.error };
+  const results = await Promise.all(timestamps.map(async (t) => {
+    const frame = await extractFrameFromStream(info.streamUrl, t);
+    if ("error" in frame)
+      return { error: frame.error };
+    return { ...frame, timestamp: formatSeconds(t) };
+  }));
+  const frames = results.filter((f) => ("data" in f));
+  const errorResult = results.find((f) => ("error" in f));
+  return { frames, duration: info.duration, error: frames.length === 0 && errorResult ? errorResult.error : null };
+}
+export async function fetchYouTubeThumbnail(videoId) {
+  try {
+    const res = await fetch(`https://img.youtube.com/vi/${videoId}/hqdefault.jpg`, {
+      signal: AbortSignal.timeout(5000)
+    });
+    if (!res.ok)
+      return null;
+    const buffer = Buffer.from(await res.arrayBuffer());
+    if (buffer.length === 0)
+      return null;
+    return { data: buffer.toString("base64"), mimeType: "image/jpeg" };
+  } catch {
+    return null;
+  }
+}
+async function tryGeminiWeb(url, prompt, model, signal, attemptErrors) {
+  try {
+    const cookies = await isGeminiWebAvailable();
+    if (!cookies)
+      return null;
+    if (signal?.aborted)
+      return null;
+    const text = await queryWithCookies(prompt, cookies, {
+      youtubeUrl: url,
+      ...model !== "gemini-3.6-flash" ? { model } : {},
+      signal,
+      timeoutMs: 120000
+    });
+    return {
+      url,
+      title: extractHeadingTitle(text) ?? "YouTube Video",
+      content: text,
+      error: null
+    };
+  } catch (err) {
+    if (shouldRethrow(err))
+      throw err;
+    if (!signal?.aborted)
+      addAttemptError(attemptErrors, "Gemini Web", err);
+    return null;
+  }
+}
+async function tryGeminiApi(url, prompt, model, signal, attemptErrors) {
+  try {
+    if (!isGeminiApiAvailableWithVideo())
+      return null;
+    if (signal?.aborted)
+      return null;
+    const text = await queryGeminiApiWithVideo(prompt, url, {
+      model,
+      signal,
+      timeoutMs: 120000
+    });
+    return {
+      url,
+      title: extractHeadingTitle(text) ?? "YouTube Video",
+      content: text,
+      error: null
+    };
+  } catch (err) {
+    if (shouldRethrow(err))
+      throw err;
+    if (!signal?.aborted)
+      addAttemptError(attemptErrors, "Gemini API", err);
+    return null;
+  }
+}
+async function tryPerplexity(url, prompt, signal, attemptErrors) {
+  try {
+    if (signal?.aborted || !isPerplexityAvailable())
+      return null;
+    const perplexityQuery = prompt === YOUTUBE_PROMPT ? `Summarize this YouTube video in detail: ${url}` : `${prompt} YouTube video: ${url}`;
+    const { answer } = await searchWithPerplexity(perplexityQuery, { signal });
+    if (!answer)
+      return null;
+    const content = `# Video Summary (via Perplexity)
+
+${answer}
+
+` + `*Full video understanding requires Gemini access. Set GEMINI_API_KEY or sign into Google in a supported Chromium browser.*`;
+    return {
+      url,
+      title: "Video Summary (via Perplexity)",
+      content,
+      error: null
+    };
+  } catch (err) {
+    if (shouldRethrow(err))
+      throw err;
+    if (!signal?.aborted)
+      addAttemptError(attemptErrors, "Perplexity", err);
+    return null;
+  }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@duckmind/dm-windows-x64",
-  "version": "0.63.0",
+  "version": "0.63.6",
   "description": "DuckMind (dm) binary payload for windows x64",
   "license": "MIT",
   "os": [
@@ -31,6 +31,14 @@
     "jiti": "2.7.0",
     "typebox": "^1.1.34",
     "yaml": "2.8.3",
+    "@mozilla/readability": "^0.6.0",
+    "defuddle": "0.19.3",
+    "linkedom": "^0.16.0",
+    "p-limit": "^6.1.0",
+    "promise.try": "^2.0.1",
+    "turndown": "^7.2.0",
+    "undici": "^8.9.0",
+    "unpdf": "^1.6.2",
     "@sinclair/typebox": "^0.34.48"
   }
 }