@duckmind/dm-windows-x64 0.62.9 → 0.63.4

package/extensions/dm-web-access/video-extract.js

@@ -0,0 +1,336 @@
+import { execFileSync } from "node:child_process";
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import { resolve, extname, basename, join, dirname } from "node:path";
+import { activityMonitor } from "./activity.js";
+import { canAttachImages } from "./feature-config.js";
+import { isGeminiWebAvailable, queryWithCookies } from "./gemini-web.js";
+import { queryGeminiApiWithVideo, getApiKey, fetchGeminiApi, getVersionedApiBase, getUploadBase, redactGeminiApiResponse } from "./gemini-api.js";
+import { extractHeadingTitle } from "./extract.js";
+import { readExecError, trimErrorText, mapFfmpegError, getWebSearchConfigPath } from "./utils.js";
+const CONFIG_PATH = getWebSearchConfigPath();
+const DEFAULT_VIDEO_PROMPT = `Extract the complete content of this video. Include:
+1. Video title (infer from content if not explicit), duration
+2. A brief summary (2-3 sentences)
+3. Full transcript with timestamps
+4. Descriptions of any code, terminal commands, diagrams, slides, or UI shown on screen
+
+Format as markdown.`;
+const VIDEO_EXTENSIONS = {
+  ".mp4": "video/mp4",
+  ".mov": "video/quicktime",
+  ".webm": "video/webm",
+  ".avi": "video/x-msvideo",
+  ".mpeg": "video/mpeg",
+  ".mpg": "video/mpeg",
+  ".wmv": "video/x-ms-wmv",
+  ".flv": "video/x-flv",
+  ".3gp": "video/3gpp",
+  ".3gpp": "video/3gpp"
+};
+function shouldRethrow(err) {
+  const message = err instanceof Error ? err.message : String(err);
+  return message.startsWith("Failed to parse ");
+}
+function normalizePreferredModel(value, fallback) {
+  if (typeof value !== "string")
+    return fallback;
+  const normalized = value.trim();
+  return normalized.length > 0 ? normalized : fallback;
+}
+function normalizeEnabled(value, fallback) {
+  return typeof value === "boolean" ? value : fallback;
+}
+function normalizeMaxSizeMB(value, fallback) {
+  if (typeof value !== "number" || !Number.isFinite(value))
+    return fallback;
+  return value > 0 ? value : fallback;
+}
+const VIDEO_CONFIG_DEFAULTS = {
+  enabled: true,
+  preferredModel: "gemini-3.6-flash",
+  maxSizeMB: 50
+};
+let cachedVideoConfig = null;
+function loadVideoConfig() {
+  if (cachedVideoConfig)
+    return cachedVideoConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedVideoConfig = { ...VIDEO_CONFIG_DEFAULTS };
+    return cachedVideoConfig;
+  }
+  const rawText = readFileSync(CONFIG_PATH, "utf-8");
+  let raw;
+  try {
+    raw = JSON.parse(rawText);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+  const v = raw.video ?? {};
+  cachedVideoConfig = {
+    enabled: normalizeEnabled(v.enabled, VIDEO_CONFIG_DEFAULTS.enabled),
+    preferredModel: normalizePreferredModel(v.preferredModel, VIDEO_CONFIG_DEFAULTS.preferredModel),
+    maxSizeMB: normalizeMaxSizeMB(v.maxSizeMB, VIDEO_CONFIG_DEFAULTS.maxSizeMB)
+  };
+  return cachedVideoConfig;
+}
+export function isVideoFile(input) {
+  const config = loadVideoConfig();
+  if (!config.enabled)
+    return null;
+  const isFilePath = input.startsWith("/") || input.startsWith("./") || input.startsWith("../") || input.startsWith("file://");
+  if (!isFilePath)
+    return null;
+  let filePath = input;
+  if (input.startsWith("file://")) {
+    try {
+      filePath = decodeURIComponent(new URL(input).pathname);
+    } catch {
+      return null;
+    }
+  }
+  const ext = extname(filePath).toLowerCase();
+  const mimeType = VIDEO_EXTENSIONS[ext];
+  if (!mimeType)
+    return null;
+  const absolutePath = resolveFilePath(filePath);
+  if (!absolutePath)
+    return null;
+  let stat;
+  try {
+    stat = statSync(absolutePath);
+  } catch {
+    return null;
+  }
+  if (!stat.isFile())
+    return null;
+  const maxBytes = config.maxSizeMB * 1024 * 1024;
+  return {
+    absolutePath,
+    mimeType,
+    sizeBytes: stat.size,
+    maxSizeBytes: maxBytes,
+    withinUploadLimit: stat.size <= maxBytes
+  };
+}
+function resolveFilePath(filePath) {
+  const absolutePath = resolve(filePath);
+  if (existsSync(absolutePath))
+    return absolutePath;
+  const dir = dirname(absolutePath);
+  const base = basename(absolutePath);
+  if (!existsSync(dir))
+    return null;
+  try {
+    const normalizedBase = normalizeSpaces(base);
+    const match = readdirSync(dir).find((f) => normalizeSpaces(f) === normalizedBase);
+    return match ? join(dir, match) : null;
+  } catch {
+    return null;
+  }
+}
+function normalizeSpaces(s) {
+  return s.replace(/[\u00A0\u2000-\u200B\u202F\u205F\u3000\uFEFF]/g, " ");
+}
+export async function extractVideo(info, signal, options) {
+  const config = loadVideoConfig();
+  const effectivePrompt = options?.prompt ?? DEFAULT_VIDEO_PROMPT;
+  const effectiveModel = options?.model ?? config.preferredModel;
+  const displayName = basename(info.absolutePath);
+  if (!info.withinUploadLimit) {
+    const sizeMB = (info.sizeBytes / 1024 / 1024).toFixed(1);
+    const maxSizeMB = (info.maxSizeBytes / 1024 / 1024).toFixed(1);
+    const error = `Local video ${displayName} is ${sizeMB} MiB, above configured video.maxSizeMB (${maxSizeMB} MiB) for Gemini analysis. Use timestamp/frames for ffmpeg frame extraction, increase video.maxSizeMB, or compress the file.`;
+    return { url: info.absolutePath, title: displayName, content: error, error };
+  }
+  const activityId = activityMonitor.logStart({ type: "fetch", url: `video:${displayName}` });
+  const result = await tryVideoGeminiApi(info, effectivePrompt, effectiveModel, signal) ?? await tryVideoGeminiWeb(info, effectivePrompt, effectiveModel, signal);
+  if (result) {
+    if (canAttachImages()) {
+      const thumbnail = await extractVideoFrame(info.absolutePath);
+      if (!("error" in thumbnail)) {
+        result.thumbnail = thumbnail;
+      }
+    }
+    activityMonitor.logComplete(activityId, 200);
+    return result;
+  }
+  if (signal?.aborted) {
+    activityMonitor.logComplete(activityId, 0);
+    return null;
+  }
+  activityMonitor.logError(activityId, "all video extraction paths failed");
+  return null;
+}
+function mapFfprobeError(err) {
+  const { code, stderr, message } = readExecError(err);
+  if (code === "ENOENT")
+    return "ffprobe is not installed. Install ffmpeg which includes ffprobe";
+  const snippet = trimErrorText(stderr || message);
+  return snippet ? `ffprobe failed: ${snippet}` : "ffprobe failed";
+}
+export async function extractVideoFrame(filePath, seconds = 1) {
+  try {
+    const buffer = execFileSync("ffmpeg", [
+      "-ss",
+      String(seconds),
+      "-i",
+      filePath,
+      "-frames:v",
+      "1",
+      "-f",
+      "image2pipe",
+      "-vcodec",
+      "mjpeg",
+      "pipe:1"
+    ], { maxBuffer: 5 * 1024 * 1024, timeout: 1e4, stdio: ["pipe", "pipe", "pipe"] });
+    if (buffer.length === 0)
+      return { error: "ffmpeg failed: empty output" };
+    return { data: buffer.toString("base64"), mimeType: "image/jpeg" };
+  } catch (err) {
+    return { error: mapFfmpegError(err) };
+  }
+}
+export async function getLocalVideoDuration(filePath) {
+  try {
+    const output = execFileSync("ffprobe", [
+      "-v",
+      "quiet",
+      "-show_entries",
+      "format=duration",
+      "-of",
+      "csv=p=0",
+      filePath
+    ], { timeout: 1e4, encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] }).trim();
+    const duration = Number.parseFloat(output);
+    if (!Number.isFinite(duration))
+      return { error: "ffprobe failed: invalid duration output" };
+    return duration;
+  } catch (err) {
+    return { error: mapFfprobeError(err) };
+  }
+}
+async function tryVideoGeminiWeb(info, prompt, model, signal) {
+  try {
+    const cookies = await isGeminiWebAvailable();
+    if (!cookies)
+      return null;
+    if (signal?.aborted)
+      return null;
+    const text = await queryWithCookies(prompt, cookies, {
+      files: [info.absolutePath],
+      ...model !== "gemini-3.6-flash" ? { model } : {},
+      signal,
+      timeoutMs: 180000
+    });
+    return {
+      url: info.absolutePath,
+      title: extractVideoTitle(text, info.absolutePath),
+      content: text,
+      error: null
+    };
+  } catch (err) {
+    if (shouldRethrow(err))
+      throw err;
+    return null;
+  }
+}
+async function tryVideoGeminiApi(info, prompt, model, signal) {
+  const apiKey = await getApiKey(signal);
+  if (!apiKey)
+    return null;
+  if (signal?.aborted)
+    return null;
+  let fileName = null;
+  try {
+    const uploaded = await uploadToFilesApi(info, apiKey, signal);
+    fileName = uploaded.name;
+    await pollFileState(fileName, apiKey, signal, 120000);
+    const text = await queryGeminiApiWithVideo(prompt, uploaded.uri, {
+      apiKey,
+      model,
+      mimeType: info.mimeType,
+      signal,
+      timeoutMs: 120000
+    });
+    return {
+      url: info.absolutePath,
+      title: extractVideoTitle(text, info.absolutePath),
+      content: text,
+      error: null
+    };
+  } catch (err) {
+    if (shouldRethrow(err))
+      throw err;
+    return null;
+  } finally {
+    if (fileName)
+      deleteGeminiFile(fileName, apiKey);
+  }
+}
+async function uploadToFilesApi(info, apiKey, signal) {
+  const displayName = basename(info.absolutePath);
+  const initRes = await fetchGeminiApi(`${getUploadBase()}/files`, {
+    method: "POST",
+    headers: {
+      "X-Goog-Upload-Protocol": "resumable",
+      "X-Goog-Upload-Command": "start",
+      "X-Goog-Upload-Header-Content-Length": String(info.sizeBytes),
+      "X-Goog-Upload-Header-Content-Type": info.mimeType,
+      "Content-Type": "application/json"
+    },
+    body: JSON.stringify({ file: { display_name: displayName } }),
+    signal
+  }, apiKey);
+  if (!initRes.ok) {
+    const text = redactGeminiApiResponse(initRes, await initRes.text(), apiKey);
+    throw new Error(`File upload init failed: ${initRes.status} (${text.slice(0, 200)})`);
+  }
+  const uploadUrl = initRes.headers.get("x-goog-upload-url");
+  if (!uploadUrl)
+    throw new Error("No upload URL in response headers");
+  const fileData = await readFile(info.absolutePath);
+  const uploadRes = await fetchGeminiApi(uploadUrl, {
+    method: "PUT",
+    headers: {
+      "Content-Length": String(info.sizeBytes),
+      "X-Goog-Upload-Offset": "0",
+      "X-Goog-Upload-Command": "upload, finalize"
+    },
+    body: fileData,
+    signal
+  }, apiKey);
+  if (!uploadRes.ok) {
+    const text = redactGeminiApiResponse(uploadRes, await uploadRes.text(), apiKey);
+    throw new Error(`File upload failed: ${uploadRes.status} (${text.slice(0, 200)})`);
+  }
+  const result = await uploadRes.json();
+  return result.file;
+}
+async function pollFileState(fileName, apiKey, signal, timeoutMs = 120000) {
+  const deadline = Date.now() + timeoutMs;
+  while (Date.now() < deadline) {
+    if (signal?.aborted)
+      throw new Error("Aborted");
+    const res = await fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`, { signal }, apiKey);
+    if (!res.ok)
+      throw new Error(`File state check failed: ${res.status}`);
+    const data = await res.json();
+    if (data.state === "ACTIVE")
+      return;
+    if (data.state === "FAILED")
+      throw new Error("File processing failed");
+    await new Promise((r) => setTimeout(r, 5000));
+  }
+  throw new Error("File processing timed out");
+}
+function deleteGeminiFile(fileName, apiKey) {
+  fetchGeminiApi(`${getVersionedApiBase()}/${fileName}`, { method: "DELETE" }, apiKey).catch((err) => {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(`Failed to delete Gemini file ${fileName}: ${message}`);
+  });
+}
+function extractVideoTitle(text, filePath) {
+  return extractHeadingTitle(text) ?? basename(filePath, extname(filePath));
+}

package/extensions/dm-web-access/xai-search.js

@@ -0,0 +1,285 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const XAI_RESPONSES_URL = "https://api.x.ai/v1/responses";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 60000;
+const AUTH_MODEL_CANDIDATES = ["grok-4.5", "grok-4.3", "grok-build-0.1"];
+let cachedConfig = null;
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf-8");
+  try {
+    cachedConfig = JSON.parse(raw);
+    return cachedConfig;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+}
+function resolveConfiguredSearchModel(value) {
+  if (value == null)
+    return;
+  if (typeof value !== "string" || value.trim().length === 0) {
+    throw new Error(`xaiSearchModel in ${CONFIG_PATH} must be a non-empty string`);
+  }
+  return value.trim();
+}
+function toRequestHeaders(headers) {
+  const requestHeaders = {};
+  for (const [name, value] of Object.entries(headers)) {
+    if (value !== null)
+      requestHeaders[name] = value;
+  }
+  return requestHeaders;
+}
+async function resolvePiAuth(ctx, modelOverride) {
+  for (const modelId of AUTH_MODEL_CANDIDATES) {
+    try {
+      const model = ctx.modelRegistry.find("xai", modelId);
+      if (!model)
+        continue;
+      const resolved = await ctx.modelRegistry.getApiKeyAndHeaders(model);
+      if (resolved.ok && resolved.apiKey) {
+        return { apiKey: resolved.apiKey, model: modelOverride ?? modelId, headers: resolved.headers ?? {} };
+      }
+    } catch {}
+  }
+  return;
+}
+export async function resolveXaiAuth(ctx, signal) {
+  const config = loadConfig();
+  const modelOverride = resolveConfiguredSearchModel(config.xaiSearchModel);
+  if (ctx) {
+    const auth = await resolvePiAuth(ctx, modelOverride);
+    if (auth)
+      return auth;
+  }
+  const hasSource = hasCredentialSource({
+    provider: "xAI",
+    configuredValue: config.xaiApiKey,
+    environmentValue: process.env.XAI_API_KEY
+  });
+  if (!hasSource)
+    return;
+  const apiKey = await resolveCredential({
+    provider: "xAI",
+    configuredValue: config.xaiApiKey,
+    environmentValue: process.env.XAI_API_KEY,
+    signal
+  });
+  return apiKey ? { apiKey, model: modelOverride ?? AUTH_MODEL_CANDIDATES[0], headers: {} } : undefined;
+}
+export async function isXaiSearchAvailable(ctx) {
+  if (ctx && await resolvePiAuth(ctx))
+    return true;
+  const config = loadConfig();
+  return hasCredentialSource({
+    provider: "xAI",
+    configuredValue: config.xaiApiKey,
+    environmentValue: process.env.XAI_API_KEY
+  });
+}
+function normalizeDomain(value) {
+  let input = value.trim().toLowerCase();
+  if (!input)
+    return null;
+  if (input.startsWith("-"))
+    input = input.slice(1).trim();
+  if (!input)
+    return null;
+  try {
+    const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+    input = parsed.hostname;
+  } catch {
+    input = input.split("/")[0]?.split(":")[0] ?? "";
+  }
+  input = input.replace(/^\.+|\.+$/g, "");
+  return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function buildInput(query, options) {
+  const lines = [
+    "Search the web and answer using only what the web results say.",
+    "Cite your sources inline."
+  ];
+  if (options.recencyFilter) {
+    const labels = {
+      day: "past 24 hours",
+      week: "past week",
+      month: "past month",
+      year: "past year"
+    };
+    lines.push(`Prefer sources from the ${labels[options.recencyFilter]}.`);
+  }
+  if (typeof options.numResults === "number" && Number.isFinite(options.numResults) && options.numResults > 0) {
+    lines.push(`Prefer around ${Math.min(Math.floor(options.numResults), 20)} distinct sources.`);
+  }
+  const allowed = [];
+  const blocked = [];
+  for (const raw of options.domainFilter ?? []) {
+    const domain = normalizeDomain(raw);
+    if (!domain)
+      continue;
+    const target = raw.trim().startsWith("-") ? blocked : allowed;
+    if (!target.includes(domain))
+      target.push(domain);
+  }
+  if (allowed.length > 0)
+    lines.push(`Only use sources from: ${allowed.slice(0, 100).join(", ")}.`);
+  if (blocked.length > 0)
+    lines.push(`Do not use sources from: ${blocked.slice(0, 100).join(", ")}.`);
+  return `${lines.join(" ")}
+
+${query}`;
+}
+function addResult(results, seen, url, title, snippet = "") {
+  if (typeof url !== "string" || url.trim().length === 0)
+    return;
+  if (seen.has(url))
+    return;
+  seen.add(url);
+  results.push({
+    title: typeof title === "string" && title.trim().length > 0 ? title : url,
+    url,
+    snippet
+  });
+}
+function extractSnippetAround(text, start, end) {
+  if (typeof start !== "number" || typeof end !== "number" || !text)
+    return "";
+  const before = Math.max(0, start - 100);
+  const after = Math.min(text.length, end + 100);
+  const snippet = text.slice(before, after).replace(/\[([^\]]*)\]\([^)]*\)/g, "$1").trim();
+  return snippet.length > 300 ? `${snippet.slice(0, 297)}...` : snippet;
+}
+function extractAnswer(output) {
+  const parts = [];
+  for (const item of output) {
+    if (!item || typeof item !== "object" || item.type !== "message")
+      continue;
+    const content = item.content;
+    if (!Array.isArray(content))
+      continue;
+    for (const part of content) {
+      if (!part || typeof part !== "object")
+        continue;
+      const text = part.text;
+      if (typeof text === "string" && text.trim().length > 0)
+        parts.push(text);
+    }
+  }
+  return parts.join(`
+`).trim();
+}
+function extractSearchResults(output, numResults) {
+  const results = [];
+  const seenUrls = new Set;
+  for (const item of output) {
+    if (!item || typeof item !== "object" || item.type !== "message")
+      continue;
+    const content = item.content;
+    if (!Array.isArray(content))
+      continue;
+    for (const part of content) {
+      if (!part || typeof part !== "object")
+        continue;
+      const text = typeof part.text === "string" ? part.text : "";
+      const annotations = part.annotations;
+      if (!Array.isArray(annotations))
+        continue;
+      for (const annotation of annotations) {
+        if (!annotation || typeof annotation !== "object")
+          continue;
+        if (annotation.type !== "url_citation")
+          continue;
+        addResult(results, seenUrls, annotation.url, annotation.title, extractSnippetAround(text, annotation.start_index, annotation.end_index));
+      }
+    }
+  }
+  for (const item of output) {
+    if (!item || typeof item !== "object" || item.type !== "web_search_call")
+      continue;
+    const value = item;
+    const actionSources = value.action && typeof value.action === "object" ? value.action.sources : undefined;
+    for (const group of [actionSources, value.sources, value.results]) {
+      if (!Array.isArray(group))
+        continue;
+      for (const source of group) {
+        if (!source || typeof source !== "object")
+          continue;
+        const record = source;
+        addResult(results, seenUrls, record.url ?? record.source_website_url, record.title ?? record.caption);
+      }
+    }
+  }
+  if (typeof numResults === "number" && Number.isFinite(numResults) && numResults > 0) {
+    return results.slice(0, Math.min(Math.floor(numResults), 20));
+  }
+  return results;
+}
+export async function searchWithXai(query, options = {}, ctx) {
+  const auth = await resolveXaiAuth(ctx, options.signal);
+  if (!auth) {
+    throw new Error(`xAI web search unavailable. Either:
+` + `  1. Use /login to sign in with a SuperGrok or X Premium subscription
+` + `  2. Create ${CONFIG_PATH} with { "xaiApiKey": "your-key" }
+` + "  3. Set XAI_API_KEY environment variable");
+  }
+  const activityId = activityMonitor.logStart({ type: "api", query });
+  try {
+    const response = await fetch(XAI_RESPONSES_URL, {
+      method: "POST",
+      headers: {
+        ...toRequestHeaders(auth.headers),
+        Authorization: `Bearer ${auth.apiKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify({
+        model: auth.model,
+        input: buildInput(query, options),
+        tools: [{ type: "web_search" }]
+      }),
+      signal: options.signal ? AbortSignal.any([AbortSignal.timeout(SEARCH_TIMEOUT_MS), options.signal]) : AbortSignal.timeout(SEARCH_TIMEOUT_MS)
+    });
+    if (!response.ok) {
+      activityMonitor.logError(activityId, `HTTP ${response.status}`);
+      const errorText = redactCredential(await response.text(), auth.apiKey);
+      throw new Error(`xAI API error ${response.status}: ${errorText.slice(0, 300)}`);
+    }
+    let parsed;
+    try {
+      parsed = await response.json();
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      throw new Error(`xAI API returned invalid JSON: ${message}`);
+    }
+    const output = Array.isArray(parsed.output) ? parsed.output : [];
+    const answer = extractAnswer(output);
+    const results = extractSearchResults(output, options.numResults);
+    if (!answer && results.length === 0) {
+      throw new Error("xAI web_search returned no answer or sources");
+    }
+    activityMonitor.logComplete(activityId, response.status);
+    return { answer, results };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    const redactedMessage = redactCredential(message, auth.apiKey);
+    if (redactedMessage.toLowerCase().includes("abort")) {
+      activityMonitor.logComplete(activityId, 0);
+    } else {
+      activityMonitor.logError(activityId, redactedMessage);
+    }
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+}