@duckmind/dm-windows-x64 0.63.0 → 0.63.4

package/extensions/dm-web-access/jina-search.js

@@ -0,0 +1,242 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const JINA_SEARCH_BASE_URL = "https://s.jina.ai/";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 60000;
+let cachedConfig = null;
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf8");
+  try {
+    const parsed = JSON.parse(raw);
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+      throw new Error("expected a JSON object");
+    }
+    cachedConfig = parsed;
+    return cachedConfig;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+}
+async function getApiKey(signal) {
+  return resolveCredential({
+    provider: "Jina Search",
+    configuredValue: loadConfig().jinaApiKey,
+    environmentValue: process.env.JINA_API_KEY,
+    signal
+  });
+}
+async function requireApiKey(signal) {
+  const apiKey = await getApiKey(signal);
+  if (!apiKey) {
+    throw new Error(`Jina Search API key not found. Either:
+` + `  1. Create ${CONFIG_PATH} with { "jinaApiKey": "your-key" }
+` + `  2. Set JINA_API_KEY environment variable
+` + "Get a key at https://jina.ai/api-dashboard");
+  }
+  return apiKey;
+}
+function normalizeCount(value) {
+  if (typeof value !== "number" || !Number.isFinite(value))
+    return 5;
+  return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function normalizeDomain(value) {
+  let input = value.trim().toLowerCase();
+  if (input.startsWith("-"))
+    input = input.slice(1).trim();
+  if (!input)
+    return null;
+  try {
+    const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+    input = parsed.hostname;
+  } catch {
+    input = input.split("/")[0]?.split(":")[0] ?? "";
+  }
+  input = input.replace(/^\.+|\.+$/g, "");
+  return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function mapDomainFilter(domainFilter) {
+  const includes = [];
+  const excludes = [];
+  for (const raw of domainFilter ?? []) {
+    const domain = normalizeDomain(raw);
+    if (!domain)
+      continue;
+    const target = raw.trim().startsWith("-") ? excludes : includes;
+    if (!target.includes(domain))
+      target.push(domain);
+  }
+  return { includes, excludes };
+}
+function buildSearchRequest(query, options, numResults) {
+  const filters = mapDomainFilter(options.domainFilter);
+  const recency = options.recencyFilter ? ` published in the past ${options.recencyFilter}` : "";
+  const exclusions = filters.excludes.map((domain) => ` -site:${domain}`).join("");
+  const constrainedQuery = `${query.trim()}${exclusions}${recency}`.trim();
+  const url = new URL(encodeURIComponent(constrainedQuery), JINA_SEARCH_BASE_URL);
+  url.searchParams.set("count", String(numResults));
+  for (const domain of filters.includes)
+    url.searchParams.append("site", domain);
+  return { url: url.toString(), filters };
+}
+function requestSignal(signal) {
+  const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+  return {
+    timeout,
+    signal: signal ? AbortSignal.any([signal, timeout]) : timeout
+  };
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function parseItems(value) {
+  if (Array.isArray(value))
+    return value;
+  if (!value || typeof value !== "object") {
+    throw new Error("Jina Search API returned invalid response: expected an object or array");
+  }
+  const envelope = value;
+  if (typeof envelope.code === "number" && envelope.code !== 200) {
+    throw new Error(`Jina Search API error ${envelope.code}: response envelope reported failure`);
+  }
+  if (!Array.isArray(envelope.data)) {
+    throw new Error("Jina Search API returned invalid response: expected data array");
+  }
+  return envelope.data;
+}
+function passesDomainFilter(url, filters) {
+  let hostname;
+  try {
+    hostname = new URL(url).hostname.toLowerCase();
+  } catch {
+    return false;
+  }
+  const matches = (domain) => hostname === domain || hostname.endsWith(`.${domain}`);
+  if (filters.includes.length > 0 && !filters.includes.some(matches))
+    return false;
+  return !filters.excludes.some(matches);
+}
+function mapItems(items, numResults, filters) {
+  const results = [];
+  const content = [];
+  const seen = new Set;
+  for (const item of items) {
+    const url = typeof item?.url === "string" ? item.url.trim() : "";
+    if (!url || seen.has(url) || !passesDomainFilter(url, filters))
+      continue;
+    seen.add(url);
+    const title = typeof item.title === "string" && item.title.trim() ? item.title.trim() : `Source ${results.length + 1}`;
+    const snippet = typeof item.description === "string" ? item.description.replace(/\s+/g, " ").trim() : "";
+    results.push({ title, url, snippet });
+    if (typeof item.content === "string" && item.content.trim()) {
+      content.push({ url, title, content: item.content.trim(), error: null });
+    }
+    if (results.length >= numResults)
+      break;
+  }
+  return { results, content };
+}
+function buildAnswer(results) {
+  return results.map((result) => result.snippet ? `${result.snippet}
+Source: ${result.title} (${result.url})` : `Source: ${result.title} (${result.url})`).join(`
+
+`);
+}
+function rethrowRequestError(err, apiKey, activityId, request, callerSignal) {
+  if (request.timeout.aborted && !callerSignal?.aborted) {
+    activityMonitor.logComplete(activityId, 408);
+    throw new Error("Jina Search API error 408: request timed out");
+  }
+  const message = errorMessage(err);
+  const redactedMessage = redactCredential(message, apiKey);
+  if (redactedMessage.toLowerCase().includes("abort"))
+    activityMonitor.logComplete(activityId, 0);
+  else
+    activityMonitor.logError(activityId, redactedMessage);
+  if (redactedMessage === message)
+    throw err;
+  const redactedError = new Error(redactedMessage);
+  if (err instanceof Error)
+    redactedError.name = err.name;
+  throw redactedError;
+}
+export function isJinaSearchAvailable() {
+  return hasCredentialSource({
+    provider: "Jina Search",
+    configuredValue: loadConfig().jinaApiKey,
+    environmentValue: process.env.JINA_API_KEY
+  });
+}
+export async function searchWithJina(query, options = {}) {
+  const apiKey = await requireApiKey(options.signal);
+  const numResults = normalizeCount(options.numResults);
+  const { url, filters } = buildSearchRequest(query, options, numResults);
+  const activityId = activityMonitor.logStart({ type: "api", query });
+  const request = requestSignal(options.signal);
+  let response;
+  try {
+    response = await fetch(url, {
+      headers: {
+        Accept: "application/json",
+        Authorization: `Bearer ${apiKey}`,
+        "User-Agent": "dm-web-access",
+        "X-Respond-With": options.includeContent ? "content" : "no-content",
+        "X-Retain-Images": "none"
+      },
+      signal: request.signal
+    });
+  } catch (err) {
+    rethrowRequestError(err, apiKey, activityId, request, options.signal);
+  }
+  if (!response.ok) {
+    let body;
+    try {
+      body = redactCredential(await response.text(), apiKey);
+    } catch (err) {
+      rethrowRequestError(err, apiKey, activityId, request, options.signal);
+    }
+    activityMonitor.logComplete(activityId, response.status);
+    throw new Error(`Jina Search API error ${response.status}: ${body.slice(0, 300)}`);
+  }
+  let raw;
+  try {
+    raw = await response.json();
+  } catch (err) {
+    if (!(err instanceof SyntaxError)) {
+      rethrowRequestError(err, apiKey, activityId, request, options.signal);
+    }
+    activityMonitor.logComplete(activityId, response.status);
+    throw new Error("Jina Search API returned invalid JSON");
+  }
+  let mapped;
+  try {
+    mapped = mapItems(parseItems(raw), numResults, filters);
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+  activityMonitor.logComplete(activityId, response.status);
+  const result = {
+    answer: buildAnswer(mapped.results),
+    results: mapped.results
+  };
+  if (options.includeContent && mapped.content.length > 0)
+    result.inlineContent = mapped.content;
+  return result;
+}

package/extensions/dm-web-access/kagi.js

@@ -0,0 +1,255 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { fetchRemoteUrl, loadFetchContentDomainPolicy, loadSsrfConfig, validateRemoteUrl } from "./ssrf-protection.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const KAGI_SEARCH_URL = "https://kagi.com/api/v1/search";
+const KAGI_EXTRACT_URL = "https://kagi.com/api/v1/extract";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 60000;
+let cachedConfig = null;
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf-8");
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+  }
+  cachedConfig = parsed;
+  return cachedConfig;
+}
+async function getApiKey(signal) {
+  return resolveCredential({
+    provider: "Kagi",
+    configuredValue: loadConfig().kagiApiKey,
+    environmentValue: process.env.KAGI_API_KEY,
+    signal
+  });
+}
+async function requireApiKey(signal) {
+  const apiKey = await getApiKey(signal);
+  if (!apiKey) {
+    throw new Error(`Kagi API key not found. Either:
+` + `  1. Create ${CONFIG_PATH} with { "kagiApiKey": "your-key" }
+` + `  2. Set KAGI_API_KEY environment variable
+` + "Create a key at https://kagi.com/settings?p=api");
+  }
+  return apiKey;
+}
+function normalizeCount(value) {
+  if (typeof value !== "number" || !Number.isFinite(value))
+    return 5;
+  return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function invalidResponse(message) {
+  return new Error(`Kagi API returned invalid response: ${message}`);
+}
+function firstString(...values) {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim())
+      return value.trim();
+  }
+  return null;
+}
+function appendSearchItems(value, results, inlineContent) {
+  if (Array.isArray(value)) {
+    for (const item of value)
+      appendSearchItems(item, results, inlineContent);
+    return;
+  }
+  if (!value || typeof value !== "object")
+    return;
+  const item = value;
+  const url = firstString(item.url, item.href, item.link);
+  if (!url)
+    return;
+  const title = firstString(item.title, item.name) ?? url;
+  const snippet = firstString(item.snippet, item.description, item.summary, item.content, item.markdown, item.text) ?? "";
+  results.push({ title, url, snippet });
+  const content = firstString(item.markdown, item.content, item.text);
+  if (content)
+    inlineContent.push({ url, title, content, error: null });
+}
+function parseErrors(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    return null;
+  const envelope = value;
+  const rawErrors = envelope.errors ?? envelope.error;
+  if (!Array.isArray(rawErrors))
+    return null;
+  const messages = rawErrors.map((entry) => {
+    if (!entry || typeof entry !== "object")
+      return String(entry);
+    const raw = entry;
+    return firstString(raw.message, raw.msg, raw.code) ?? JSON.stringify(raw);
+  });
+  return messages.length > 0 ? messages.join("; ") : null;
+}
+function parseSearchResponse(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw invalidResponse("expected an object envelope");
+  const message = parseErrors(value);
+  if (message)
+    throw invalidResponse(message);
+  const envelope = value;
+  const results = [];
+  const inlineContent = [];
+  const data = envelope.data;
+  const items = typeof data === "object" && data !== null && !Array.isArray(data) ? data.search : data;
+  appendSearchItems(items, results, inlineContent);
+  return { results, inlineContent };
+}
+function parseExtractResponse(value, requestedUrl) {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw invalidResponse("expected extract object envelope");
+  const message = parseErrors(value);
+  if (message)
+    throw invalidResponse(message);
+  const envelope = value;
+  const candidates = Array.isArray(envelope.data) ? envelope.data : [envelope.data ?? envelope];
+  for (const candidate of candidates) {
+    if (!candidate || typeof candidate !== "object" || Array.isArray(candidate))
+      continue;
+    const item = candidate;
+    const content = firstString(item.markdown, item.content, item.text);
+    if (!content)
+      continue;
+    return {
+      url: firstString(item.url, item.href, item.link) ?? requestedUrl,
+      title: firstString(item.title, item.name) ?? requestedUrl,
+      content,
+      error: null
+    };
+  }
+  return null;
+}
+function buildAnswer(results) {
+  return results.map((result) => result.snippet ? `${result.snippet}
+Source: ${result.title} (${result.url})` : `Source: ${result.title} (${result.url})`).join(`
+
+`);
+}
+export function isKagiAvailable() {
+  return hasCredentialSource({ provider: "Kagi", configuredValue: loadConfig().kagiApiKey, environmentValue: process.env.KAGI_API_KEY });
+}
+export async function searchWithKagi(query, options = {}) {
+  const apiKey = await requireApiKey(options.signal);
+  const numResults = normalizeCount(options.numResults);
+  const activityId = activityMonitor.logStart({ type: "api", query });
+  let response;
+  try {
+    response = await fetch(KAGI_SEARCH_URL, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", Accept: "application/json" },
+      body: JSON.stringify({ query, limit: numResults }),
+      signal: options.signal ? AbortSignal.any([AbortSignal.timeout(SEARCH_TIMEOUT_MS), options.signal]) : AbortSignal.timeout(SEARCH_TIMEOUT_MS)
+    });
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (redactedMessage.toLowerCase().includes("abort"))
+      activityMonitor.logComplete(activityId, 0);
+    else
+      activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+  if (!response.ok) {
+    activityMonitor.logComplete(activityId, response.status);
+    const errorText = redactCredential(await response.text(), apiKey);
+    throw new Error(`Kagi API error ${response.status}: ${errorText.slice(0, 300)}`);
+  }
+  let rawData;
+  try {
+    rawData = await response.json();
+  } catch (err) {
+    activityMonitor.logComplete(activityId, response.status);
+    throw new Error(`Kagi API returned invalid JSON: ${errorMessage(err)}`);
+  }
+  const parsed = parseSearchResponse(rawData);
+  activityMonitor.logComplete(activityId, response.status);
+  const results = parsed.results.slice(0, numResults);
+  const mapped = { answer: buildAnswer(results), results };
+  if (options.includeContent) {
+    const urls = new Set(results.map((result) => result.url));
+    const inlineContent = parsed.inlineContent.filter((content) => urls.has(content.url));
+    if (inlineContent.length > 0)
+      mapped.inlineContent = inlineContent;
+  }
+  return mapped;
+}
+export function isKagiExtractAvailable() {
+  return isKagiAvailable();
+}
+export async function extractWithKagi(url, signal, options = {}) {
+  const ssrf = options.ssrf ?? loadSsrfConfig();
+  const domainPolicy = loadFetchContentDomainPolicy();
+  await validateRemoteUrl(url, {
+    allowRanges: ssrf.allowRanges,
+    trustEnvProxy: ssrf.trustEnvProxy,
+    domainPolicy,
+    ...options.lookup ? { lookup: options.lookup } : {}
+  });
+  const apiKey = await requireApiKey(signal);
+  const activityId = activityMonitor.logStart({ type: "api", query: `kagi extract: ${url}` });
+  let response;
+  try {
+    response = await fetchRemoteUrl(KAGI_EXTRACT_URL, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", Accept: "application/json" },
+      body: JSON.stringify({ pages: [{ url }] }),
+      signal: signal ? AbortSignal.any([AbortSignal.timeout(options.timeoutMs ?? SEARCH_TIMEOUT_MS), signal]) : AbortSignal.timeout(options.timeoutMs ?? SEARCH_TIMEOUT_MS)
+    }, {
+      allowRanges: ssrf.allowRanges,
+      trustEnvProxy: ssrf.trustEnvProxy,
+      onRedirect: ({ from, to, init }) => to.origin === from.origin ? init : { ...init, headers: { "Content-Type": "application/json", Accept: "application/json" } },
+      ...options.lookup ? { lookup: options.lookup } : {}
+    });
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (redactedMessage.toLowerCase().includes("abort"))
+      activityMonitor.logComplete(activityId, 0);
+    else
+      activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+  if (!response.ok) {
+    activityMonitor.logComplete(activityId, response.status);
+    const errorText = redactCredential(await response.text(), apiKey);
+    throw new Error(`Kagi Extract API error ${response.status}: ${errorText.slice(0, 300)}`);
+  }
+  let rawData;
+  try {
+    rawData = await response.json();
+  } catch (err) {
+    activityMonitor.logComplete(activityId, response.status);
+    throw new Error(`Kagi Extract API returned invalid JSON: ${errorMessage(err)}`);
+  }
+  const parsed = parseExtractResponse(rawData, url);
+  activityMonitor.logComplete(activityId, response.status);
+  return parsed;
+}