@duckmind/dm-windows-x64 0.62.9 → 0.63.4

package/extensions/dm-web-access/bocha.js

@@ -0,0 +1,214 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const BOCHA_SEARCH_URL = "https://api.bochaai.com/v1/web-search";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 60000;
+let cachedConfig = null;
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf-8");
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+  }
+  cachedConfig = parsed;
+  return cachedConfig;
+}
+async function getApiKey(signal) {
+  return resolveCredential({
+    provider: "Bocha",
+    configuredValue: loadConfig().bochaApiKey,
+    environmentValue: process.env.BOCHA_API_KEY,
+    signal
+  });
+}
+async function requireApiKey(signal) {
+  const apiKey = await getApiKey(signal);
+  if (!apiKey) {
+    throw new Error(`Bocha API key not found. Either:
+` + `  1. Create ${CONFIG_PATH} with { "bochaApiKey": "your-key" }
+` + `  2. Set BOCHA_API_KEY environment variable
+` + "Create a key at https://open.bochaai.com/");
+  }
+  return apiKey;
+}
+function normalizeCount(value) {
+  if (typeof value !== "number" || !Number.isFinite(value))
+    return 8;
+  return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function mapFreshness(value) {
+  switch (value) {
+    case "day":
+      return "oneDay";
+    case "week":
+      return "oneWeek";
+    case "month":
+      return "oneMonth";
+    case "year":
+      return "oneYear";
+    default:
+      return "noLimit";
+  }
+}
+function normalizeDomain(value) {
+  let input = value.trim().toLowerCase();
+  if (!input)
+    return null;
+  if (input.startsWith("-"))
+    input = input.slice(1).trim();
+  if (!input)
+    return null;
+  try {
+    const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+    input = parsed.hostname;
+  } catch {
+    input = input.split("/")[0]?.split(":")[0] ?? "";
+  }
+  input = input.replace(/^\.+|\.+$/g, "");
+  return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function parseDomainFilter(domainFilter) {
+  const filters = { include: [], exclude: [] };
+  for (const raw of domainFilter ?? []) {
+    const domain = normalizeDomain(raw);
+    if (!domain)
+      continue;
+    const target = raw.trim().startsWith("-") ? filters.exclude : filters.include;
+    if (!target.includes(domain))
+      target.push(domain);
+  }
+  return filters;
+}
+function passesDomainFilters(url, filters) {
+  if (filters.include.length === 0 && filters.exclude.length === 0)
+    return true;
+  let hostname;
+  try {
+    hostname = new URL(url).hostname.toLowerCase();
+  } catch {
+    return false;
+  }
+  const matches = (domain) => hostname === domain || hostname.endsWith(`.${domain}`);
+  if (filters.exclude.some(matches))
+    return false;
+  return filters.include.length === 0 || filters.include.some(matches);
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function invalidResponse(message) {
+  return new Error(`Bocha API returned invalid response: ${message}`);
+}
+function firstString(...values) {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim())
+      return value.trim();
+  }
+  return null;
+}
+function parseSearchResponse(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value))
+    throw invalidResponse("expected an object envelope");
+  const envelope = value;
+  if (envelope.code !== undefined && Number(envelope.code) !== 200) {
+    throw invalidResponse(`code ${String(envelope.code)}: ${firstString(envelope.msg) ?? "unknown error"}`);
+  }
+  const data = envelope.data;
+  const pages = typeof data === "object" && data !== null && !Array.isArray(data) ? data.webPages : undefined;
+  const items = typeof pages === "object" && pages !== null && !Array.isArray(pages) ? pages.value : undefined;
+  if (!Array.isArray(items))
+    throw invalidResponse("missing data.webPages.value array");
+  const results = [];
+  for (const item of items) {
+    if (!item || typeof item !== "object" || Array.isArray(item))
+      continue;
+    const entry = item;
+    const url = firstString(entry.url, entry.link, entry.href);
+    if (!url)
+      continue;
+    const title = firstString(entry.title, entry.name) ?? url;
+    const snippet = firstString(entry.summary, entry.snippet, entry.description, entry.content) ?? "";
+    results.push({ title, url, snippet });
+  }
+  return { results };
+}
+function buildAnswer(results) {
+  return results.map((result) => result.snippet ? `${result.snippet}
+Source: ${result.title} (${result.url})` : `Source: ${result.title} (${result.url})`).join(`
+
+`);
+}
+export function isBochaAvailable() {
+  return hasCredentialSource({ provider: "Bocha", configuredValue: loadConfig().bochaApiKey, environmentValue: process.env.BOCHA_API_KEY });
+}
+export async function searchWithBocha(query, options = {}) {
+  const apiKey = await requireApiKey(options.signal);
+  const numResults = normalizeCount(options.numResults);
+  const filters = parseDomainFilter(options.domainFilter);
+  const activityId = activityMonitor.logStart({ type: "api", query });
+  let response;
+  try {
+    response = await fetch(BOCHA_SEARCH_URL, {
+      method: "POST",
+      headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json", Accept: "application/json" },
+      body: JSON.stringify({ query, count: numResults, freshness: mapFreshness(options.recencyFilter), summary: true }),
+      signal: options.signal ? AbortSignal.any([AbortSignal.timeout(SEARCH_TIMEOUT_MS), options.signal]) : AbortSignal.timeout(SEARCH_TIMEOUT_MS)
+    });
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (redactedMessage.toLowerCase().includes("abort"))
+      activityMonitor.logComplete(activityId, 0);
+    else
+      activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+  if (!response.ok) {
+    activityMonitor.logComplete(activityId, response.status);
+    const errorText = redactCredential(await response.text(), apiKey);
+    throw new Error(`Bocha API error ${response.status}: ${errorText.slice(0, 300)}`);
+  }
+  let rawData;
+  try {
+    rawData = await response.json();
+  } catch (err) {
+    activityMonitor.logComplete(activityId, response.status);
+    throw new Error(`Bocha API returned invalid JSON: ${errorMessage(err)}`);
+  }
+  let parsed;
+  try {
+    parsed = parseSearchResponse(rawData);
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+  activityMonitor.logComplete(activityId, response.status);
+  const results = parsed.results.filter((result) => passesDomainFilters(result.url, filters)).slice(0, numResults);
+  return { answer: buildAnswer(results), results };
+}

package/extensions/dm-web-access/brave.js

@@ -0,0 +1,196 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { fetchWithCredentialRedirects, getWebSearchConfigPath, resolveApiBaseUrl } from "./utils.js";
+const BRAVE_API_BASE_URL = "https://api.search.brave.com/res/v1";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 30000;
+let cachedConfig = null;
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf-8");
+  try {
+    cachedConfig = JSON.parse(raw);
+    return cachedConfig;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+}
+async function getApiKey(signal) {
+  return resolveCredential({
+    provider: "Brave",
+    configuredValue: loadConfig().braveApiKey,
+    environmentValue: process.env.BRAVE_API_KEY,
+    signal
+  });
+}
+function getApiUrl() {
+  return `${resolveApiBaseUrl({
+    configKey: "braveBaseUrl",
+    configuredValue: loadConfig().braveBaseUrl,
+    defaultValue: BRAVE_API_BASE_URL,
+    environmentKey: "BRAVE_BASE_URL",
+    environmentValue: process.env.BRAVE_BASE_URL
+  })}/web/search`;
+}
+function normalizeCount(value) {
+  if (typeof value !== "number" || !Number.isFinite(value))
+    return 5;
+  return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function normalizeDomain(value) {
+  let input = value.trim().toLowerCase();
+  if (!input)
+    return null;
+  if (input.startsWith("-"))
+    input = input.slice(1).trim();
+  if (!input)
+    return null;
+  try {
+    const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+    input = parsed.hostname;
+  } catch {
+    input = input.split("/")[0]?.split(":")[0] ?? "";
+  }
+  input = input.replace(/^\.+|\.+$/g, "");
+  return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function normalizeDomainFilters(domainFilter) {
+  const filters = { allowed: [], blocked: [] };
+  if (!domainFilter?.length)
+    return filters;
+  for (const raw of domainFilter) {
+    const domain = normalizeDomain(raw);
+    if (!domain)
+      continue;
+    const target = raw.trim().startsWith("-") ? filters.blocked : filters.allowed;
+    if (!target.includes(domain))
+      target.push(domain);
+  }
+  return filters;
+}
+function buildBraveQuery(query, domainFilter) {
+  const filters = normalizeDomainFilters(domainFilter);
+  const parts = [query];
+  if (filters.allowed.length === 1) {
+    parts.push(`site:${filters.allowed[0]}`);
+  } else if (filters.allowed.length > 1) {
+    parts.push(filters.allowed.map((domain) => `site:${domain}`).join(" OR "));
+  }
+  for (const domain of filters.blocked) {
+    parts.push(`NOT site:${domain}`);
+  }
+  return parts.join(" ");
+}
+function hostMatchesDomain(hostname, domain) {
+  return hostname === domain || hostname.endsWith(`.${domain}`);
+}
+function matchesDomainFilters(url, filters) {
+  if (filters.allowed.length === 0 && filters.blocked.length === 0)
+    return true;
+  let hostname = "";
+  try {
+    hostname = new URL(url).hostname.toLowerCase();
+  } catch {
+    return false;
+  }
+  if (filters.allowed.length > 0 && !filters.allowed.some((domain) => hostMatchesDomain(hostname, domain))) {
+    return false;
+  }
+  return !filters.blocked.some((domain) => hostMatchesDomain(hostname, domain));
+}
+export function isBraveAvailable() {
+  return hasCredentialSource({
+    provider: "Brave",
+    configuredValue: loadConfig().braveApiKey,
+    environmentValue: process.env.BRAVE_API_KEY
+  });
+}
+export async function searchWithBrave(query, options = {}) {
+  const apiUrl = getApiUrl();
+  const apiKey = await getApiKey(options.signal);
+  if (!apiKey) {
+    throw new Error(`Brave Search API key not found. Either:
+` + `  1. Create ${CONFIG_PATH} with { "braveApiKey": "your-key" }
+` + `  2. Set BRAVE_API_KEY environment variable
+` + "Get a key at https://brave.com/search/api/");
+  }
+  const numResults = normalizeCount(options.numResults);
+  const domainFilters = normalizeDomainFilters(options.domainFilter);
+  const searchQuery = buildBraveQuery(query, options.domainFilter);
+  const activityId = activityMonitor.logStart({ type: "api", query: searchQuery });
+  const params = new URLSearchParams({
+    q: searchQuery,
+    count: String(options.domainFilter?.length ? 20 : numResults)
+  });
+  if (options.recencyFilter) {
+    const freshnessMap = {
+      day: "pd",
+      week: "pw",
+      month: "pm",
+      year: "py"
+    };
+    const freshness = freshnessMap[options.recencyFilter];
+    if (freshness)
+      params.set("freshness", freshness);
+  }
+  try {
+    const response = await fetchWithCredentialRedirects(`${apiUrl}?${params.toString()}`, {
+      method: "GET",
+      headers: {
+        "X-Subscription-Token": apiKey,
+        Accept: "application/json",
+        "Accept-Encoding": "gzip"
+      },
+      signal: options.signal ? AbortSignal.any([AbortSignal.timeout(SEARCH_TIMEOUT_MS), options.signal]) : AbortSignal.timeout(SEARCH_TIMEOUT_MS)
+    }, ["X-Subscription-Token"]);
+    if (!response.ok) {
+      activityMonitor.logError(activityId, `HTTP ${response.status}`);
+      const errorText = redactCredential(await response.text(), apiKey);
+      throw new Error(`Brave Search API error ${response.status}: ${errorText.slice(0, 300)}`);
+    }
+    const data = await response.json();
+    activityMonitor.logComplete(activityId, response.status);
+    const results = [];
+    for (const item of data.web?.results ?? []) {
+      if (!item.url || !matchesDomainFilters(item.url, domainFilters))
+        continue;
+      results.push({
+        title: item.title || item.url,
+        url: item.url,
+        snippet: item.description || ""
+      });
+      if (results.length >= numResults)
+        break;
+    }
+    const answer = results.map((result) => {
+      if (result.snippet)
+        return `${result.snippet}
+Source: ${result.title} (${result.url})`;
+      return `Source: ${result.title} (${result.url})`;
+    }).join(`
+
+`);
+    return { answer, results };
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (redactedMessage.toLowerCase().includes("abort")) {
+      activityMonitor.logComplete(activityId, 0);
+    } else {
+      activityMonitor.logError(activityId, redactedMessage);
+    }
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+}

package/extensions/dm-web-access/brightdata-unlocker.js

@@ -0,0 +1,202 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { validateRemoteUrl } from "./ssrf-protection.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const CONFIG_PATH = getWebSearchConfigPath();
+const BRIGHTDATA_REQUEST_URL = "https://api.brightdata.com/request";
+const EXTRACT_TIMEOUT_MS = 60000;
+const DEFAULT_MAX_REDIRECTS = 5;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const ZONE_PATTERN = /^[a-z0-9_-]+$/i;
+let cachedConfig = null;
+function parseFailureDetail(err) {
+  const message = err instanceof Error ? err.message : String(err);
+  const position = message.match(/at position \d+(?: \(line \d+ column \d+\))?/);
+  return position ? `invalid JSON ${position[0]}` : "invalid JSON";
+}
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf8");
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${parseFailureDetail(err)}`);
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+  }
+  cachedConfig = parsed;
+  return cachedConfig;
+}
+export function clearBrightDataUnlockerConfigCache() {
+  cachedConfig = null;
+}
+function normalizeZone(value) {
+  if (typeof value !== "string")
+    return null;
+  const trimmed = value.trim();
+  if (!trimmed)
+    return null;
+  return ZONE_PATTERN.test(trimmed) ? trimmed : null;
+}
+function zoneSetting() {
+  const fromEnv = process.env.BRIGHTDATA_UNLOCKER_ZONE;
+  if (typeof fromEnv === "string" && fromEnv.trim())
+    return { raw: fromEnv.trim(), label: "BRIGHTDATA_UNLOCKER_ZONE" };
+  const configured = loadConfig().brightdataUnlockerZone;
+  if (typeof configured === "string" && configured.trim())
+    return { raw: configured.trim(), label: `brightdataUnlockerZone in ${CONFIG_PATH}` };
+  return null;
+}
+function getZone() {
+  const setting = zoneSetting();
+  return setting ? normalizeZone(setting.raw) : null;
+}
+function requireZone() {
+  const setting = zoneSetting();
+  const zone = setting ? normalizeZone(setting.raw) : null;
+  if (zone)
+    return zone;
+  if (setting) {
+    throw new Error(`Invalid Bright Data Unlocker zone: ${setting.label} must be a zone name of letters, digits, "-", or "_". ` + 'The zone must be of type "unblocker"; a SERP zone will not serve Web Unlocker requests.');
+  }
+  throw new Error(`Bright Data Web Unlocker zone not configured. Either:
+` + `  1. Set brightdataUnlockerZone in ${CONFIG_PATH}
+` + `  2. Set BRIGHTDATA_UNLOCKER_ZONE environment variable
+` + 'The zone must be of type "unblocker"; a SERP zone will not serve Web Unlocker requests.');
+}
+async function getApiKey(signal) {
+  const apiKey = await resolveCredential({
+    provider: "Bright Data",
+    configuredValue: loadConfig().brightdataApiKey,
+    environmentValue: process.env.BRIGHTDATA_API_KEY,
+    signal
+  });
+  if (!apiKey) {
+    throw new Error(`Bright Data API key not found. Either:
+` + `  1. Create ${CONFIG_PATH} with { "brightdataApiKey": "your-key" }
+` + `  2. Set BRIGHTDATA_API_KEY environment variable
+` + "Get a key at https://brightdata.com/cp/setting/users");
+  }
+  return apiKey;
+}
+function requestSignal(timeoutMs, signal) {
+  const timeout = AbortSignal.timeout(timeoutMs);
+  return signal ? AbortSignal.any([timeout, signal]) : timeout;
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function isAbortError(err) {
+  return errorMessage(err).toLowerCase().includes("abort");
+}
+function ssrfOptions(options) {
+  return {
+    allowRanges: options?.ssrf?.allowRanges ?? [],
+    trustEnvProxy: options?.ssrf?.trustEnvProxy ?? false,
+    ...options?.lookup ? { lookup: options.lookup } : {}
+  };
+}
+function withoutSensitiveHeaders(headers) {
+  const next = { ...headers };
+  delete next.Authorization;
+  delete next.authorization;
+  delete next.Cookie;
+  delete next.cookie;
+  delete next["X-API-Key"];
+  delete next["x-api-key"];
+  return next;
+}
+async function fetchBrightDataApi(url, init, options) {
+  let current = await validateRemoteUrl(url, ssrfOptions(options));
+  let headers = init.headers;
+  for (let redirects = 0;; redirects++) {
+    const response = await fetch(current, { ...init, headers, redirect: "manual" });
+    if (!REDIRECT_STATUSES.has(response.status))
+      return response;
+    const location = response.headers.get("location");
+    if (!location)
+      return response;
+    if (redirects === DEFAULT_MAX_REDIRECTS)
+      throw new Error(`Too many redirects fetching ${current.toString()}`);
+    const next = await validateRemoteUrl(new URL(location, current), ssrfOptions(options));
+    if (next.origin !== current.origin)
+      headers = withoutSensitiveHeaders(headers);
+    current = next;
+  }
+}
+function unlockerBody(url, zone) {
+  return {
+    url,
+    zone,
+    format: "raw",
+    data_format: "markdown"
+  };
+}
+async function brightDataRequest(url, zone, signal, options) {
+  const apiKey = await getApiKey(signal);
+  const headers = {
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${apiKey}`
+  };
+  const activityId = activityMonitor.logStart({ type: "fetch", url });
+  try {
+    const response = await fetchBrightDataApi(BRIGHTDATA_REQUEST_URL, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(unlockerBody(url, zone)),
+      signal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal)
+    }, options);
+    if (!response.ok) {
+      const errorText = await response.text().catch(() => "");
+      throw new Error(`Bright Data Web Unlocker error ${response.status}: ${redactCredential(errorText, apiKey).slice(0, 300)}`);
+    }
+    const text = await response.text();
+    activityMonitor.logComplete(activityId, response.status);
+    return text;
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (isAbortError(err))
+      activityMonitor.logComplete(activityId, 0);
+    else
+      activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+}
+function headingTitle(text) {
+  const match = text.match(/^#{1,2}\s+(.+)/m);
+  if (!match)
+    return "";
+  return match[1].replace(/\*+/g, "").trim();
+}
+export function isBrightDataUnlockerAvailable() {
+  if (getZone() === null)
+    return false;
+  return hasCredentialSource({
+    provider: "Bright Data",
+    configuredValue: loadConfig().brightdataApiKey,
+    environmentValue: process.env.BRIGHTDATA_API_KEY
+  });
+}
+export async function extractWithBrightDataUnlocker(url, signal, options) {
+  const zone = requireZone();
+  await validateRemoteUrl(url, ssrfOptions(options));
+  const raw = await brightDataRequest(url, zone, signal, options);
+  const content = raw.trim();
+  if (!content)
+    return null;
+  return { url, title: headingTitle(content), content, error: null };
+}