@duckmind/dm-windows-x64 0.63.0 → 0.63.6

package/extensions/dm-web-access/brightdata-unlocker.js

@@ -0,0 +1,202 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { validateRemoteUrl } from "./ssrf-protection.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const CONFIG_PATH = getWebSearchConfigPath();
+const BRIGHTDATA_REQUEST_URL = "https://api.brightdata.com/request";
+const EXTRACT_TIMEOUT_MS = 60000;
+const DEFAULT_MAX_REDIRECTS = 5;
+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
+const ZONE_PATTERN = /^[a-z0-9_-]+$/i;
+let cachedConfig = null;
+function parseFailureDetail(err) {
+  const message = err instanceof Error ? err.message : String(err);
+  const position = message.match(/at position \d+(?: \(line \d+ column \d+\))?/);
+  return position ? `invalid JSON ${position[0]}` : "invalid JSON";
+}
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf8");
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${parseFailureDetail(err)}`);
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+  }
+  cachedConfig = parsed;
+  return cachedConfig;
+}
+export function clearBrightDataUnlockerConfigCache() {
+  cachedConfig = null;
+}
+function normalizeZone(value) {
+  if (typeof value !== "string")
+    return null;
+  const trimmed = value.trim();
+  if (!trimmed)
+    return null;
+  return ZONE_PATTERN.test(trimmed) ? trimmed : null;
+}
+function zoneSetting() {
+  const fromEnv = process.env.BRIGHTDATA_UNLOCKER_ZONE;
+  if (typeof fromEnv === "string" && fromEnv.trim())
+    return { raw: fromEnv.trim(), label: "BRIGHTDATA_UNLOCKER_ZONE" };
+  const configured = loadConfig().brightdataUnlockerZone;
+  if (typeof configured === "string" && configured.trim())
+    return { raw: configured.trim(), label: `brightdataUnlockerZone in ${CONFIG_PATH}` };
+  return null;
+}
+function getZone() {
+  const setting = zoneSetting();
+  return setting ? normalizeZone(setting.raw) : null;
+}
+function requireZone() {
+  const setting = zoneSetting();
+  const zone = setting ? normalizeZone(setting.raw) : null;
+  if (zone)
+    return zone;
+  if (setting) {
+    throw new Error(`Invalid Bright Data Unlocker zone: ${setting.label} must be a zone name of letters, digits, "-", or "_". ` + 'The zone must be of type "unblocker"; a SERP zone will not serve Web Unlocker requests.');
+  }
+  throw new Error(`Bright Data Web Unlocker zone not configured. Either:
+` + `  1. Set brightdataUnlockerZone in ${CONFIG_PATH}
+` + `  2. Set BRIGHTDATA_UNLOCKER_ZONE environment variable
+` + 'The zone must be of type "unblocker"; a SERP zone will not serve Web Unlocker requests.');
+}
+async function getApiKey(signal) {
+  const apiKey = await resolveCredential({
+    provider: "Bright Data",
+    configuredValue: loadConfig().brightdataApiKey,
+    environmentValue: process.env.BRIGHTDATA_API_KEY,
+    signal
+  });
+  if (!apiKey) {
+    throw new Error(`Bright Data API key not found. Either:
+` + `  1. Create ${CONFIG_PATH} with { "brightdataApiKey": "your-key" }
+` + `  2. Set BRIGHTDATA_API_KEY environment variable
+` + "Get a key at https://brightdata.com/cp/setting/users");
+  }
+  return apiKey;
+}
+function requestSignal(timeoutMs, signal) {
+  const timeout = AbortSignal.timeout(timeoutMs);
+  return signal ? AbortSignal.any([timeout, signal]) : timeout;
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function isAbortError(err) {
+  return errorMessage(err).toLowerCase().includes("abort");
+}
+function ssrfOptions(options) {
+  return {
+    allowRanges: options?.ssrf?.allowRanges ?? [],
+    trustEnvProxy: options?.ssrf?.trustEnvProxy ?? false,
+    ...options?.lookup ? { lookup: options.lookup } : {}
+  };
+}
+function withoutSensitiveHeaders(headers) {
+  const next = { ...headers };
+  delete next.Authorization;
+  delete next.authorization;
+  delete next.Cookie;
+  delete next.cookie;
+  delete next["X-API-Key"];
+  delete next["x-api-key"];
+  return next;
+}
+async function fetchBrightDataApi(url, init, options) {
+  let current = await validateRemoteUrl(url, ssrfOptions(options));
+  let headers = init.headers;
+  for (let redirects = 0;; redirects++) {
+    const response = await fetch(current, { ...init, headers, redirect: "manual" });
+    if (!REDIRECT_STATUSES.has(response.status))
+      return response;
+    const location = response.headers.get("location");
+    if (!location)
+      return response;
+    if (redirects === DEFAULT_MAX_REDIRECTS)
+      throw new Error(`Too many redirects fetching ${current.toString()}`);
+    const next = await validateRemoteUrl(new URL(location, current), ssrfOptions(options));
+    if (next.origin !== current.origin)
+      headers = withoutSensitiveHeaders(headers);
+    current = next;
+  }
+}
+function unlockerBody(url, zone) {
+  return {
+    url,
+    zone,
+    format: "raw",
+    data_format: "markdown"
+  };
+}
+async function brightDataRequest(url, zone, signal, options) {
+  const apiKey = await getApiKey(signal);
+  const headers = {
+    "Content-Type": "application/json",
+    Authorization: `Bearer ${apiKey}`
+  };
+  const activityId = activityMonitor.logStart({ type: "fetch", url });
+  try {
+    const response = await fetchBrightDataApi(BRIGHTDATA_REQUEST_URL, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(unlockerBody(url, zone)),
+      signal: requestSignal(options?.timeoutMs ?? EXTRACT_TIMEOUT_MS, signal)
+    }, options);
+    if (!response.ok) {
+      const errorText = await response.text().catch(() => "");
+      throw new Error(`Bright Data Web Unlocker error ${response.status}: ${redactCredential(errorText, apiKey).slice(0, 300)}`);
+    }
+    const text = await response.text();
+    activityMonitor.logComplete(activityId, response.status);
+    return text;
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (isAbortError(err))
+      activityMonitor.logComplete(activityId, 0);
+    else
+      activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+}
+function headingTitle(text) {
+  const match = text.match(/^#{1,2}\s+(.+)/m);
+  if (!match)
+    return "";
+  return match[1].replace(/\*+/g, "").trim();
+}
+export function isBrightDataUnlockerAvailable() {
+  if (getZone() === null)
+    return false;
+  return hasCredentialSource({
+    provider: "Bright Data",
+    configuredValue: loadConfig().brightdataApiKey,
+    environmentValue: process.env.BRIGHTDATA_API_KEY
+  });
+}
+export async function extractWithBrightDataUnlocker(url, signal, options) {
+  const zone = requireZone();
+  await validateRemoteUrl(url, ssrfOptions(options));
+  const raw = await brightDataRequest(url, zone, signal, options);
+  const content = raw.trim();
+  if (!content)
+    return null;
+  return { url, title: headingTitle(content), content, error: null };
+}

package/extensions/dm-web-access/brightdata.js

@@ -0,0 +1,334 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const BRIGHTDATA_API_URL = "https://api.brightdata.com/request";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 60000;
+const ZONE_PATTERN = /^[A-Za-z0-9_-]+$/;
+const RECENCY_TBS = {
+  day: "qdr:d",
+  week: "qdr:w",
+  month: "qdr:m",
+  year: "qdr:y"
+};
+let cachedConfig = null;
+function configParseDetail(err) {
+  const position = errorMessage(err).match(/at position \d+(?: \(line \d+ column \d+\))?/i);
+  return position ? `not valid JSON, ${position[0]}` : "not valid JSON";
+}
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf-8");
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${configParseDetail(err)}`);
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+  }
+  cachedConfig = parsed;
+  return cachedConfig;
+}
+async function getApiKey(signal) {
+  return resolveCredential({
+    provider: "Bright Data",
+    configuredValue: loadConfig().brightdataApiKey,
+    environmentValue: process.env.BRIGHTDATA_API_KEY,
+    signal
+  });
+}
+async function requireApiKey(signal) {
+  const apiKey = await getApiKey(signal);
+  if (!apiKey) {
+    throw new Error(`Bright Data API key not found. Either:
+` + `  1. Create ${CONFIG_PATH} with { "brightdataApiKey": "your-key" }
+` + `  2. Set BRIGHTDATA_API_KEY environment variable
+` + "Get a key at https://brightdata.com/cp/setting/users");
+  }
+  return apiKey;
+}
+function normalizeZone(value) {
+  if (typeof value !== "string")
+    return null;
+  const trimmed = value.trim();
+  if (!trimmed)
+    return null;
+  return ZONE_PATTERN.test(trimmed) ? trimmed : null;
+}
+function serpZoneSetting() {
+  const fromEnv = process.env.BRIGHTDATA_SERP_ZONE;
+  if (typeof fromEnv === "string" && fromEnv.trim()) {
+    return { raw: fromEnv.trim(), label: "BRIGHTDATA_SERP_ZONE" };
+  }
+  const configured = loadConfig().brightdataSerpZone;
+  if (typeof configured === "string" && configured.trim()) {
+    return { raw: configured.trim(), label: `brightdataSerpZone in ${CONFIG_PATH}` };
+  }
+  return null;
+}
+function getSerpZone() {
+  const setting = serpZoneSetting();
+  return setting ? normalizeZone(setting.raw) : null;
+}
+function requireSerpZone() {
+  const setting = serpZoneSetting();
+  const zone = setting ? normalizeZone(setting.raw) : null;
+  if (zone)
+    return zone;
+  if (setting) {
+    throw new Error(`Bright Data SERP zone is invalid: ${setting.label} must be a zone name of letters, digits, "-", or "_" ` + `(got "${untrustedText(setting.raw, 60)}").
+` + "The zone must be of Bright Data type `serp`; a Web Unlocker zone (type `unblocker`) is a different product and does not return SERP JSON.\n" + "Create or rename one at https://brightdata.com/cp/zones");
+  }
+  throw new Error(`Bright Data SERP zone is invalid or missing. Either:
+` + `  1. Create ${CONFIG_PATH} with { "brightdataSerpZone": "your_serp_zone" }
+` + `  2. Set BRIGHTDATA_SERP_ZONE environment variable
+` + "The zone must be of Bright Data type `serp`; a Web Unlocker zone is a different product and does not return SERP JSON.\n" + "Create one at https://brightdata.com/cp/zones");
+}
+function normalizeCount(value) {
+  if (typeof value !== "number" || !Number.isFinite(value))
+    return 5;
+  return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function normalizeDomain(value) {
+  let input = value.trim().toLowerCase();
+  if (!input)
+    return null;
+  if (input.startsWith("-"))
+    input = input.slice(1).trim();
+  if (!input)
+    return null;
+  try {
+    const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+    input = parsed.hostname;
+  } catch {
+    input = input.split("/")[0]?.split(":")[0] ?? "";
+  }
+  input = input.replace(/^\.+|\.+$/g, "");
+  return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function parseDomainFilter(domainFilter) {
+  const filters = { include: [], exclude: [] };
+  if (!domainFilter?.length)
+    return filters;
+  for (const raw of domainFilter) {
+    const domain = normalizeDomain(raw);
+    if (!domain)
+      continue;
+    const target = raw.trim().startsWith("-") ? filters.exclude : filters.include;
+    if (!target.includes(domain))
+      target.push(domain);
+  }
+  return filters;
+}
+function buildSearchQuery(query, filters) {
+  const parts = [query];
+  if (filters.include.length === 1) {
+    parts.push(`site:${filters.include[0]}`);
+  } else if (filters.include.length > 1) {
+    parts.push(filters.include.map((domain) => `site:${domain}`).join(" OR "));
+  }
+  for (const domain of filters.exclude) {
+    parts.push(`-site:${domain}`);
+  }
+  return parts.join(" ");
+}
+function buildSerpUrl(searchQuery, numResults, recencyFilter) {
+  const params = new URLSearchParams({ q: searchQuery });
+  params.set("num", String(Math.min(numResults + 5, 20)));
+  const tbs = recencyFilter ? RECENCY_TBS[recencyFilter] : undefined;
+  if (tbs)
+    params.set("tbs", tbs);
+  params.set("brd_json", "1");
+  return `https://www.google.com/search?${params.toString()}`;
+}
+function domainMatches(hostname, domain) {
+  return hostname === domain || hostname.endsWith(`.${domain}`);
+}
+function passesDomainFilters(url, filters) {
+  if (filters.include.length === 0 && filters.exclude.length === 0)
+    return true;
+  let hostname;
+  try {
+    hostname = new URL(url).hostname.toLowerCase();
+  } catch {
+    return false;
+  }
+  if (filters.exclude.some((domain) => domainMatches(hostname, domain)))
+    return false;
+  if (filters.include.length === 0)
+    return true;
+  return filters.include.some((domain) => domainMatches(hostname, domain));
+}
+function requestSignal(signal) {
+  const timeout = AbortSignal.timeout(SEARCH_TIMEOUT_MS);
+  return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+const STATUS_SHAPED_PATTERN = /\b(?:error|status|http)[\s:=-]{1,4}(\d{3})\b/gi;
+const QUOTA_SHAPED_PATTERN = /rate limit|quota|too many requests/gi;
+function untrustedText(text, limit = 300) {
+  return text.replace(/\s+/g, " ").trim().slice(0, limit).replace(STATUS_SHAPED_PATTERN, "upstream $1").replace(QUOTA_SHAPED_PATTERN, "upstream rate-limit notice");
+}
+function invalidResponse(zone, message) {
+  return new Error(`Bright Data API returned invalid response for zone ${zone}: ${message}`);
+}
+function envelopeError(envelope, apiKey) {
+  const parts = [];
+  const { error, errors } = envelope;
+  if (typeof error === "string" && error.trim())
+    parts.push(error.trim());
+  else if (error && typeof error === "object")
+    parts.push(JSON.stringify(error));
+  if (Array.isArray(errors) && errors.length > 0)
+    parts.push(JSON.stringify(errors));
+  else if (typeof errors === "string" && errors.trim())
+    parts.push(errors.trim());
+  if (parts.length === 0)
+    return null;
+  for (const key of ["code", "error_code"]) {
+    const code = envelope[key];
+    if (typeof code === "string" && code.trim())
+      parts.push(`${key} ${code.trim()}`);
+    else if (typeof code === "number")
+      parts.push(`${key} ${code}`);
+  }
+  return untrustedText(redactCredential(parts.join(", "), apiKey), 200);
+}
+function parseSerpResponse(value, zone, apiKey) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw invalidResponse(zone, "expected an object envelope");
+  }
+  const envelope = value;
+  const upstreamError = envelopeError(envelope, apiKey);
+  if (upstreamError) {
+    throw invalidResponse(zone, `Bright Data reported an error instead of a SERP: ${upstreamError}`);
+  }
+  if (envelope.organic === undefined || envelope.organic === null) {
+    throw invalidResponse(zone, "expected an organic array and the envelope carried none. A `serp` zone queried with brd_json=1 " + "returns { organic: [...] }; a zone of type `unblocker`, or a missing brd_json=1, is the usual cause");
+  }
+  if (!Array.isArray(envelope.organic))
+    throw invalidResponse(zone, "expected organic array");
+  const organic = [];
+  for (const [index, entry] of envelope.organic.entries()) {
+    if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
+      throw invalidResponse(zone, `expected organic[${index}] object`);
+    }
+    organic.push(entry);
+  }
+  return { organic };
+}
+function mapResults(organic, numResults, filters) {
+  if (!Array.isArray(organic))
+    return [];
+  const mapped = [];
+  for (const item of organic) {
+    const url = typeof item.link === "string" ? item.link.trim() : "";
+    if (!url || !passesDomainFilters(url, filters))
+      continue;
+    const title = typeof item.title === "string" ? item.title.trim() : "";
+    mapped.push({
+      title: title || `Source ${mapped.length + 1}`,
+      url,
+      snippet: typeof item.description === "string" ? item.description.replace(/\s+/g, " ").trim() : ""
+    });
+    if (mapped.length >= numResults)
+      break;
+  }
+  return mapped;
+}
+function buildAnswer(results) {
+  return results.map((result) => result.snippet ? `${result.snippet}
+Source: ${result.title} (${result.url})` : `Source: ${result.title} (${result.url})`).join(`
+
+`);
+}
+export function isBrightDataAvailable() {
+  try {
+    if (getSerpZone() === null)
+      return false;
+    return hasCredentialSource({
+      provider: "Bright Data",
+      configuredValue: loadConfig().brightdataApiKey,
+      environmentValue: process.env.BRIGHTDATA_API_KEY
+    });
+  } catch {
+    return false;
+  }
+}
+export async function searchWithBrightData(query, options = {}) {
+  const zone = requireSerpZone();
+  const apiKey = await requireApiKey(options.signal);
+  const numResults = normalizeCount(options.numResults);
+  const filters = parseDomainFilter(options.domainFilter);
+  const searchQuery = buildSearchQuery(query, filters);
+  const body = {
+    url: buildSerpUrl(searchQuery, numResults, options.recencyFilter),
+    zone,
+    format: "raw",
+    data_format: "parsed_light"
+  };
+  const activityId = activityMonitor.logStart({ type: "api", query: searchQuery });
+  let response;
+  try {
+    response = await fetch(BRIGHTDATA_API_URL, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify(body),
+      signal: requestSignal(options.signal)
+    });
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (redactedMessage.toLowerCase().includes("abort"))
+      activityMonitor.logComplete(activityId, 0);
+    else
+      activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+  if (!response.ok) {
+    activityMonitor.logComplete(activityId, response.status);
+    const errorText = untrustedText(redactCredential(await response.text(), apiKey));
+    throw new Error(`Bright Data API error ${response.status} for zone ${zone}: ${errorText}`);
+  }
+  const raw = await response.text();
+  if (!raw.trim()) {
+    activityMonitor.logComplete(activityId, response.status);
+    throw new Error(`Bright Data API returned empty response for zone ${zone}`);
+  }
+  let rawData;
+  try {
+    rawData = JSON.parse(raw);
+  } catch (err) {
+    activityMonitor.logComplete(activityId, response.status);
+    const body = untrustedText(redactCredential(raw, apiKey));
+    throw new Error(`Bright Data API returned invalid JSON for zone ${zone}: ${body}`);
+  }
+  let data;
+  try {
+    data = parseSerpResponse(rawData, zone, apiKey);
+  } catch (err) {
+    activityMonitor.logComplete(activityId, response.status);
+    throw err;
+  }
+  activityMonitor.logComplete(activityId, response.status);
+  const results = mapResults(data.organic, numResults, filters);
+  return { answer: buildAnswer(results), results };
+}