@duckmind/dm-windows-x64 0.63.0 → 0.63.4

package/extensions/dm-web-access/rsc-extract.js

@@ -0,0 +1,347 @@
+export function extractRSCContent(html) {
+  if (!html.includes("self.__next_f.push")) {
+    return null;
+  }
+  const chunkMap = new Map;
+  const scriptRegex = /<script>self\.__next_f\.push\(\[1,"([\s\S]*?)"\]\)<\/script>/g;
+  for (const match of html.matchAll(scriptRegex)) {
+    let content;
+    try {
+      content = JSON.parse('"' + match[1] + '"');
+    } catch {
+      continue;
+    }
+    for (const line of content.split(`
+`)) {
+      if (!line.trim())
+        continue;
+      const colonIdx = line.indexOf(":");
+      if (colonIdx <= 0 || colonIdx > 4)
+        continue;
+      const id = line.slice(0, colonIdx);
+      if (!/^[0-9a-f]+$/i.test(id))
+        continue;
+      const payload = line.slice(colonIdx + 1);
+      if (!payload)
+        continue;
+      const existing = chunkMap.get(id);
+      if (!existing || payload.length > existing.length) {
+        chunkMap.set(id, payload);
+      }
+    }
+  }
+  if (chunkMap.size === 0)
+    return null;
+  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/);
+  const title = titleMatch?.[1]?.split("|")[0]?.trim() || "";
+  const parsedCache = new Map;
+  function getParsedChunk(id) {
+    if (parsedCache.has(id))
+      return parsedCache.get(id);
+    const chunk = chunkMap.get(id);
+    if (!chunk || !chunk.startsWith("[")) {
+      parsedCache.set(id, null);
+      return null;
+    }
+    try {
+      const parsed = JSON.parse(chunk);
+      parsedCache.set(id, parsed);
+      return parsed;
+    } catch {
+      parsedCache.set(id, null);
+      return null;
+    }
+  }
+  const visitedRefs = new Set;
+  function extractNode(node, ctx = { inTable: false, inCode: false }) {
+    if (node === null || node === undefined)
+      return "";
+    if (typeof node === "string") {
+      const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
+      if (refMatch) {
+        const refId = refMatch[1];
+        if (visitedRefs.has(refId))
+          return "";
+        visitedRefs.add(refId);
+        const refNode = getParsedChunk(refId);
+        const result = refNode ? extractNode(refNode, ctx) : "";
+        visitedRefs.delete(refId);
+        return result;
+      }
+      if (!ctx.inCode && (node === "$undefined" || node === "$" || /^\$[A-Z]/.test(node)))
+        return "";
+      return node.trim() ? node : "";
+    }
+    if (typeof node === "number")
+      return String(node);
+    if (typeof node === "boolean")
+      return "";
+    if (!Array.isArray(node))
+      return "";
+    if (node[0] === "$" && typeof node[1] === "string") {
+      const tag = node[1];
+      const props = node[3] || {};
+      const skipTags = [
+        "script",
+        "style",
+        "svg",
+        "path",
+        "circle",
+        "link",
+        "meta",
+        "template",
+        "button",
+        "input",
+        "nav",
+        "footer",
+        "aside"
+      ];
+      if (skipTags.includes(tag))
+        return "";
+      if (tag.startsWith("$L")) {
+        const refId = tag.slice(2);
+        if (visitedRefs.has(refId))
+          return "";
+        if (props.baseId && props.children) {
+          return `## ${String(props.children)}
+
+`;
+        }
+        visitedRefs.add(refId);
+        const refNode = getParsedChunk(refId);
+        let result = "";
+        if (refNode) {
+          result = extractNode(refNode, ctx);
+        } else if (props.children) {
+          result = extractNode(props.children, ctx);
+        }
+        visitedRefs.delete(refId);
+        return result;
+      }
+      const children = props.children;
+      const content = children ? extractNode(children, ctx) : "";
+      switch (tag) {
+        case "h1":
+          return `# ${content.trim()}
+
+`;
+        case "h2":
+          return `## ${content.trim()}
+
+`;
+        case "h3":
+          return `### ${content.trim()}
+
+`;
+        case "h4":
+          return `#### ${content.trim()}
+
+`;
+        case "h5":
+          return `##### ${content.trim()}
+
+`;
+        case "h6":
+          return `###### ${content.trim()}
+
+`;
+        case "p":
+          return ctx.inTable ? content : `${content.trim()}
+
+`;
+        case "code": {
+          const codeContent = children ? extractNode(children, { ...ctx, inCode: true }) : "";
+          return ctx.inCode ? codeContent : `\`${codeContent}\``;
+        }
+        case "pre": {
+          const preContent = children ? extractNode(children, { ...ctx, inCode: true }) : "";
+          return "```\n" + preContent + "\n```\n\n";
+        }
+        case "strong":
+        case "b":
+          return `**${content}**`;
+        case "em":
+        case "i":
+          return `*${content}*`;
+        case "li":
+          return `- ${content.trim()}
+`;
+        case "ul":
+        case "ol":
+          return content + `
+`;
+        case "blockquote":
+          return `> ${content.trim()}
+
+`;
+        case "table":
+          return extractTable(node) + `
+`;
+        case "thead":
+        case "tbody":
+        case "tr":
+        case "th":
+        case "td":
+          return content;
+        case "div":
+          if (props.role === "alert" || props["data-slot"] === "alert") {
+            return `> ${content.trim()}
+
+`;
+          }
+          return content;
+        case "a": {
+          const href = props.href;
+          return href && !href.startsWith("#") ? `[${content}](${href})` : content;
+        }
+        default:
+          return content;
+      }
+    }
+    return node.map((n) => extractNode(n, ctx)).join("");
+  }
+  function extractTable(tableNode) {
+    const props = tableNode[3] || {};
+    const rows = [];
+    let headerRowCount = 0;
+    function walkTable(node, isHeader = false) {
+      if (node === null || node === undefined)
+        return;
+      if (typeof node === "string") {
+        const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
+        if (refMatch && !visitedRefs.has(refMatch[1])) {
+          visitedRefs.add(refMatch[1]);
+          const refNode = getParsedChunk(refMatch[1]);
+          if (refNode)
+            walkTable(refNode, isHeader);
+          visitedRefs.delete(refMatch[1]);
+        }
+        return;
+      }
+      if (!Array.isArray(node))
+        return;
+      if (node[0] === "$") {
+        const tag = node[1];
+        const nodeProps = node[3] || {};
+        if (tag.startsWith("$L")) {
+          const refId = tag.slice(2);
+          if (!visitedRefs.has(refId)) {
+            visitedRefs.add(refId);
+            const refNode = getParsedChunk(refId);
+            if (refNode)
+              walkTable(refNode, isHeader);
+            visitedRefs.delete(refId);
+          }
+          return;
+        }
+        if (tag === "thead")
+          walkTable(nodeProps.children, true);
+        else if (tag === "tbody")
+          walkTable(nodeProps.children, false);
+        else if (tag === "tr") {
+          const cells = [];
+          walkCells(nodeProps.children, cells);
+          if (cells.length > 0) {
+            rows.push(cells);
+            if (isHeader)
+              headerRowCount++;
+          }
+        } else
+          walkTable(nodeProps.children, isHeader);
+      } else {
+        for (const child of node)
+          walkTable(child, isHeader);
+      }
+    }
+    function walkCells(node, cells) {
+      if (node === null || node === undefined)
+        return;
+      if (typeof node === "string") {
+        const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
+        if (refMatch && !visitedRefs.has(refMatch[1])) {
+          visitedRefs.add(refMatch[1]);
+          const refNode = getParsedChunk(refMatch[1]);
+          if (refNode)
+            walkCells(refNode, cells);
+          visitedRefs.delete(refMatch[1]);
+        }
+        return;
+      }
+      if (!Array.isArray(node))
+        return;
+      if (node[0] === "$" && (node[1] === "td" || node[1] === "th")) {
+        const cellProps = node[3] || {};
+        const text = extractNode(cellProps.children, { inTable: true, inCode: false }).trim().replace(/\n/g, " ").replace(/\\/g, "\\\\").replace(/\|/g, "\\|");
+        cells.push(text);
+      } else if (node[0] === "$" && typeof node[1] === "string" && node[1].startsWith("$L")) {
+        const refId = node[1].slice(2);
+        if (!visitedRefs.has(refId)) {
+          visitedRefs.add(refId);
+          const refNode = getParsedChunk(refId);
+          if (refNode)
+            walkCells(refNode, cells);
+          visitedRefs.delete(refId);
+        }
+      } else {
+        for (const child of node)
+          walkCells(child, cells);
+      }
+    }
+    walkTable(props.children);
+    if (rows.length === 0)
+      return "";
+    const colCount = Math.max(...rows.map((r) => r.length));
+    let md = "";
+    for (let i = 0;i < rows.length; i++) {
+      const row = rows[i].concat(Array(colCount - rows[i].length).fill(""));
+      md += "| " + row.join(" | ") + ` |
+`;
+      if (i === headerRowCount - 1 || headerRowCount === 0 && i === 0) {
+        md += "| " + Array(colCount).fill("---").join(" | ") + ` |
+`;
+      }
+    }
+    return md;
+  }
+  const mainChunk = getParsedChunk("23");
+  if (mainChunk) {
+    const content = extractNode(mainChunk);
+    if (content.trim().length > 100) {
+      const cleaned = content.replace(/\n{3,}/g, `
+
+`).trim();
+      return { title, content: cleaned };
+    }
+  }
+  const contentParts = [];
+  for (const [id] of chunkMap) {
+    if (id === "23")
+      continue;
+    const parsed = getParsedChunk(id);
+    if (!parsed)
+      continue;
+    visitedRefs.clear();
+    const text = extractNode(parsed);
+    if (text.trim().length > 50 && !text.includes("page was not found") && !text.includes("404")) {
+      contentParts.push({ order: parseInt(id, 16), text: text.trim() });
+    }
+  }
+  if (contentParts.length === 0)
+    return null;
+  contentParts.sort((a, b) => a.order - b.order);
+  const seen = new Set;
+  const uniqueParts = [];
+  for (const part of contentParts) {
+    const key = part.text.slice(0, 150);
+    if (!seen.has(key)) {
+      seen.add(key);
+      uniqueParts.push(part.text);
+    }
+  }
+  const content = uniqueParts.join(`
+
+`).replace(/\n{3,}/g, `
+
+`).trim();
+  return content.length > 100 ? { title, content } : null;
+}

package/extensions/dm-web-access/search1api.js

@@ -0,0 +1,245 @@
+import { existsSync, readFileSync } from "node:fs";
+import { activityMonitor } from "./activity.js";
+import { hasCredentialSource, redactCredential, resolveCredential } from "./credential-source.js";
+import { getWebSearchConfigPath } from "./utils.js";
+const SEARCH1API_SEARCH_URL = "https://api.search1api.com/search";
+const SEARCH1API_CRAWL_URL = "https://api.search1api.com/crawl";
+const CONFIG_PATH = getWebSearchConfigPath();
+const SEARCH_TIMEOUT_MS = 60000;
+const CRAWL_TIMEOUT_MS = 60000;
+let cachedConfig = null;
+function loadConfig() {
+  if (cachedConfig)
+    return cachedConfig;
+  if (!existsSync(CONFIG_PATH)) {
+    cachedConfig = {};
+    return cachedConfig;
+  }
+  const raw = readFileSync(CONFIG_PATH, "utf-8");
+  let parsed;
+  try {
+    parsed = JSON.parse(raw);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(`Failed to parse ${CONFIG_PATH}: ${message}`);
+  }
+  if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error(`Invalid config in ${CONFIG_PATH}: expected a JSON object`);
+  }
+  cachedConfig = parsed;
+  return cachedConfig;
+}
+async function getApiKey(signal) {
+  const key = await resolveCredential({
+    provider: "Search1API",
+    configuredValue: loadConfig().search1apiApiKey,
+    environmentValue: process.env.SEARCH1API_KEY,
+    signal
+  });
+  if (!key) {
+    throw new Error(`Search1API key not found. Either:
+` + `  1. Create ${CONFIG_PATH} with { "search1apiApiKey": "your-key" }
+` + `  2. Set SEARCH1API_KEY environment variable
+` + "Create a key at https://dashboard.search1api.com");
+  }
+  return key;
+}
+export function isSearch1APIAvailable() {
+  return hasCredentialSource({
+    provider: "Search1API",
+    configuredValue: loadConfig().search1apiApiKey,
+    environmentValue: process.env.SEARCH1API_KEY
+  });
+}
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function requestSignal(signal, timeoutMs) {
+  const timeout = AbortSignal.timeout(timeoutMs);
+  return signal ? AbortSignal.any([signal, timeout]) : timeout;
+}
+function normalizeNumResults(value) {
+  if (typeof value !== "number" || !Number.isFinite(value))
+    return 5;
+  return Math.max(1, Math.min(Math.floor(value), 20));
+}
+function normalizeDomain(value) {
+  let input = value.trim().toLowerCase();
+  if (!input)
+    return null;
+  if (input.startsWith("-"))
+    input = input.slice(1).trim();
+  if (!input)
+    return null;
+  try {
+    const parsed = input.includes("://") ? new URL(input) : new URL(`https://${input}`);
+    input = parsed.hostname;
+  } catch {
+    input = input.split("/")[0]?.split(":")[0] ?? "";
+  }
+  input = input.replace(/^\.+|\.+$/g, "");
+  return /^[a-z0-9][a-z0-9.-]*\.[a-z]{2,}$/i.test(input) ? input : null;
+}
+function mapDomainFilter(domainFilter) {
+  const includeSites = [];
+  const excludeSites = [];
+  for (const raw of domainFilter ?? []) {
+    const domain = normalizeDomain(raw);
+    if (!domain)
+      continue;
+    const target = raw.trim().startsWith("-") ? excludeSites : includeSites;
+    if (!target.includes(domain))
+      target.push(domain);
+  }
+  return { includeSites, excludeSites };
+}
+function buildSearchBody(query, options) {
+  const numResults = normalizeNumResults(options.numResults);
+  const { includeSites, excludeSites } = mapDomainFilter(options.domainFilter);
+  return {
+    query,
+    max_results: numResults,
+    crawl_results: options.includeContent ? numResults : 0,
+    ...includeSites.length > 0 ? { include_sites: includeSites } : {},
+    ...excludeSites.length > 0 ? { exclude_sites: excludeSites } : {},
+    ...options.recencyFilter ? { time_range: options.recencyFilter } : {}
+  };
+}
+async function search1APIJsonRequest(label, url, apiKey, body, timeoutMs, signal) {
+  let response;
+  try {
+    response = await fetch(url, {
+      method: "POST",
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+        "Content-Type": "application/json"
+      },
+      body: JSON.stringify(body),
+      signal: requestSignal(signal, timeoutMs)
+    });
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+  const raw = await response.text();
+  if (!response.ok) {
+    throw new Error(`Search1API ${label} API error ${response.status}: ${redactCredential(raw, apiKey).slice(0, 300)}`);
+  }
+  try {
+    return JSON.parse(raw);
+  } catch (err) {
+    throw new Error(`Search1API ${label} API returned invalid JSON: ${errorMessage(err)}`);
+  }
+}
+function mapSearchResults(results) {
+  if (!Array.isArray(results)) {
+    throw new Error("Search1API Search API returned an unexpected response shape");
+  }
+  return results.flatMap((item) => {
+    if (!item || typeof item.link !== "string" || item.link.trim().length === 0)
+      return [];
+    const url = item.link.trim();
+    return [{
+      title: typeof item.title === "string" && item.title.trim() ? item.title.trim() : url,
+      url,
+      snippet: typeof item.snippet === "string" ? item.snippet.replace(/\s+/g, " ").trim() : ""
+    }];
+  });
+}
+function mapInlineContent(results) {
+  if (!Array.isArray(results))
+    return [];
+  return results.flatMap((item) => {
+    if (!item || typeof item.link !== "string" || item.link.trim().length === 0)
+      return [];
+    if (typeof item.content !== "string" || item.content.trim().length === 0)
+      return [];
+    return [{
+      url: item.link.trim(),
+      title: typeof item.title === "string" ? item.title.trim() : "",
+      content: item.content,
+      error: null
+    }];
+  });
+}
+function buildAnswer(results) {
+  return results.map((result) => {
+    if (result.snippet)
+      return `${result.snippet}
+Source: ${result.title} (${result.url})`;
+    return `Source: ${result.title} (${result.url})`;
+  }).join(`
+
+`);
+}
+export async function searchWithSearch1API(query, options = {}) {
+  const apiKey = await getApiKey(options.signal);
+  const activityId = activityMonitor.logStart({ type: "api", query });
+  try {
+    const data = await search1APIJsonRequest("Search", SEARCH1API_SEARCH_URL, apiKey, buildSearchBody(query, options), SEARCH_TIMEOUT_MS, options.signal);
+    const results = mapSearchResults(data.results);
+    const response = { answer: buildAnswer(results), results };
+    if (options.includeContent) {
+      const inlineContent = mapInlineContent(data.results);
+      if (inlineContent.length > 0)
+        response.inlineContent = inlineContent;
+    }
+    activityMonitor.logComplete(activityId, 200);
+    return response;
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (redactedMessage.toLowerCase().includes("abort"))
+      activityMonitor.logComplete(activityId, 0);
+    else
+      activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+}
+export async function extractWithSearch1API(url, signal, options = {}) {
+  const apiKey = await getApiKey(signal);
+  const activityId = activityMonitor.logStart({ type: "fetch", url });
+  try {
+    const data = await search1APIJsonRequest("Crawl", SEARCH1API_CRAWL_URL, apiKey, { url }, typeof options.timeoutMs === "number" && Number.isFinite(options.timeoutMs) ? Math.max(1, Math.floor(options.timeoutMs)) : CRAWL_TIMEOUT_MS, signal);
+    const result = data.results;
+    if (!result || typeof result !== "object") {
+      throw new Error("Search1API Crawl API returned an unexpected response shape");
+    }
+    const content = typeof result.content === "string" ? result.content.trim() : "";
+    if (!content) {
+      activityMonitor.logComplete(activityId, 200);
+      return null;
+    }
+    activityMonitor.logComplete(activityId, 200);
+    return {
+      url,
+      title: typeof result.title === "string" ? result.title.trim() : "",
+      content,
+      error: null
+    };
+  } catch (err) {
+    const message = errorMessage(err);
+    const redactedMessage = redactCredential(message, apiKey);
+    if (redactedMessage.toLowerCase().includes("abort"))
+      activityMonitor.logComplete(activityId, 0);
+    else
+      activityMonitor.logError(activityId, redactedMessage);
+    if (redactedMessage === message)
+      throw err;
+    const redactedError = new Error(redactedMessage);
+    if (err instanceof Error)
+      redactedError.name = err.name;
+    throw redactedError;
+  }
+}