@duckmind/dm-windows-x64 0.62.1 → 0.62.9

package/extensions/greedysearch-dm/src/fetcher.mjs

@@ -1,577 +0,0 @@
-let _deps = null;
-async function loadDeps() {
-  if (_deps)
-    return _deps;
-  const [{ Readability }, { JSDOM }, { default: TurndownService }] = await Promise.all([
-    import("@mozilla/readability"),
-    import("jsdom"),
-    import("turndown")
-  ]);
-  const turndown = new TurndownService({
-    headingStyle: "atx",
-    bulletListMarker: "-",
-    codeBlockStyle: "fenced"
-  });
-  turndown.addRule("removeDataUrls", {
-    filter: (node) => node.tagName === "IMG" && node.getAttribute("src")?.startsWith("data:"),
-    replacement: () => ""
-  });
-  _deps = { Readability, JSDOM, turndown };
-  return _deps;
-}
-const DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
-const DEFAULT_HEADERS = {
-  "user-agent": DEFAULT_USER_AGENT,
-  accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-  "accept-language": "en-US,en;q=0.9",
-  "accept-encoding": "gzip, deflate, br",
-  "cache-control": "no-cache",
-  pragma: "no-cache",
-  "sec-ch-ua": '"Chromium";v="122", "Not(A:Brand";v="24", "Google Chrome";v="122"',
-  "sec-ch-ua-mobile": "?0",
-  "sec-ch-ua-platform": '"Windows"',
-  "sec-fetch-dest": "document",
-  "sec-fetch-mode": "navigate",
-  "sec-fetch-site": "none",
-  "sec-fetch-user": "?1",
-  "upgrade-insecure-requests": "1"
-};
-const PRIVATE_URL_PATTERNS = [
-  /^localhost$/i,
-  /^127\.\d+\.\d+\.\d+$/,
-  /^0\.0\.0\.0$/,
-  /^\[::1\]$/,
-  /^10\./,
-  /^172\.(1[6-9]|2\d|3[01])\./,
-  /^192\.168\./,
-  /^169\.254\./,
-  /^fc00:/i,
-  /^fe80:/i,
-  /\.local$/i,
-  /\.internal$/i,
-  /\.localhost$/i
-];
-const IPV4_MAPPED_DOTTED_RE = /^::ffff:(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/i;
-const IPV4_MAPPED_HEX_RE = /^::ffff:([0-9a-f]{1,4}):([0-9a-f]{1,4})$/i;
-const INET_ATON_SEGMENT_RE = /^(?:0x[0-9a-f]+|0[0-7]*|[1-9][0-9]*)$/i;
-function normalizeNumericIPv4(hostname) {
-  if (!hostname || hostname.includes(":"))
-    return null;
-  const parts = hostname.split(".");
-  if (parts.length < 1 || parts.length > 4)
-    return null;
-  if (!parts.every((p) => INET_ATON_SEGMENT_RE.test(p)))
-    return null;
-  const nums = parts.map((p) => {
-    if (/^0x/i.test(p))
-      return parseInt(p, 16);
-    if (/^0[0-7]+$/.test(p))
-      return parseInt(p, 8);
-    return parseInt(p, 10);
-  });
-  if (nums.some((n) => !Number.isFinite(n) || n < 0))
-    return null;
-  let bytes;
-  if (nums.length === 1) {
-    if (nums[0] > 4294967295)
-      return null;
-    bytes = [
-      nums[0] >>> 24 & 255,
-      nums[0] >>> 16 & 255,
-      nums[0] >>> 8 & 255,
-      nums[0] & 255
-    ];
-  } else if (nums.length === 2) {
-    if (nums[0] > 255 || nums[1] > 16777215)
-      return null;
-    bytes = [nums[0], nums[1] >>> 16 & 255, nums[1] >>> 8 & 255, nums[1] & 255];
-  } else if (nums.length === 3) {
-    if (nums[0] > 255 || nums[1] > 255 || nums[2] > 65535)
-      return null;
-    bytes = [nums[0], nums[1], nums[2] >>> 8 & 255, nums[2] & 255];
-  } else {
-    if (nums.some((n) => n > 255))
-      return null;
-    bytes = nums;
-  }
-  return bytes.join(".");
-}
-function extractMappedIPv4(core) {
-  const dottedMatch = core.match(IPV4_MAPPED_DOTTED_RE);
-  if (dottedMatch) {
-    return `${dottedMatch[1]}.${dottedMatch[2]}.${dottedMatch[3]}.${dottedMatch[4]}`;
-  }
-  const hexMatch = core.match(IPV4_MAPPED_HEX_RE);
-  if (hexMatch) {
-    const hi = parseInt(hexMatch[1], 16);
-    const lo = parseInt(hexMatch[2], 16);
-    if (hi > 65535 || lo > 65535)
-      return null;
-    return [
-      hi >>> 8 & 255,
-      hi & 255,
-      lo >>> 8 & 255,
-      lo & 255
-    ].join(".");
-  }
-  return null;
-}
-function isPrivateIPv4(ip) {
-  return PRIVATE_URL_PATTERNS.some((pattern) => pattern.test(ip));
-}
-export function defaultFetchHeaders(overrides = {}) {
-  return { ...DEFAULT_HEADERS, ...overrides };
-}
-export function isPrivateUrl(url) {
-  try {
-    if (typeof url !== "string" || !url.trim()) {
-      return { blocked: true, reason: "URL must be a non-empty string" };
-    }
-    const parsed = new URL(url);
-    if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
-      return {
-        blocked: true,
-        reason: `Protocol not allowed: ${parsed.protocol}`
-      };
-    }
-    const hostname = parsed.hostname.toLowerCase();
-    for (const pattern of PRIVATE_URL_PATTERNS) {
-      if (pattern.test(hostname)) {
-        return {
-          blocked: true,
-          reason: `Private/internal address: ${hostname}`
-        };
-      }
-    }
-    if (hostname.startsWith("[") && hostname.endsWith("]")) {
-      const core = hostname.slice(1, -1);
-      const mapped = extractMappedIPv4(core);
-      if (mapped && isPrivateIPv4(mapped)) {
-        return {
-          blocked: true,
-          reason: `Private/internal address: ${hostname} (maps to ${mapped})`
-        };
-      }
-    }
-    const normalized = normalizeNumericIPv4(hostname);
-    if (normalized && isPrivateIPv4(normalized)) {
-      return {
-        blocked: true,
-        reason: `Private/internal address: ${hostname} (normalizes to ${normalized})`
-      };
-    }
-    return { blocked: false };
-  } catch (error) {
-    return { blocked: true, reason: `Invalid URL: ${error.message}` };
-  }
-}
-export function rewriteGitHubUrl(url) {
-  try {
-    const parsed = new URL(url);
-    if (!(parsed.hostname === "github.com" || parsed.hostname.endsWith(".github.com"))) {
-      return url;
-    }
-    const parts = parsed.pathname.split("/").filter(Boolean);
-    if (parts.length < 5) {
-      return url;
-    }
-    const [owner, repo, type, ref, ...fileParts] = parts;
-    if (type !== "blob") {
-      return url;
-    }
-    const rawPath = fileParts.join("/");
-    const rawUrl = `https://raw.githubusercontent.com/${owner}/${repo}/${ref}/${rawPath}`;
-    return rawUrl;
-  } catch {
-    return url;
-  }
-}
-export async function fetchSourceHttp(url, options = {}) {
-  const privateCheck = isPrivateUrl(url);
-  if (privateCheck.blocked) {
-    return {
-      ok: false,
-      url,
-      finalUrl: url,
-      status: 403,
-      error: `Blocked: ${privateCheck.reason}`,
-      needsBrowser: false
-    };
-  }
-  const originalUrl = url;
-  url = rewriteGitHubUrl(url);
-  if (url !== originalUrl) {
-    console.error(`[fetcher] Rewrote GitHub URL: ${originalUrl.slice(0, 60)}... → raw.githubusercontent.com`);
-  }
-  const { timeoutMs = 15000, userAgent, signal } = options;
-  const controller = new AbortController;
-  const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
-  if (signal) {
-    signal.addEventListener("abort", () => controller.abort(), { once: true });
-  }
-  try {
-    const response = await fetch(url, {
-      method: "GET",
-      headers: {
-        ...DEFAULT_HEADERS,
-        "user-agent": userAgent || DEFAULT_USER_AGENT
-      },
-      redirect: "follow",
-      signal: controller.signal
-    });
-    clearTimeout(timeoutId);
-    const contentType = response.headers.get("content-type") || "";
-    const finalUrl = response.url;
-    const lastModified = response.headers.get("last-modified") || "";
-    const finalPrivateCheck = isPrivateUrl(finalUrl);
-    if (finalPrivateCheck.blocked) {
-      return {
-        ok: false,
-        url,
-        finalUrl,
-        status: response.status,
-        error: `Blocked: ${finalPrivateCheck.reason}`,
-        needsBrowser: false
-      };
-    }
-    let isRawGitHub = false;
-    try {
-      const finalHost = new URL(finalUrl).hostname.toLowerCase();
-      isRawGitHub = finalHost === "raw.githubusercontent.com";
-    } catch {}
-    if (contentType.includes("text/plain") && isRawGitHub) {
-      const text = await response.text();
-      return {
-        ok: true,
-        url: originalUrl,
-        finalUrl,
-        status: response.status,
-        title: finalUrl.split("/").pop() || "GitHub File",
-        byline: "",
-        siteName: "GitHub",
-        lang: "",
-        publishedTime: lastModified,
-        lastModified,
-        markdown: text,
-        contentLength: text.length,
-        excerpt: text.slice(0, 300).replaceAll(/\n/g, " "),
-        needsBrowser: false
-      };
-    }
-    if (!contentType.includes("text/html") && !contentType.includes("application/xhtml")) {
-      return {
-        ok: false,
-        url,
-        finalUrl,
-        status: response.status,
-        error: `Unsupported content type: ${contentType}`,
-        needsBrowser: false
-      };
-    }
-    const html = await response.text();
-    const quickCheck = detectBotBlock(response.status, html, finalUrl, url);
-    if (quickCheck.blocked) {
-      return {
-        ok: false,
-        url,
-        finalUrl,
-        status: response.status,
-        error: `Blocked: ${quickCheck.reason}`,
-        needsBrowser: true
-      };
-    }
-    const extracted = await extractContent(html, finalUrl);
-    const quality = checkContentQuality(extracted);
-    if (!quality.ok) {
-      return {
-        ok: false,
-        url,
-        finalUrl,
-        status: response.status,
-        error: `Low quality content: ${quality.reason}`,
-        needsBrowser: true
-      };
-    }
-    return {
-      ok: true,
-      url,
-      finalUrl,
-      status: response.status,
-      title: extracted.title,
-      byline: extracted.byline,
-      siteName: extracted.siteName,
-      lang: extracted.lang,
-      publishedTime: extracted.publishedTime || lastModified,
-      lastModified,
-      markdown: extracted.markdown,
-      excerpt: extracted.excerpt,
-      contentLength: extracted.markdown.length,
-      needsBrowser: false
-    };
-  } catch (error) {
-    clearTimeout(timeoutId);
-    const needsBrowser = isNetworkErrorRetryableWithBrowser(error);
-    return {
-      ok: false,
-      url,
-      finalUrl: url,
-      status: 0,
-      error: error.message,
-      needsBrowser
-    };
-  }
-}
-export function detectBotBlock(status, html, finalUrl, originalUrl) {
-  const title = html.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1]?.toLowerCase() || "";
-  const sample = html.slice(0, 30000).toLowerCase();
-  const combined = `${title} ${sample}`;
-  if (status === 403 || status === 429 || status === 503) {
-    return { blocked: true, reason: `HTTP ${status}` };
-  }
-  const blockSignals = [
-    {
-      pattern: /class=["'][^"']*captcha["']|<div[^>]*id=["']captcha/i,
-      reason: "captcha"
-    },
-    {
-      pattern: /g-recaptcha|data-sitekey|i['"]m not a robot/i,
-      reason: "captcha"
-    },
-    {
-      pattern: /checking your browser.{0,100}please wait|cf-browser-verification/i,
-      reason: "cloudflare challenge"
-    },
-    {
-      pattern: /just a moment.{0,50}security check|ddos protection by cloudflare/i,
-      reason: "cloudflare challenge"
-    },
-    {
-      pattern: /unusual traffic.{0,50}from your computer network/i,
-      reason: "unusual traffic"
-    },
-    {
-      pattern: /bot detected|automated.{0,20}request/i,
-      reason: "bot detection"
-    },
-    {
-      pattern: /enable\s+javascript\s+to\s+view|javascript\s+is\s+required.{0,50}enabled/i,
-      reason: "requires javascript"
-    },
-    { pattern: /access denied|accessdenied/i, reason: "access denied" },
-    {
-      pattern: /protected by anubis|anubis uses a proof-of-work/i,
-      reason: "anubis challenge"
-    }
-  ];
-  for (const signal of blockSignals) {
-    if (signal.pattern.test(combined)) {
-      return { blocked: true, reason: signal.reason };
-    }
-  }
-  const loginRedirect = detectLoginRedirect(originalUrl, finalUrl, html);
-  if (loginRedirect) {
-    return { blocked: true, reason: loginRedirect };
-  }
-  return { blocked: false };
-}
-const AUTH_DOMAINS = [
-  "accounts.google.com",
-  "login.microsoftonline.com",
-  "login.live.com",
-  "auth0.com",
-  "okta.com",
-  "auth.mozilla.auth0.com",
-  "id.atlassian.com"
-];
-const AUTH_HOSTNAME_PREFIXES = [
-  "login.",
-  "signin.",
-  "auth.",
-  "sso.",
-  "accounts.",
-  "idp."
-];
-const LOGIN_CONTENT_PATTERNS = [
-  "sign in to continue",
-  "log in to continue",
-  "authentication required",
-  "create an account to continue",
-  "subscribe to continue reading",
-  "members only"
-];
-function detectLoginRedirect(requestedUrl, finalUrl, html) {
-  try {
-    const requested = new URL(requestedUrl);
-    const final = new URL(finalUrl);
-    if (requested.hostname.toLowerCase() === final.hostname.toLowerCase()) {
-      return;
-    }
-    const finalHost = final.hostname.toLowerCase();
-    if (AUTH_DOMAINS.some((d) => finalHost === d || finalHost.endsWith(`.${d}`))) {
-      return `redirected to login (${final.hostname})`;
-    }
-    if (AUTH_HOSTNAME_PREFIXES.some((p) => finalHost.startsWith(p))) {
-      return `redirected to login (${final.hostname})`;
-    }
-    const sample = html.slice(0, 20000).toLowerCase();
-    if (LOGIN_CONTENT_PATTERNS.some((p) => sample.includes(p))) {
-      return `redirected to login page (${final.hostname})`;
-    }
-  } catch {}
-  return;
-}
-function isNetworkErrorRetryableWithBrowser(error) {
-  const message = error.message.toLowerCase();
-  return message.includes("fetch failed") || message.includes("unable to verify") || message.includes("certificate") || message.includes("timeout");
-}
-function extractMetaDate(document) {
-  const selectors = [
-    'meta[property="article:published_time"]',
-    'meta[name="article:published_time"]',
-    'meta[property="og:published_time"]',
-    'meta[name="publication_date"]',
-    'meta[name="date"]',
-    'meta[itemprop="datePublished"]',
-    'time[itemprop="datePublished"]',
-    'meta[name="DC.date"]'
-  ];
-  for (const sel of selectors) {
-    const el = document.querySelector(sel);
-    const val = el?.getAttribute("content") || el?.getAttribute("datetime") || "";
-    if (val)
-      return val;
-  }
-  return "";
-}
-export async function extractContent(html, url) {
-  const { Readability, JSDOM, turndown } = await loadDeps();
-  const dom = new JSDOM(html, { url });
-  try {
-    const document = dom.window.document;
-    const reader = new Readability(document);
-    const article = reader.parse();
-    if (article && article.content) {
-      const markdown = turndown.turndown(article.content);
-      const cleanMarkdown = markdown.replaceAll(/\n{3,}/g, `
-
-`).trim();
-      const publishedTime = article.publishedTime || extractMetaDate(document) || "";
-      return {
-        title: article.title || document.title || url,
-        byline: article.byline || "",
-        siteName: article.siteName || "",
-        lang: article.lang || "",
-        publishedTime,
-        markdown: cleanMarkdown,
-        excerpt: cleanMarkdown.slice(0, 300).replaceAll(/\n/g, " ")
-      };
-    }
-    const body = document.body;
-    if (body) {
-      const clone = body.cloneNode(true);
-      clone.querySelectorAll("script, style, nav, footer, header, aside").forEach((el) => el.remove());
-      const text = clone.textContent || "";
-      const cleanText = text.replaceAll(/\s+/g, " ").trim();
-      return {
-        title: document.title || url,
-        byline: "",
-        siteName: "",
-        lang: "",
-        publishedTime: extractMetaDate(document),
-        markdown: cleanText,
-        excerpt: cleanText.slice(0, 300)
-      };
-    }
-    return {
-      title: url,
-      byline: "",
-      siteName: "",
-      lang: "",
-      publishedTime: "",
-      markdown: "",
-      excerpt: ""
-    };
-  } finally {
-    dom.window.close();
-  }
-}
-export function checkContentQuality(extracted) {
-  const markdown = extracted.markdown.trim().toLowerCase();
-  const title = (extracted.title || "").toLowerCase();
-  if (extracted.markdown.trim().length < 100) {
-    return { ok: false, reason: "content too short (< 100 chars)" };
-  }
-  const lc = markdown.toLowerCase();
-  const suspiciousChecks = [
-    {
-      check: () => lc.includes("loading") && lc.includes("please wait"),
-      desc: "loading page"
-    },
-    {
-      check: () => lc.includes("please ensure javascript is enabled"),
-      desc: "requires javascript"
-    },
-    {
-      check: () => lc.includes("enable javascript to view"),
-      desc: "requires javascript"
-    },
-    {
-      check: () => lc.includes("just a moment"),
-      desc: "cloudflare challenge detected in content"
-    },
-    {
-      check: () => lc.includes("verify you are human"),
-      desc: "human verification"
-    },
-    {
-      check: () => lc.includes("captcha required"),
-      desc: "captcha in extracted content"
-    },
-    {
-      check: () => lc.includes("access denied"),
-      desc: "access denied in content"
-    },
-    {
-      check: () => /^\s{0,10}sign\s{1,5}in\s{0,10}$|^\s{0,10}log\s{1,5}in\s{0,10}$/im.test(markdown),
-      desc: "login form only"
-    }
-  ];
-  for (const { check, desc } of suspiciousChecks) {
-    if (check()) {
-      return { ok: false, reason: desc };
-    }
-  }
-  if (title.includes("just a moment") || title.includes("checking your browser")) {
-    return { ok: false, reason: "cloudflare challenge page detected in title" };
-  }
-  return { ok: true };
-}
-export function shouldUseBrowser(url) {
-  try {
-    const parsed = new URL(url);
-    const hostname = parsed.hostname.toLowerCase();
-    const pathname = parsed.pathname.toLowerCase();
-    const jsHeavyDomains = [
-      "react.dev",
-      "nextjs.org",
-      "vuejs.org",
-      "angular.io",
-      "svelte.dev",
-      "docs.expo.dev",
-      "tailwindcss.com",
-      "storybook.js.org"
-    ];
-    if (jsHeavyDomains.some((d) => hostname === d || hostname.endsWith(`.${d}`))) {
-      return true;
-    }
-    if (pathname.includes("/playground") || pathname.includes("/demo") || pathname.includes("/app")) {
-      return true;
-    }
-    if (parsed.hash && parsed.hash.length > 1) {
-      return true;
-    }
-    return false;
-  } catch {
-    return false;
-  }
-}

package/extensions/greedysearch-dm/src/formatters/results.js

@@ -1,95 +0,0 @@
-import { formatEngineName } from "../utils/helpers.js";
-import { renderSynthesis } from "./synthesis.js";
-const MAX_LINE_WIDTH = 800;
-function _truncateLongLines(text) {
-  return text.split(`
-`).map((line) => line.length > MAX_LINE_WIDTH ? line.slice(0, MAX_LINE_WIDTH - 1) + "…" : line).join(`
-`);
-}
-export function formatResults(engine, data) {
-  const lines = [];
-  if (engine === "all") {
-    return _truncateLongLines(formatAllEnginesResult(data, lines));
-  }
-  return _truncateLongLines(formatSingleEngineResult(data, lines));
-}
-function formatAllEnginesResult(data, lines) {
-  const synthesis = data._synthesis;
-  const dedupedSources = data._sources;
-  const needsHuman = data._needsHumanVerification;
-  const research = data._research;
-  if (needsHuman) {
-    const engines = Array.isArray(needsHuman.engines) ? needsHuman.engines.join(", ") : "one or more engines";
-    lines.push("## Manual verification required");
-    lines.push(String(needsHuman.message || "Visible Chrome is open. Solve the verification challenge, then rerun the same search."));
-    lines.push(`Engines: ${engines}`);
-    lines.push("");
-  }
-  if (synthesis?.answer) {
-    if (research?.mode === "iterative")
-      renderResearchHeader(lines, research);
-    renderSynthesis(lines, synthesis, dedupedSources || [], 6);
-    const synthesizedBy = String(synthesis.synthesizedBy || "configured synthesizer");
-    lines.push(research?.mode === "iterative" ? `*Research mode: iterative planning, source fetching, citation audit, and bundle output*
-` : `*Synthesized by ${synthesizedBy} from multi-engine results and fetched sources*
-`);
-    return lines.join(`
-`).trim();
-  }
-  for (const [eng, result] of Object.entries(data)) {
-    if (eng.startsWith("_"))
-      continue;
-    lines.push(`
-## ${formatEngineName(eng)}`);
-    formatEngineResult(result, lines, 3);
-  }
-  return lines.join(`
-`).trim();
-}
-function renderResearchHeader(lines, research) {
-  const floor = research.floor;
-  const metrics = floor?.metrics;
-  const bundle = research.bundle;
-  const manifest = research.manifest;
-  lines.push("## Research Run");
-  lines.push(`- Status: ${floor?.floorMet ? "floor met" : "partial / floor unmet"}`);
-  if (manifest?.terminationReason)
-    lines.push(`- Stop reason: ${String(manifest.terminationReason)}`);
-  if (metrics) {
-    lines.push(`- Evidence: ${metrics.fetchedOk || 0} fetched sources, ${metrics.primarySources || 0} primary/official, ${metrics.claims || 0} claims, ${metrics.cited || 0} citations`);
-    lines.push(`- Questions: ${metrics.closedQuestions || 0}/${metrics.totalQuestions || 0} closed${metrics.openQuestions ? `, ${metrics.openQuestions} open` : ""}`);
-  }
-  if (bundle?.dir)
-    lines.push(`- Bundle: ${String(bundle.dir)}`);
-  lines.push("");
-}
-function formatSingleEngineResult(data, lines) {
-  const needsHuman = data._needsHumanVerification;
-  if (needsHuman) {
-    const engines = Array.isArray(needsHuman.engines) ? needsHuman.engines.join(", ") : "this engine";
-    lines.push("## Manual verification required");
-    lines.push(String(needsHuman.message || "Visible Chrome is open. Solve the verification challenge, then rerun the same search."));
-    lines.push(`Engines: ${engines}`);
-    lines.push("");
-  }
-  formatEngineResult(data, lines, 5);
-  return lines.join(`
-`).trim();
-}
-function formatEngineResult(data, lines, maxSources) {
-  if (data.error) {
-    lines.push(`Error: ${data.error}`);
-    return;
-  }
-  if (data.answer) {
-    lines.push(String(data.answer));
-  }
-  const sources = data.sources;
-  if (Array.isArray(sources) && sources.length > 0) {
-    lines.push(`
-Sources:`);
-    for (const s of sources.slice(0, maxSources)) {
-      lines.push(`- [${s.title || s.url}](${s.url})`);
-    }
-  }
-}