@duckmind/dm-windows-x64 0.62.3 → 0.63.0

package/extensions/greedysearch-dm/src/search/fetch-source.mjs

@@ -1,470 +0,0 @@
-import {
-  fetchSourceHttp,
-  extractContent,
-  detectBotBlock,
-  checkContentQuality,
-  defaultFetchHeaders,
-  isPrivateUrl
-} from "../fetcher.mjs";
-import { fetchGitHubContent, parseGitHubUrl } from "../github.mjs";
-import { fetchRedditContent, parseRedditUrl } from "../reddit.mjs";
-import { trimContentHeadTail } from "../utils/content.mjs";
-import { cdp, closeTab, openNewTab } from "./chrome.mjs";
-import { SOURCE_FETCH_CONCURRENCY } from "./constants.mjs";
-import { extractPdfMarkdown } from "./pdf.mjs";
-import { trimText } from "./sources.mjs";
-async function fetchSourceViaChrome(tab, url, maxChars = 8000) {
-  const start = Date.now();
-  try {
-    const frames = await cdp(["evalraw", tab, "Page.getFrameTree", "{}"]).then((r) => JSON.parse(r)).catch(() => null);
-    const frameId = frames?.frameTree?.frame?.id || undefined;
-    const raw = await cdp([
-      "evalraw",
-      tab,
-      "Network.loadNetworkResource",
-      JSON.stringify({
-        frameId,
-        url,
-        options: { disableCache: true, includeCredentials: false }
-      })
-    ], 20000);
-    const result = JSON.parse(raw);
-    const resource = result.resource;
-    if (!resource?.success || !resource.httpStatusCode) {
-      return {
-        url,
-        error: resource?.netErrorName || resource?.netError || "loadNetworkResource failed",
-        source: "chrome",
-        duration: Date.now() - start,
-        needsFallback: true
-      };
-    }
-    let body = "";
-    if (resource.stream) {
-      try {
-        const ioRaw = await cdp([
-          "evalraw",
-          tab,
-          "IO.read",
-          JSON.stringify({ handle: resource.stream })
-        ], 1e4);
-        const ioResult = JSON.parse(ioRaw);
-        body = ioResult.data || "";
-        await cdp([
-          "evalraw",
-          tab,
-          "IO.close",
-          JSON.stringify({ handle: resource.stream })
-        ]).catch(() => {});
-      } catch {}
-    }
-    if (!body || body.length < 100) {
-      return {
-        url,
-        error: "Empty response body from Network.loadNetworkResource",
-        source: "chrome",
-        duration: Date.now() - start,
-        needsFallback: true
-      };
-    }
-    const botCheck = detectBotBlock(resource.httpStatusCode, body, url, url);
-    if (botCheck.blocked) {
-      return {
-        url,
-        status: resource.httpStatusCode,
-        error: `Blocked: ${botCheck.reason}`,
-        source: "chrome",
-        duration: Date.now() - start,
-        needsBrowser: true
-      };
-    }
-    const extracted = await extractContent(body, url);
-    const quality = checkContentQuality(extracted);
-    if (!quality.ok) {
-      return {
-        url,
-        status: resource.httpStatusCode,
-        error: `Low quality: ${quality.reason}`,
-        source: "chrome",
-        duration: Date.now() - start,
-        needsBrowser: true
-      };
-    }
-    const content = trimContentHeadTail(extracted.markdown, maxChars);
-    return {
-      url,
-      finalUrl: url,
-      status: resource.httpStatusCode,
-      contentType: "text/markdown",
-      lastModified: "",
-      publishedTime: extracted.publishedTime || "",
-      byline: extracted.byline || "",
-      siteName: extracted.siteName || "",
-      lang: extracted.lang || "",
-      title: extracted.title || url,
-      snippet: extracted.excerpt,
-      content,
-      contentChars: content.length,
-      source: "chrome",
-      duration: Date.now() - start
-    };
-  } catch (error) {
-    return {
-      url,
-      error: error.message,
-      source: "chrome",
-      duration: Date.now() - start,
-      needsFallback: true
-    };
-  }
-}
-function isLikelyPdfUrl(url) {
-  try {
-    const parsed = new URL(url);
-    return parsed.pathname.toLowerCase().endsWith(".pdf");
-  } catch {
-    return false;
-  }
-}
-async function fetchPdfSourceHttp(url, maxChars = 8000) {
-  const privateCheck = isPrivateUrl(url);
-  if (privateCheck.blocked) {
-    return {
-      url,
-      finalUrl: url,
-      status: 403,
-      error: `Blocked: ${privateCheck.reason}`,
-      source: "pdf-http"
-    };
-  }
-  const controller = new AbortController;
-  const timeoutId = setTimeout(() => controller.abort(), 20000);
-  const start = Date.now();
-  try {
-    const response = await fetch(url, {
-      method: "GET",
-      redirect: "follow",
-      signal: controller.signal,
-      headers: defaultFetchHeaders({
-        accept: "application/pdf,application/octet-stream;q=0.9,*/*;q=0.5"
-      })
-    });
-    clearTimeout(timeoutId);
-    const contentType = response.headers.get("content-type") || "";
-    const finalUrl = response.url || url;
-    const contentLength = Number.parseInt(response.headers.get("content-length") || "0", 10);
-    if (response.status >= 400) {
-      return {
-        url,
-        finalUrl,
-        status: response.status,
-        error: `HTTP ${response.status}`,
-        source: "pdf-http",
-        duration: Date.now() - start
-      };
-    }
-    if (!contentType.toLowerCase().includes("application/pdf") && !isLikelyPdfUrl(finalUrl)) {
-      return null;
-    }
-    if (contentLength > 30 * 1024 * 1024) {
-      return {
-        url,
-        finalUrl,
-        status: response.status,
-        error: `PDF too large: ${contentLength} bytes`,
-        source: "pdf-http",
-        duration: Date.now() - start
-      };
-    }
-    const buffer = Buffer.from(await response.arrayBuffer());
-    const pdf = await extractPdfMarkdown(buffer, finalUrl);
-    if (!pdf || pdf.error) {
-      return {
-        url,
-        finalUrl,
-        status: response.status,
-        error: pdf?.error || "PDF text extraction failed",
-        source: "pdf-http",
-        duration: Date.now() - start
-      };
-    }
-    const content = trimContentHeadTail(pdf.content, maxChars);
-    return {
-      url,
-      finalUrl,
-      status: response.status,
-      contentType: "application/pdf",
-      lastModified: response.headers.get("last-modified") || "",
-      title: pdf.title,
-      snippet: trimText(content, 320),
-      content,
-      contentChars: content.length,
-      pages: pdf.pages,
-      source: "pdf-http",
-      duration: Date.now() - start
-    };
-  } catch (error) {
-    clearTimeout(timeoutId);
-    return {
-      url,
-      finalUrl: url,
-      error: error.message || String(error),
-      source: "pdf-http",
-      duration: Date.now() - start
-    };
-  }
-}
-export async function fetchSourceContent(url, maxChars = 8000) {
-  const start = Date.now();
-  if (isLikelyPdfUrl(url)) {
-    const pdfResult = await fetchPdfSourceHttp(url, maxChars);
-    if (pdfResult?.content || pdfResult?.status === 403)
-      return pdfResult;
-  }
-  if (parseGitHubUrl(url)) {
-    const parsed = parseGitHubUrl(url);
-    if (parsed && (parsed.type === "root" || parsed.type === "tree" || parsed.type === "blob" && !parsed.path?.includes("."))) {
-      const ghResult = await fetchGitHubContent(url);
-      if (ghResult.ok) {
-        const content = trimContentHeadTail(ghResult.content, maxChars);
-        return {
-          url,
-          finalUrl: url,
-          status: 200,
-          contentType: "text/markdown",
-          lastModified: "",
-          title: ghResult.title,
-          snippet: content.slice(0, 320),
-          content,
-          contentChars: content.length,
-          source: "github-api",
-          ...ghResult.tree && { tree: ghResult.tree },
-          duration: Date.now() - start
-        };
-      }
-      process.stderr.write(`[greedysearch] GitHub API fetch failed, trying HTTP: ${ghResult.error}
-`);
-    }
-  }
-  const redditInfo = parseRedditUrl(url);
-  if (redditInfo?.type === "post") {
-    process.stderr.write(`[greedysearch] Using Reddit JSON API for: ${url.slice(0, 60)}...
-`);
-    const redditResult = await fetchRedditContent(url, maxChars);
-    if (redditResult.ok) {
-      const content = trimContentHeadTail(redditResult.markdown, maxChars);
-      return {
-        url,
-        finalUrl: redditResult.finalUrl,
-        status: redditResult.status,
-        contentType: "text/markdown",
-        lastModified: redditResult.lastModified || "",
-        publishedTime: redditResult.publishedTime || "",
-        byline: redditResult.byline || "",
-        siteName: redditResult.siteName || "",
-        lang: redditResult.lang || "",
-        title: redditResult.title,
-        snippet: redditResult.excerpt,
-        content,
-        contentChars: content.length,
-        source: "reddit-api",
-        duration: Date.now() - start
-      };
-    }
-    process.stderr.write(`[greedysearch] Reddit API fetch failed, falling back to HTTP: ${redditResult.error}
-`);
-  }
-  const httpResult = await fetchSourceHttp(url, { timeoutMs: 1e4 });
-  if (httpResult.ok) {
-    const content = trimContentHeadTail(httpResult.markdown, maxChars);
-    return {
-      url,
-      finalUrl: httpResult.finalUrl,
-      status: httpResult.status,
-      contentType: "text/markdown",
-      lastModified: httpResult.lastModified || "",
-      publishedTime: httpResult.publishedTime || "",
-      byline: httpResult.byline || "",
-      siteName: httpResult.siteName || "",
-      lang: httpResult.lang || "",
-      title: httpResult.title,
-      snippet: httpResult.excerpt,
-      content,
-      contentChars: content.length,
-      source: "http",
-      duration: Date.now() - start
-    };
-  }
-  if (httpResult.needsBrowser) {
-    try {
-      const chromeTab = await openNewTab();
-      try {
-        const chromeResult = await fetchSourceViaChrome(chromeTab, url, maxChars);
-        if (chromeResult.content && chromeResult.content.length > 100) {
-          return chromeResult;
-        }
-      } finally {
-        await closeTab(chromeTab);
-      }
-    } catch {}
-  }
-  process.stderr.write(`[greedysearch] HTTP failed for ${url.slice(0, 60)}, trying browser...
-`);
-  return await fetchSourceContentBrowser(url, maxChars);
-}
-async function waitForPageReady(tab, timeoutMs = 4000, intervalMs = 200) {
-  const deadline = Date.now() + timeoutMs;
-  while (Date.now() < deadline) {
-    try {
-      const ready = await cdp([
-        "eval",
-        tab,
-        `document.readyState === "complete" && !!document.body && document.body.innerText.length > 500`
-      ]);
-      if (ready.trim() === "true")
-        return;
-    } catch {}
-    await new Promise((r) => setTimeout(r, intervalMs));
-  }
-}
-async function fetchSourceContentBrowser(url, maxChars = 8000) {
-  const start = Date.now();
-  let tab;
-  try {
-    tab = await openNewTab();
-  } catch (e) {
-    return {
-      url,
-      title: "",
-      content: null,
-      snippet: "",
-      contentChars: 0,
-      error: `openNewTab failed: ${e.message}`,
-      source: "browser",
-      duration: Date.now() - start
-    };
-  }
-  try {
-    await cdp(["nav", tab, url], 30000);
-    await waitForPageReady(tab);
-    const content = await cdp([
-      "eval",
-      tab,
-      String.raw`
-			(function(){
-				var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
-				var text = (el || document.body).innerText;
-				return JSON.stringify({
-					title: document.title,
-					content: text.replace(/\s+/g, ' ').trim(),
-					url: location.href
-				});
-			})()
-		`
-    ]);
-    const parsed = JSON.parse(content);
-    const finalContent = trimContentHeadTail(parsed.content, maxChars);
-    return {
-      url,
-      finalUrl: parsed.url || url,
-      status: 200,
-      contentType: "text/plain",
-      lastModified: "",
-      title: parsed.title,
-      snippet: trimText(finalContent, 320),
-      content: finalContent,
-      contentChars: finalContent.length,
-      source: "browser",
-      duration: Date.now() - start
-    };
-  } catch (error) {
-    return {
-      url,
-      title: "",
-      content: null,
-      snippet: "",
-      contentChars: 0,
-      error: error.message,
-      source: "browser",
-      duration: Date.now() - start
-    };
-  } finally {
-    await closeTab(tab);
-  }
-}
-export async function fetchMultipleSources(sources, maxSources = 5, maxChars = 8000, concurrency = SOURCE_FETCH_CONCURRENCY) {
-  const toFetch = sources.slice(0, maxSources);
-  if (toFetch.length === 0)
-    return [];
-  const workerCount = Math.min(toFetch.length, Math.max(1, Number.parseInt(String(concurrency), 10) || SOURCE_FETCH_CONCURRENCY));
-  process.stderr.write(`[greedysearch] Fetching content from ${toFetch.length} sources via HTTP (concurrency ${workerCount})...
-`);
-  const fetched = new Array(toFetch.length);
-  let nextIndex = 0;
-  let completed = 0;
-  async function worker() {
-    while (true) {
-      const index = nextIndex++;
-      if (index >= toFetch.length)
-        return;
-      const s = toFetch[index];
-      const url = s.canonicalUrl || s.url;
-      process.stderr.write(`[greedysearch] [${index + 1}/${toFetch.length}] Fetching: ${url.slice(0, 60)}...
-`);
-      const result = await fetchSourceContent(url, maxChars).catch((e) => ({
-        url,
-        title: "",
-        content: null,
-        snippet: "",
-        contentChars: 0,
-        error: e.message,
-        source: "error",
-        duration: 0
-      }));
-      fetched[index] = {
-        id: s.id,
-        ...result
-      };
-      if (result.content && result.content.length > 100) {
-        process.stderr.write(`[greedysearch] ✓ ${result.source}: ${result.content.length} chars
-`);
-      } else if (result.error) {
-        process.stderr.write(`[greedysearch] ✗ ${result.error.slice(0, 80)}
-`);
-      }
-      completed += 1;
-      process.stderr.write(`PROGRESS:fetch:${completed}/${toFetch.length}
-`);
-    }
-  }
-  await Promise.all(Array.from({ length: workerCount }, () => worker()));
-  const successful = fetched.filter((f) => f.content && f.content.length > 100);
-  const httpCount = fetched.filter((f) => f.source === "http").length;
-  const browserCount = fetched.filter((f) => f.source === "browser").length;
-  process.stderr.write(`[greedysearch] Fetched ${successful.length}/${fetched.length} sources ` + `(HTTP: ${httpCount}, Browser: ${browserCount})
-`);
-  return fetched;
-}
-export async function fetchTopSource(url) {
-  const tab = await openNewTab();
-  try {
-    await cdp(["nav", tab, url], 30000);
-    await waitForPageReady(tab);
-    const content = await cdp([
-      "eval",
-      tab,
-      String.raw`
-      (function(){
-        var el = document.querySelector('article, [role="main"], main, .post-content, .article-body, #content, .content');
-        var text = (el || document.body).innerText;
-        return text.replace(/\s+/g, ' ').trim();
-      })()
-    `
-    ]);
-    return { url, content };
-  } catch (e) {
-    return { url, content: null, error: e.message };
-  } finally {
-    await closeTab(tab);
-  }
-}

package/extensions/greedysearch-dm/src/search/file-sources.mjs

@@ -1,31 +0,0 @@
-import { mkdirSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-const DEFAULT_DIR = join(process.cwd(), ".dm", "greedysearch-sources");
-export function writeSourcesToFiles(fetchedSources, dir = DEFAULT_DIR) {
-  mkdirSync(dir, { recursive: true });
-  return fetchedSources.map((source) => {
-    if (!source.content || source.content.length < 10) {
-      return source;
-    }
-    const safeId = String(source.id || "unknown").replace(/[^a-zA-Z0-9_-]/g, "");
-    const urlSlug = (source.canonicalUrl || source.url || "").replace(/^https?:\/\//, "").replace(/[^a-zA-Z0-9]/g, "-").slice(0, 40);
-    const filename = `${safeId}-${urlSlug}.md`;
-    const filepath = join(dir, filename);
-    const header = `---
-url: ${source.finalUrl || source.url}
-title: ${source.title || ""}
-source: ${source.source || "unknown"}
-status: ${source.status || ""}
-chars: ${source.contentChars || source.content.length}
----
-
-`;
-    writeFileSync(filepath, header + source.content, "utf8");
-    const { content, ...rest } = source;
-    return {
-      ...rest,
-      contentPath: filepath,
-      contentChars: source.contentChars || content.length
-    };
-  });
-}

package/extensions/greedysearch-dm/src/search/output.mjs

@@ -1,51 +0,0 @@
-import { mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-const __dir = import.meta.dirname || new URL(".", import.meta.url).pathname.replace(/^\/([A-Z]:)/, "$1");
-export function slugify(query) {
-  return query.toLowerCase().replaceAll(/[^a-z0-9]+/g, "-").replaceAll(/^-|-$/g, "").slice(0, 60);
-}
-const RESULTS_MAX_AGE_MS = 7 * 24 * 60 * 60 * 1000;
-const RESULTS_MIN_KEEP = 10;
-function purgeOldResults(dir) {
-  try {
-    const files = readdirSync(dir).filter((f) => f.endsWith(".json") || f.endsWith(".md")).map((f) => ({ f, mtime: statSync(join(dir, f)).mtimeMs })).sort((a, b) => b.mtime - a.mtime);
-    const cutoff = Date.now() - RESULTS_MAX_AGE_MS;
-    for (let i = RESULTS_MIN_KEEP;i < files.length; i++) {
-      if (files[i].mtime < cutoff) {
-        rmSync(join(dir, files[i].f), { force: true });
-      }
-    }
-  } catch {}
-}
-export function resultsDir() {
-  const dir = join(__dir, "..", "..", "results");
-  mkdirSync(dir, { recursive: true });
-  purgeOldResults(dir);
-  return dir;
-}
-export function writeOutput(data, outFile, { inline = false, synthesize = false, query = "" } = {}) {
-  const json = `${JSON.stringify(data, null, 2)}
-`;
-  if (outFile) {
-    writeFileSync(outFile, json, "utf8");
-    process.stderr.write(`Results written to ${outFile}
-`);
-    return;
-  }
-  if (inline) {
-    process.stdout.write(json);
-    return;
-  }
-  const ts = new Date().toISOString().replaceAll("T", "_").replaceAll(/[:.]/g, "-").slice(0, 19);
-  const slug = slugify(query);
-  const base = join(resultsDir(), `${ts}_${slug}`);
-  writeFileSync(`${base}.json`, json, "utf8");
-  if (synthesize && data._synthesis?.answer) {
-    writeFileSync(`${base}-synthesis.md`, data._synthesis.answer, "utf8");
-    process.stdout.write(`${base}-synthesis.md
-`);
-  } else {
-    process.stdout.write(`${base}.json
-`);
-  }
-}

package/extensions/greedysearch-dm/src/search/paths.mjs

@@ -1,48 +0,0 @@
-import { existsSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { fileURLToPath } from "node:url";
-export function moduleDirectory(metaUrl) {
-  return fileURLToPath(new URL(".", metaUrl)).replace(/^\/([A-Z]:)/, "$1");
-}
-function uniqueExisting(candidates, exists = existsSync) {
-  const seen = new Set;
-  for (const candidate of candidates.filter(Boolean)) {
-    if (seen.has(candidate))
-      continue;
-    seen.add(candidate);
-    if (exists(candidate))
-      return candidate;
-  }
-  return candidates.filter(Boolean).at(-1);
-}
-export function resolveGreedySearchExtractorScript(script, {
-  moduleDir,
-  entrypoint = process.argv[1],
-  env = process.env,
-  exists = existsSync
-} = {}) {
-  const entryDir = entrypoint ? dirname(entrypoint) : null;
-  const extensionDir = env.GREEDY_SEARCH_EXTENSION_DIR?.trim() || null;
-  return uniqueExisting([
-    moduleDir ? join(moduleDir, "..", "..", "extractors", script) : null,
-    moduleDir ? join(moduleDir, "..", "extractors", script) : null,
-    extensionDir ? join(extensionDir, "extractors", script) : null,
-    entryDir ? join(entryDir, "..", "extractors", script) : null
-  ], exists);
-}
-export function resolveGreedySearchLaunchScript({
-  moduleDir,
-  entrypoint = process.argv[1],
-  env = process.env,
-  exists = existsSync
-} = {}) {
-  const entryDir = entrypoint ? dirname(entrypoint) : null;
-  const extensionDir = env.GREEDY_SEARCH_EXTENSION_DIR?.trim() || null;
-  return uniqueExisting([
-    moduleDir ? join(moduleDir, "..", "..", "bin", "launch.mjs") : null,
-    moduleDir ? join(moduleDir, "launch.mjs") : null,
-    extensionDir ? join(extensionDir, "bin", "launch.mjs") : null,
-    entryDir ? join(entryDir, "launch.mjs") : null,
-    entryDir ? join(entryDir, "..", "bin", "launch.mjs") : null
-  ], exists);
-}

package/extensions/greedysearch-dm/src/search/pdf.mjs

@@ -1,64 +0,0 @@
-function ensurePdfDomPolyfills() {
-  if (typeof globalThis.DOMMatrix === "undefined") {
-    globalThis.DOMMatrix = class DOMMatrix {
-      constructor(_init = undefined) {}
-      multiplySelf() {
-        return this;
-      }
-      preMultiplySelf() {
-        return this;
-      }
-      translateSelf() {
-        return this;
-      }
-      scaleSelf() {
-        return this;
-      }
-      rotateSelf() {
-        return this;
-      }
-    };
-  }
-  if (typeof globalThis.ImageData === "undefined") {
-    globalThis.ImageData = class ImageData {
-      constructor(data = undefined, width = 0, height = 0) {
-        this.data = data;
-        this.width = width;
-        this.height = height;
-      }
-    };
-  }
-  if (typeof globalThis.Path2D === "undefined") {
-    globalThis.Path2D = class Path2D {
-      constructor(_path = undefined) {}
-    };
-  }
-}
-async function loadPdfParseCtor() {
-  ensurePdfDomPolyfills();
-  const mod = await import("pdf-parse");
-  const ctor = mod.PDFParse ?? mod.default;
-  if (!ctor)
-    throw new Error("pdf-parse did not export PDFParse");
-  return ctor;
-}
-export async function extractPdfMarkdown(buffer, url) {
-  try {
-    const PDFParseCtor = await loadPdfParseCtor();
-    const parser = new PDFParseCtor({ data: new Uint8Array(buffer) });
-    await parser.load();
-    const data = await parser.getText();
-    const text = data.text?.trim();
-    if (!text)
-      return null;
-    return {
-      title: new URL(url).pathname.split("/").pop() || "Document.pdf",
-      content: `## PDF Content (${data.total} pages)
-
-${text}`,
-      pages: data.total
-    };
-  } catch (error) {
-    return { error: error.message || String(error) };
-  }
-}