@kenkaiiii/ggcoder 4.3.231 → 4.3.232

package/dist/tools/html-extract.js

@@ -0,0 +1,130 @@
+/**
+ * Minimum article length (in characters, per Readability's `length` field)
+ * required before we trust the extraction. Below this we fall back to the
+ * regex-based `htmlToCleanText` so short/empty pages don't degrade output.
+ */
+const CHAR_THRESHOLD = 250;
+/** Raised when one of the optional extraction dependencies is not installed. */
+export class ExtractorUnavailable extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "ExtractorUnavailable";
+    }
+}
+let cached = null;
+/**
+ * Lazily resolve the HTML→Markdown extraction stack (linkedom, Readability,
+ * Turndown + GFM plugin). Mirrors `loadChromium()` in screenshot.ts: dynamic
+ * imports behind a non-literal specifier so `tsc` does not statically require
+ * the optional dependencies, with results cached. Throws `ExtractorUnavailable`
+ * if any are missing so callers can degrade to the regex path.
+ */
+export async function loadExtractor() {
+    if (cached)
+        return cached;
+    const linkedomName = "linkedom";
+    const readabilityName = "@mozilla/readability";
+    const turndownName = "turndown";
+    const gfmName = "turndown-plugin-gfm";
+    try {
+        const [linkedom, readability, turndown, gfmMod] = (await Promise.all([
+            import(linkedomName),
+            import(readabilityName),
+            import(turndownName),
+            import(gfmName),
+        ]));
+        if (!linkedom.parseHTML || !readability.Readability || !turndown.default || !gfmMod.gfm) {
+            throw new ExtractorUnavailable("extraction modules loaded but expected exports are missing");
+        }
+        cached = {
+            parseHTML: linkedom.parseHTML,
+            Readability: readability.Readability,
+            TurndownService: turndown.default,
+            gfm: gfmMod.gfm,
+        };
+        return cached;
+    }
+    catch (err) {
+        if (err instanceof ExtractorUnavailable)
+            throw err;
+        throw new ExtractorUnavailable(err instanceof Error ? err.message : "failed to load extraction modules");
+    }
+}
+function buildTurndown(mods) {
+    const service = new mods.TurndownService({
+        headingStyle: "atx",
+        codeBlockStyle: "fenced",
+        bulletListMarker: "-",
+    });
+    service.use(mods.gfm);
+    // Preserve fenced code blocks: keep <pre>/<code> structure intact rather
+    // than collapsing whitespace inside them.
+    service.addRule("fencedPre", {
+        filter: ["pre"],
+        replacement(_content, node) {
+            const code = (node.textContent ?? "").replace(/\n$/, "");
+            return `\n\n\`\`\`\n${code}\n\`\`\`\n\n`;
+        },
+    });
+    // Drop anchors without an href (navigation/JS noise) — keep their text only.
+    service.addRule("bareAnchor", {
+        filter(node) {
+            return node.nodeName === "A" && !node.getAttribute("href");
+        },
+        replacement(content) {
+            return content;
+        },
+    });
+    return service;
+}
+/**
+ * Run Readability over the page HTML and convert the extracted main content to
+ * clean Markdown (headings, lists, GFM tables, fenced code preserved). Returns
+ * `null` when extraction fails or the article is too short, so the caller can
+ * fall back to `htmlToCleanText`.
+ */
+export async function extractToMarkdown(html, url) {
+    const mods = await loadExtractor();
+    let document;
+    try {
+        document = mods.parseHTML(html).document;
+    }
+    catch {
+        return null;
+    }
+    // Readability reads <base>/document URL for resolving links; set a baseURI
+    // hint via a <base> tag when none is present so relative links resolve.
+    try {
+        if (!document.querySelector("base")) {
+            const base = document.createElement("base");
+            base.setAttribute("href", url);
+            document.head?.appendChild(base);
+        }
+    }
+    catch {
+        // best-effort; ignore if the DOM shim disallows it
+    }
+    let article;
+    try {
+        article = new mods.Readability(document, { charThreshold: CHAR_THRESHOLD }).parse();
+    }
+    catch {
+        return null;
+    }
+    if (!article || !article.content)
+        return null;
+    if ((article.length ?? 0) < CHAR_THRESHOLD)
+        return null;
+    let markdown;
+    try {
+        markdown = buildTurndown(mods).turndown(article.content);
+    }
+    catch {
+        return null;
+    }
+    markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
+    if (!markdown)
+        return null;
+    return { markdown, title: article.title ?? undefined };
+}
+//# sourceMappingURL=html-extract.js.map

package/dist/tools/html-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-extract.js","sourceRoot":"","sources":["../../src/tools/html-extract.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,cAAc,GAAG,GAAG,CAAC;AAE3B,gFAAgF;AAChF,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AA0CD,IAAI,MAAM,GAA4B,IAAI,CAAC;AAE3C;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,MAAM,YAAY,GAAW,UAAU,CAAC;IACxC,MAAM,eAAe,GAAW,sBAAsB,CAAC;IACvD,MAAM,YAAY,GAAW,UAAU,CAAC;IACxC,MAAM,OAAO,GAAW,qBAAqB,CAAC;IAE9C,IAAI,CAAC;QACH,MAAM,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC;YACnE,MAAM,CAAC,YAAY,CAAC;YACpB,MAAM,CAAC,eAAe,CAAC;YACvB,MAAM,CAAC,YAAY,CAAC;YACpB,MAAM,CAAC,OAAO,CAAC;SAChB,CAAC,CAAmE,CAAC;QAEtE,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,CAAC,WAAW,CAAC,WAAW,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YACxF,MAAM,IAAI,oBAAoB,CAAC,4DAA4D,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,GAAG;YACP,SAAS,EAAE,QAAQ,CAAC,SAAS;YAC7B,WAAW,EAAE,WAAW,CAAC,WAAW;YACpC,eAAe,EAAE,QAAQ,CAAC,OAAO;YACjC,GAAG,EAAE,MAAM,CAAC,GAAG;SAChB,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,oBAAoB;YAAE,MAAM,GAAG,CAAC;QACnD,MAAM,IAAI,oBAAoB,CAC5B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,mCAAmC,CACzE,CAAC;IACJ,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAsB;IAC3C,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC;QACvC,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,gBAAgB,EAAE,GAAG;KACtB,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtB,yEAAyE;IACzE,0CAA0C;IAC1C,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE;QAC3B,MAAM,EAAE,CAAC,KAAK,CAAC;QACf,WAAW,CAAC,QAAgB,EAAE,IAAqC;YACjE,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACzD,OAAO,eAAe,IAAI,cAAc,CAAC;QAC3C,CAAC;KACF,CAAC,CAAC;IACH,6EAA6E;IAC7E,OAAO,CAAC,OAAO,CAAC,YAAY,EAAE;QAC5B,MAAM,CAAC,IAAqE;YAC1E,OAAO,IAAI,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC7D,CAAC;QACD,WAAW,CAAC,OAAe;YACzB,OAAO,OAAO,CAAC;QACjB,CAAC;KACF,CAAC,CAAC;IACH,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,GAAW;IAEX,MAAM,IAAI,GAAG,MAAM,aAAa,EAAE,CAAC;IAEnC,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,2EAA2E;IAC3E,wEAAwE;IACxE,IAAI,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC/B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,mDAAmD;IACrD,CAAC;IAED,IAAI,OAAkC,CAAC;IACvC,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,cAAc,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IACtF,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,cAAc;QAAE,OAAO,IAAI,CAAC;IAExD,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IAED,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IACtD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,SAAS,EAAE,CAAC;AACzD,CAAC"}

package/dist/tools/html-extract.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=html-extract.test.d.ts.map

package/dist/tools/html-extract.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-extract.test.d.ts","sourceRoot":"","sources":["../../src/tools/html-extract.test.ts"],"names":[],"mappings":""}

package/dist/tools/html-extract.test.js

@@ -0,0 +1,60 @@
+import { describe, expect, it } from "vitest";
+import { extractToMarkdown, loadExtractor } from "./html-extract.js";
+// Probe at module load so describe.skipIf sees the real value at collection time.
+const extractorInstalled = await loadExtractor()
+    .then(() => true)
+    .catch(() => false);
+const ARTICLE_HTML = `
+<html>
+  <head><title>Test Article</title></head>
+  <body>
+    <nav>Home Products Pricing Docs</nav>
+    <aside class="advert">Sponsored: buy now</aside>
+    <article>
+      <h1>Understanding the Fetch API</h1>
+      <p>The Fetch API provides a modern interface for making HTTP requests in
+      JavaScript. It returns promises and is widely supported across browsers.
+      This paragraph is intentionally long enough to clear the Readability
+      content-length threshold so extraction succeeds in tests reliably.</p>
+      <h2>Example</h2>
+      <pre><code>const res = await fetch(url);
+const data = await res.json();</code></pre>
+      <h2>Comparison</h2>
+      <table>
+        <thead><tr><th>Method</th><th>Returns</th></tr></thead>
+        <tbody>
+          <tr><td>fetch</td><td>Promise</td></tr>
+          <tr><td>XHR</td><td>callback</td></tr>
+        </tbody>
+      </table>
+      <ul><li>First item</li><li>Second item</li></ul>
+      <p>A closing paragraph with more substantial body text so the article
+      comfortably exceeds the minimum length that Readability requires before it
+      is considered a real article worth extracting.</p>
+    </article>
+    <footer>Legal links and copyright</footer>
+  </body>
+</html>
+`;
+describe.skipIf(!extractorInstalled)("extractToMarkdown", () => {
+    it("extracts main content as markdown with title, code fence, and table", async () => {
+        const result = await extractToMarkdown(ARTICLE_HTML, "https://example.com/article");
+        expect(result).not.toBeNull();
+        const md = result?.markdown ?? "";
+        expect(result?.title).toBe("Test Article");
+        expect(md).toContain("Understanding the Fetch API");
+        expect(md).toContain("```");
+        expect(md).toContain("await fetch(url)");
+        // GFM table rendered with pipe separators.
+        expect(md).toMatch(/\|\s*Method\s*\|/);
+        expect(md).toContain("First item");
+        // Nav, ads, and footer boilerplate dropped.
+        expect(md).not.toContain("Sponsored: buy now");
+        expect(md).not.toContain("Home Products Pricing");
+    });
+    it("returns null for trivially short content", async () => {
+        const result = await extractToMarkdown("<html><body><p>Too short.</p></body></html>", "https://example.com/short");
+        expect(result).toBeNull();
+    });
+});
+//# sourceMappingURL=html-extract.test.js.map

package/dist/tools/html-extract.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-extract.test.js","sourceRoot":"","sources":["../../src/tools/html-extract.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAErE,kFAAkF;AAClF,MAAM,kBAAkB,GAAG,MAAM,aAAa,EAAE;KAC7C,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC;KAChB,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;AAEtB,MAAM,YAAY,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+BpB,CAAC;AAEF,QAAQ,CAAC,MAAM,CAAC,CAAC,kBAAkB,CAAC,CAAC,mBAAmB,EAAE,GAAG,EAAE;IAC7D,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,YAAY,EAAE,6BAA6B,CAAC,CAAC;QACpF,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,EAAE,GAAG,MAAM,EAAE,QAAQ,IAAI,EAAE,CAAC;QAElC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC3C,MAAM,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,6BAA6B,CAAC,CAAC;QACpD,MAAM,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QAC5B,MAAM,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;QACzC,2CAA2C;QAC3C,MAAM,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACvC,MAAM,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QACnC,4CAA4C;QAC5C,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC;QAC/C,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;QACxD,MAAM,MAAM,GAAG,MAAM,iBAAiB,CACpC,6CAA6C,EAC7C,2BAA2B,CAC5B,CAAC;QACF,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/tools/pdf-extract.d.ts

@@ -0,0 +1,18 @@
+/**
+ * PDF text extraction via the optional `unpdf` dependency (bundles pdf.js,
+ * zero-config). Lazy-loaded behind a function so the base install works without
+ * it; callers degrade to an install-hint string when it is absent.
+ */
+/** Raised when the optional `unpdf` dependency is not installed. */
+export declare class PdfExtractorUnavailable extends Error {
+    constructor(message: string);
+}
+/**
+ * Extract merged page text from a PDF buffer. Returns the concatenated text and
+ * total page count. Throws `PdfExtractorUnavailable` if `unpdf` is not present.
+ */
+export declare function extractPdfText(bytes: Uint8Array): Promise<{
+    text: string;
+    pages: number;
+}>;
+//# sourceMappingURL=pdf-extract.d.ts.map

package/dist/tools/pdf-extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extract.d.ts","sourceRoot":"","sources":["../../src/tools/pdf-extract.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,oEAAoE;AACpE,qBAAa,uBAAwB,SAAQ,KAAK;gBACpC,OAAO,EAAE,MAAM;CAI5B;AA6BD;;;GAGG;AACH,wBAAsB,cAAc,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAKhG"}

package/dist/tools/pdf-extract.js

@@ -0,0 +1,43 @@
+/**
+ * PDF text extraction via the optional `unpdf` dependency (bundles pdf.js,
+ * zero-config). Lazy-loaded behind a function so the base install works without
+ * it; callers degrade to an install-hint string when it is absent.
+ */
+/** Raised when the optional `unpdf` dependency is not installed. */
+export class PdfExtractorUnavailable extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "PdfExtractorUnavailable";
+    }
+}
+let cached = null;
+async function loadUnpdf() {
+    if (cached)
+        return cached;
+    // Non-literal specifier so tsc does not statically resolve the optional dep.
+    const moduleName = "unpdf";
+    try {
+        const mod = (await import(moduleName));
+        if (!mod.getDocumentProxy || !mod.extractText) {
+            throw new PdfExtractorUnavailable("unpdf loaded but expected exports are missing");
+        }
+        cached = mod;
+        return cached;
+    }
+    catch (err) {
+        if (err instanceof PdfExtractorUnavailable)
+            throw err;
+        throw new PdfExtractorUnavailable(err instanceof Error ? err.message : "failed to load unpdf");
+    }
+}
+/**
+ * Extract merged page text from a PDF buffer. Returns the concatenated text and
+ * total page count. Throws `PdfExtractorUnavailable` if `unpdf` is not present.
+ */
+export async function extractPdfText(bytes) {
+    const unpdf = await loadUnpdf();
+    const pdf = await unpdf.getDocumentProxy(bytes);
+    const { totalPages, text } = await unpdf.extractText(pdf, { mergePages: true });
+    return { text, pages: totalPages };
+}
+//# sourceMappingURL=pdf-extract.js.map

package/dist/tools/pdf-extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extract.js","sourceRoot":"","sources":["../../src/tools/pdf-extract.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,oEAAoE;AACpE,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAChD,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AAUD,IAAI,MAAM,GAAuB,IAAI,CAAC;AAEtC,KAAK,UAAU,SAAS;IACtB,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAC1B,6EAA6E;IAC7E,MAAM,UAAU,GAAW,OAAO,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,CAA2B,CAAC;QACjE,IAAI,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAC9C,MAAM,IAAI,uBAAuB,CAAC,+CAA+C,CAAC,CAAC;QACrF,CAAC;QACD,MAAM,GAAG,GAAG,CAAC;QACb,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,GAAG,YAAY,uBAAuB;YAAE,MAAM,GAAG,CAAC;QACtD,MAAM,IAAI,uBAAuB,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC;IACjG,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAiB;IACpD,MAAM,KAAK,GAAG,MAAM,SAAS,EAAE,CAAC;IAChC,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;IAChD,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAChF,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC;AACrC,CAAC"}

package/dist/tools/pdf-extract.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=pdf-extract.test.d.ts.map

package/dist/tools/pdf-extract.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extract.test.d.ts","sourceRoot":"","sources":["../../src/tools/pdf-extract.test.ts"],"names":[],"mappings":""}

package/dist/tools/pdf-extract.test.js

@@ -0,0 +1,15 @@
+import { readFile } from "node:fs/promises";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "vitest";
+import { extractPdfText } from "./pdf-extract.js";
+const fixturePath = fileURLToPath(new URL("./__fixtures__/sample.pdf", import.meta.url));
+const unpdfInstalled = await import("unpdf").then(() => true).catch(() => false);
+describe.skipIf(!unpdfInstalled)("extractPdfText", () => {
+    it("extracts text and page count from a minimal PDF", async () => {
+        const bytes = new Uint8Array(await readFile(fixturePath));
+        const { text, pages } = await extractPdfText(bytes);
+        expect(text).toContain("Hello PDF World");
+        expect(pages).toBe(1);
+    });
+});
+//# sourceMappingURL=pdf-extract.test.js.map

package/dist/tools/pdf-extract.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pdf-extract.test.js","sourceRoot":"","sources":["../../src/tools/pdf-extract.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAElD,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,2BAA2B,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAEzF,MAAM,cAAc,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC;AAEjF,QAAQ,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,CAAC,gBAAgB,EAAE,GAAG,EAAE;IACtD,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC;QAC1D,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC,KAAK,CAAC,CAAC;QAEpD,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAC1C,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACxB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/tools/prompt-hints.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"prompt-hints.d.ts","sourceRoot":"","sources":["../../src/tools/prompt-hints.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAyBpD,CAAC;AAEF,oEAAoE;AACpE,eAAO,MAAM,kBAAkB,EAAE,SAAS,MAAM,EAoB/C,CAAC"}
+{"version":3,"file":"prompt-hints.d.ts","sourceRoot":"","sources":["../../src/tools/prompt-hints.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CA4BpD,CAAC;AAEF,oEAAoE;AACpE,eAAO,MAAM,kBAAkB,EAAE,SAAS,MAAM,EAoB/C,CAAC"}

package/dist/tools/prompt-hints.js

@@ -12,7 +12,8 @@ export const TOOL_PROMPT_HINTS = {
     find: "Find files/dirs by name pattern. Faster than bash find, respects .gitignore.",
     grep: "Regex search across files. Use for usages, definitions, imports.",
     source_path: "Resolve installed package/repo source via opensrc. Use before assuming dependency APIs; inspect returned absolute path with read/grep/find/ls.",
-    web_search: "Search the web. Use before web_fetch to find pages.",
+    web_search: "Search the web. Use before web_fetch to find pages; supports include/exclude_domains and a time_range recency filter.",
+    web_fetch: "Fetch page content as Markdown (or text/html). Pass `urls` to fetch many at once; reads PDFs, follows safe redirects, and prefers a site's /llms.txt for docs.",
     task_output: "Read new output from a background process by id.",
     task_stop: "Stop a background process by id.",
     goals: "Manage durable Goal runs for /goal and Ctrl+G workflows. Use for Goal setup, coordinator evidence, worker tasks, verifier records, final completion audits, blockers, and completion state.",

package/dist/tools/prompt-hints.js.map

@@ -1 +1 @@
-{"version":3,"file":"prompt-hints.js","sourceRoot":"","sources":["../../src/tools/prompt-hints.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAA2B;IACvD,IAAI,EAAE,uDAAuD;IAC7D,KAAK,EAAE,gFAAgF;IACvF,IAAI,EAAE,kIAAkI;IACxI,IAAI,EAAE,oHAAoH;IAC1H,IAAI,EAAE,8EAA8E;IACpF,IAAI,EAAE,kEAAkE;IACxE,WAAW,EACT,gJAAgJ;IAClJ,UAAU,EAAE,qDAAqD;IACjE,WAAW,EAAE,kDAAkD;IAC/D,SAAS,EAAE,kCAAkC;IAC7C,KAAK,EACH,6LAA6L;IAC/L,UAAU,EACR,yGAAyG;IAC3G,SAAS,EAAE,0EAA0E;IACrF,QAAQ,EAAE,uEAAuE;IACjF,KAAK,EAAE,oDAAoD;IAC3D,uCAAuC,EACrC,qNAAqN;IACvN,oCAAoC,EAClC,6LAA6L;IAC/L,iCAAiC,EAC/B,+OAA+O;CAClP,CAAC;AAEF,oEAAoE;AACpE,MAAM,CAAC,MAAM,kBAAkB,GAAsB;IACnD,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,IAAI;IACJ,aAAa;IACb,WAAW;IACX,aAAa;IACb,WAAW;IACX,OAAO;IACP,YAAY;IACZ,WAAW;IACX,UAAU;IACV,OAAO;IACP,uCAAuC;IACvC,oCAAoC;IACpC,iCAAiC;CAClC,CAAC"}
+{"version":3,"file":"prompt-hints.js","sourceRoot":"","sources":["../../src/tools/prompt-hints.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAA2B;IACvD,IAAI,EAAE,uDAAuD;IAC7D,KAAK,EAAE,gFAAgF;IACvF,IAAI,EAAE,kIAAkI;IACxI,IAAI,EAAE,oHAAoH;IAC1H,IAAI,EAAE,8EAA8E;IACpF,IAAI,EAAE,kEAAkE;IACxE,WAAW,EACT,gJAAgJ;IAClJ,UAAU,EACR,uHAAuH;IACzH,SAAS,EACP,gKAAgK;IAClK,WAAW,EAAE,kDAAkD;IAC/D,SAAS,EAAE,kCAAkC;IAC7C,KAAK,EACH,6LAA6L;IAC/L,UAAU,EACR,yGAAyG;IAC3G,SAAS,EAAE,0EAA0E;IACrF,QAAQ,EAAE,uEAAuE;IACjF,KAAK,EAAE,oDAAoD;IAC3D,uCAAuC,EACrC,qNAAqN;IACvN,oCAAoC,EAClC,6LAA6L;IAC/L,iCAAiC,EAC/B,+OAA+O;CAClP,CAAC;AAEF,oEAAoE;AACpE,MAAM,CAAC,MAAM,kBAAkB,GAAsB;IACnD,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,IAAI;IACJ,aAAa;IACb,WAAW;IACX,aAAa;IACb,WAAW;IACX,OAAO;IACP,YAAY;IACZ,WAAW;IACX,UAAU;IACV,OAAO;IACP,uCAAuC;IACvC,oCAAoC;IACpC,iCAAiC;CAClC,CAAC"}

package/dist/tools/web-fetch.d.ts

@@ -6,10 +6,25 @@ import type { AgentTool } from "@kenkaiiii/gg-agent";
  */
 export declare function isBlockedUrl(urlString: string): boolean;
 export declare function htmlToCleanText(html: string): string;
+type LlmsCandidateKind = "llms" | "llms-full" | "llms-ctx" | "page-md";
+interface LlmsCandidate {
+    url: string;
+    label: string;
+    kind: LlmsCandidateKind;
+    priority: number;
+}
+export declare function buildLlmsCandidates(url: string, maxLength: number): LlmsCandidate[];
 export declare function createWebFetchTool(): AgentTool<typeof parameters>;
 declare const parameters: z.ZodObject<{
-    url: z.ZodString;
+    url: z.ZodOptional<z.ZodString>;
+    urls: z.ZodOptional<z.ZodArray<z.ZodString>>;
     max_length: z.ZodOptional<z.ZodNumber>;
+    format: z.ZodOptional<z.ZodEnum<{
+        text: "text";
+        html: "html";
+        markdown: "markdown";
+    }>>;
+    prefer_llms_txt: z.ZodOptional<z.ZodBoolean>;
 }, z.core.$strip>;
 export {};
 //# sourceMappingURL=web-fetch.d.ts.map

package/dist/tools/web-fetch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"web-fetch.d.ts","sourceRoot":"","sources":["../../src/tools/web-fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAErD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CA0CvD;AAmGD,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAYpD;AAED,wBAAgB,kBAAkB,IAAI,SAAS,CAAC,OAAO,UAAU,CAAC,CA0DjE;AAED,QAAA,MAAM,UAAU;;;iBAGd,CAAC"}
+{"version":3,"file":"web-fetch.d.ts","sourceRoot":"","sources":["../../src/tools/web-fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,SAAS,EAAe,MAAM,qBAAqB,CAAC;AAIlE;;;GAGG;AACH,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CA0CvD;AAmGD,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAYpD;AAqBD,KAAK,iBAAiB,GAAG,MAAM,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,CAAC;AAEvE,UAAU,aAAa;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,iBAAiB,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAuOD,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,aAAa,EAAE,CAuCnF;AAuED,wBAAgB,kBAAkB,IAAI,SAAS,CAAC,OAAO,UAAU,CAAC,CAkCjE;AA0BD,QAAA,MAAM,UAAU;;;;;;;;;;iBAoBZ,CAAC"}