@intentic/webq 1.317.0

package/src/lib/dom.ts

@@ -0,0 +1,107 @@
+// parse5's tree plus the traversal helpers every stage (pruning, markdown, links) needs; parse5 over jsdom since webq
+// ships baked into the sandbox image, and every dependency is pull size. Scripts never execute: this tree is data.
+import { type DefaultTreeAdapterMap, parse } from "parse5";
+
+export type Document = DefaultTreeAdapterMap["document"];
+export type Node = DefaultTreeAdapterMap["node"];
+export type Element = DefaultTreeAdapterMap["element"];
+export type TextNode = DefaultTreeAdapterMap["textNode"];
+
+export const isElement = (node: Node): node is Element => "tagName" in node;
+export const isText = (node: Node): node is TextNode => node.nodeName === "#text";
+
+export const parseHtml = (html: string): Document => parse(html);
+
+export const attr = (el: Element, name: string): string | undefined => el.attrs.find((a) => a.name === name)?.value;
+
+// Tags whose text is never page content; a <script>'s body is code, skipped by every text walk.
+const NON_CONTENT = new Set(["script", "style", "noscript", "template", "svg", "canvas"]);
+
+export const childElements = (el: Element): Element[] => el.childNodes.filter(isElement);
+
+/** Depth-first visible text of a subtree, single-space normalized. */
+export const textOf = (node: Node): string => collectText(node).join(" ").replaceAll(/\s+/g, " ").trim();
+
+/** Raw text of a subtree with whitespace kept, what a <pre> means by its contents. */
+export const rawTextOf = (node: Node): string => collectText(node).join("");
+
+const collectText = (node: Node): string[] => {
+    if (isText(node)) {
+        return [node.value];
+    }
+    if (!isElement(node) || NON_CONTENT.has(node.tagName)) {
+        return [];
+    }
+    return node.childNodes.flatMap((child) => collectText(child));
+};
+
+/** Every element with the given tag under (and including) the root, document order. */
+export const elementsByTag = (root: Node, tag: string): Element[] => {
+    const found: Element[] = [];
+    walk(root, (el) => {
+        if (el.tagName === tag) {
+            found.push(el);
+        }
+    });
+    return found;
+};
+
+export const walk = (node: Node, visit: (el: Element) => void): void => {
+    if (isElement(node)) {
+        visit(node);
+    }
+    if (isElement(node) || node.nodeName === "#document" || node.nodeName === "#document-fragment") {
+        for (const child of (node as { childNodes: Node[] }).childNodes) {
+            walk(child, visit);
+        }
+    }
+};
+
+// Package-private; bodyOf and metaOf are the shapes every consumer actually wants.
+const findFirst = (root: Node, match: (el: Element) => boolean): Element | undefined => {
+    if (isElement(root) && match(root)) {
+        return root;
+    }
+    if (!("childNodes" in root)) {
+        return undefined;
+    }
+    for (const child of (root as { childNodes: Node[] }).childNodes) {
+        const hit = findFirst(child, match);
+        if (hit !== undefined) {
+            return hit;
+        }
+    }
+    return undefined;
+};
+
+export const bodyOf = (doc: Document): Element | undefined => findFirst(doc, (el) => el.tagName === "body");
+
+/** Detach an element from its parent — parse5 trees are plain objects, so removal is list surgery. */
+export const remove = (el: Element): void => {
+    const parent = el.parentNode;
+    if (parent === null || parent === undefined) {
+        return;
+    }
+    const index = parent.childNodes.indexOf(el);
+    if (index >= 0) {
+        parent.childNodes.splice(index, 1);
+    }
+};
+
+/** Page metadata every capsule and index line leads with. */
+export interface PageMeta {
+    readonly title: string;
+    readonly description: string | undefined;
+    readonly lang: string | undefined;
+}
+
+export const metaOf = (doc: Document): PageMeta => {
+    const title = textOf(findFirst(doc, (el) => el.tagName === "title") ?? ({ nodeName: "#text", value: "" } as Node));
+    const description = findFirst(doc, (el) => el.tagName === "meta" && attr(el, "name") === "description");
+    const html = findFirst(doc, (el) => el.tagName === "html");
+    return {
+        title,
+        description: description === undefined ? undefined : attr(description, "content"),
+        lang: html === undefined ? undefined : attr(html, "lang"),
+    };
+};

package/src/lib/env.ts

@@ -0,0 +1,6 @@
+/* The one directory webq owns beyond the shared agent-CLI layout (@intentic/agent-cli/env). */
+import { join } from "node:path";
+import { toolHome } from "@intentic/agent-cli/env";
+
+// One store for every render mode: a static fetch and a browser fetch of the same URL are one network trip.
+export const cacheDir = (): string => join(toolHome("webq"), "cache");

package/src/lib/flags.ts

@@ -0,0 +1,41 @@
+/* Flag parsers and the flag groups both verbs share — declared once so `webq fetch` and `webq crawl`. */
+import { countParser } from "@intentic/agent-cli/flags";
+import type { BrowserMode } from "./page.js";
+
+const browserModeParser = (raw: string): BrowserMode => {
+    if (raw === "auto" || raw === "never" || raw === "force") {
+        return raw;
+    }
+    throw new Error(`--browser takes auto|never|force, got "${raw}"`);
+};
+
+/** A URL an agent typed: a bare host is meant as https. */
+export const urlParser = (raw: string): string => {
+    const trimmed = raw.trim();
+    return /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;
+};
+
+// The options shared verbatim by fetch and crawl.
+export const sharedFlagParameters = {
+    raw: { kind: "boolean", default: false, brief: "Whole page, skip fit-pruning of chrome" },
+    query: { kind: "parsed", parse: String, optional: true, brief: "Keep only blocks relevant to this (BM25); crawl: also steer link order" },
+    browser: { kind: "parsed", parse: browserModeParser, default: "auto", brief: "JS rendering: auto|never|force" },
+    out: { kind: "parsed", parse: String, optional: true, brief: "Directory for the markdown files" },
+    fresh: { kind: "boolean", default: false, brief: "Bypass the cache for this run" },
+    maxAge: { kind: "parsed", parse: countParser, optional: true, brief: "Cache freshness window in seconds (default 900)" },
+    timeout: { kind: "parsed", parse: countParser, default: "20", brief: "Per-page deadline in seconds" },
+    threshold: { kind: "parsed", parse: countParser, optional: true, brief: "Fit-pruning score floor (default 0.48; lower keeps more)" },
+    json: { kind: "boolean", default: false, brief: "Machine-readable result on stdout" },
+} as const;
+
+export interface SharedFlags {
+    readonly raw: boolean;
+    readonly query?: string;
+    readonly browser: BrowserMode;
+    readonly out?: string;
+    readonly fresh: boolean;
+    readonly maxAge?: number;
+    readonly timeout: number;
+    readonly threshold?: number;
+    readonly json: boolean;
+}

package/src/lib/http.ts

@@ -0,0 +1,101 @@
+/* The static fetch path — where every page starts, and where most docs pages end. */
+
+export interface HttpFetchOptions {
+    readonly timeoutMs?: number;
+    readonly maxBytes?: number;
+    readonly userAgent?: string;
+}
+
+export interface HttpPage {
+    readonly url: string;
+    /** Where redirects landed — the base every relative link resolves against. */
+    readonly finalUrl: string;
+    readonly status: number;
+    readonly contentType: string;
+    readonly body: string;
+    /** True when the byte cap cut the body short — the capsule must say so. */
+    readonly truncated: boolean;
+}
+
+// A current-Chrome UA with webq named at the end: enough for the CDNs that 403 anything without a browser
+// token, honest to anyone who reads their logs. The browser fallback presents real Chromium anyway.
+export const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 webq";
+
+const DEFAULT_TIMEOUT_MS = 20_000;
+const DEFAULT_MAX_BYTES = 10 * 1024 * 1024;
+
+export const httpFetch = async (url: string, options: HttpFetchOptions = {}): Promise<HttpPage> => {
+    const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
+    const response = await fetch(url, {
+        redirect: "follow",
+        signal: AbortSignal.timeout(timeoutMs),
+        headers: {
+            "user-agent": options.userAgent ?? USER_AGENT,
+            accept: "text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5",
+            "accept-language": "en-US,en;q=0.9",
+        },
+    });
+    const contentType = response.headers.get("content-type") ?? "";
+    const { bytes, truncated } = await readCapped(response, maxBytes, timeoutMs);
+    return {
+        url,
+        finalUrl: response.url === "" ? url : response.url,
+        status: response.status,
+        contentType,
+        body: decode(bytes, contentType),
+        truncated,
+    };
+};
+
+const readCapped = async (response: Response, maxBytes: number, timeoutMs: number): Promise<{ bytes: Uint8Array; truncated: boolean }> => {
+    if (response.body === null) {
+        return { bytes: new Uint8Array(0), truncated: false };
+    }
+    const deadline = Date.now() + timeoutMs;
+    const chunks: Uint8Array[] = [];
+    let total = 0;
+    const reader = response.body.getReader();
+    for (;;) {
+        if (Date.now() > deadline) {
+            await reader.cancel();
+            return { bytes: concat(chunks, total), truncated: true };
+        }
+        const { done, value } = await reader.read();
+        if (done) {
+            return { bytes: concat(chunks, total), truncated: false };
+        }
+        chunks.push(value);
+        total += value.byteLength;
+        if (total >= maxBytes) {
+            await reader.cancel();
+            return { bytes: concat(chunks, total).slice(0, maxBytes), truncated: true };
+        }
+    }
+};
+
+const concat = (chunks: Uint8Array[], total: number): Uint8Array => {
+    const merged = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        merged.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return merged;
+};
+
+const decode = (bytes: Uint8Array, contentType: string): string => {
+    const headerCharset = /charset=["']?([\w-]+)/i.exec(contentType)?.[1];
+    const sniffed = headerCharset ?? sniffMetaCharset(bytes);
+    try {
+        return new TextDecoder(sniffed ?? "utf8", { fatal: false }).decode(bytes);
+    } catch {
+        return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+    }
+};
+
+// The <meta charset> lives in the first kilobytes by spec; sniff it from an ASCII-safe decode of the head.
+const sniffMetaCharset = (bytes: Uint8Array): string | undefined => {
+    const head = new TextDecoder("latin1").decode(bytes.slice(0, 4096));
+    return (/<meta[^>]+charset=["']?([\w-]+)/i.exec(head) ?? /<\?xml[^>]+encoding=["']([\w-]+)/i.exec(head))?.[1];
+};

package/src/lib/links.test.ts

@@ -0,0 +1,37 @@
+import { parseHtml } from "./dom.js";
+import { extractLinks, normalizeUrl, sameOrigin } from "./links.js";
+
+describe("extractLinks", () => {
+    it("resolves, dedupes and strips fragments, keeping the best anchor text", () => {
+        const doc = parseHtml(
+            `<html><body>
+                <a href="/a">Alpha</a>
+                <a href="/a#section"></a>
+                <a href="https://other.example/x">Away</a>
+                <a href="mailto:a@b.c">mail</a>
+                <a href="javascript:void(0)">js</a>
+            </body></html>`,
+        );
+        const links = extractLinks(doc, "https://ex.com/docs/");
+        expect(links).toEqual([
+            { url: "https://ex.com/a", text: "Alpha" },
+            { url: "https://other.example/x", text: "Away" },
+        ]);
+    });
+});
+
+describe("normalizeUrl", () => {
+    it("drops fragments and non-http schemes", () => {
+        expect(normalizeUrl("https://ex.com/a#b")).toBe("https://ex.com/a");
+        expect(normalizeUrl("ftp://ex.com/a")).toBeUndefined();
+        expect(normalizeUrl("not a url")).toBeUndefined();
+    });
+});
+
+describe("sameOrigin", () => {
+    it("compares scheme, host and port", () => {
+        expect(sameOrigin("https://ex.com/a", "https://ex.com/b")).toBe(true);
+        expect(sameOrigin("https://ex.com/a", "https://sub.ex.com/a")).toBe(false);
+        expect(sameOrigin("https://ex.com/a", "http://ex.com/a")).toBe(false);
+    });
+});

package/src/lib/links.ts

@@ -0,0 +1,47 @@
+/* Link harvesting for the crawler, off the ORIGINAL tree — pruning strips navigation, and navigation is exactly where a site's structure lives. */
+import { attr, elementsByTag, type Node, textOf } from "./dom.js";
+
+export interface Link {
+    readonly url: string;
+    readonly text: string;
+}
+
+export const extractLinks = (root: Node, baseUrl: string): Link[] => {
+    const seen = new Map<string, Link>();
+    for (const anchor of elementsByTag(root, "a")) {
+        const url = normalizeUrl(attr(anchor, "href"), baseUrl);
+        if (url === undefined) {
+            continue;
+        }
+        const existing = seen.get(url);
+        const text = textOf(anchor).slice(0, 200);
+        if (existing === undefined || (existing.text === "" && text !== "")) {
+            seen.set(url, { url, text });
+        }
+    }
+    return [...seen.values()];
+};
+
+export const normalizeUrl = (raw: string | undefined, base?: string): string | undefined => {
+    if (raw === undefined || raw.trim() === "") {
+        return undefined;
+    }
+    try {
+        const url = base === undefined ? new URL(raw.trim()) : new URL(raw.trim(), base);
+        if (url.protocol !== "http:" && url.protocol !== "https:") {
+            return undefined;
+        }
+        url.hash = "";
+        return url.href;
+    } catch {
+        return undefined;
+    }
+};
+
+export const sameOrigin = (a: string, b: string): boolean => {
+    try {
+        return new URL(a).origin === new URL(b).origin;
+    } catch {
+        return false;
+    }
+};

package/src/lib/markdown.test.ts

@@ -0,0 +1,76 @@
+import { bodyOf, parseHtml } from "./dom.js";
+import { renderMarkdown } from "./markdown.js";
+
+const md = (html: string, baseUrl?: string): string => {
+    const body = bodyOf(parseHtml(`<html><body>${html}</body></html>`));
+    if (body === undefined) {
+        throw new Error("fixture has no body");
+    }
+    return renderMarkdown(body, baseUrl === undefined ? {} : { baseUrl });
+};
+
+describe("renderMarkdown", () => {
+    it("renders headings, paragraphs and inline marks", () => {
+        expect(md("<h2>Title</h2><p>Plain <strong>bold</strong> <em>italic</em> <code>code()</code>.</p>")).toBe(
+            "## Title\n\nPlain **bold** *italic* `code()`.\n",
+        );
+    });
+
+    it("pools loose container text into its own paragraph (flow)", () => {
+        expect(md("<div>loose text<p>inner</p></div>")).toBe("loose text\n\ninner\n");
+    });
+
+    it("renders nested lists with indentation and honors ol start", () => {
+        expect(md("<ul><li>one</li><li>two<ul><li>two-a</li></ul></li></ul><ol start='3'><li>three</li></ol>")).toBe(
+            "- one\n- two\n    - two-a\n\n3. three\n",
+        );
+    });
+
+    it("fences code with its language and outsizes inner backtick runs", () => {
+        expect(md("<pre><code class='language-ts'>const x = `hi`;</code></pre>")).toBe("```ts\nconst x = `hi`;\n```\n");
+        expect(md("<pre><code>a ``` b</code></pre>")).toBe("````\na ``` b\n````\n");
+    });
+
+    it("breaks lines at each line element of an editor-rendered code block, keeping blank lines", () => {
+        const lines = "<div class='cm-line'>const a = 1;</div><div class='cm-line'><br></div><div class='cm-line'>f(a);</div>";
+        expect(md(`<pre><code>${lines}</code></pre>`)).toBe("```\nconst a = 1;\n\nf(a);\n```\n");
+        // Pretty-printed markup: the newline between line elements is not a second line break.
+        expect(md("<pre><code><div>a</div>\n<div>b</div></code></pre>")).toBe("```\na\nb\n```\n");
+        // Highlighter spans with real newlines stay as they are.
+        expect(md("<pre><code><span class='line'>a</span>\n<span class='line'>b</span></code></pre>")).toBe("```\na\nb\n```\n");
+    });
+
+    it("renders GFM tables, escaping pipes and taking th as header", () => {
+        expect(md("<table><thead><tr><th>A</th><th>B|C</th></tr></thead><tbody><tr><td>1</td><td>2</td></tr></tbody></table>")).toBe(
+            "| A | B\\|C |\n| --- | --- |\n| 1 | 2 |\n",
+        );
+    });
+
+    it("absolutizes links against the page URL and unwraps unlinkable ones", () => {
+        expect(md("<p><a href='/x'>rel</a> <a href='javascript:alert(1)'>js</a> <a href='mailto:a@b.c'>mail</a></p>", "https://ex.com/docs/")).toBe(
+            "[rel](https://ex.com/x) js mail\n",
+        );
+    });
+
+    it("absolutizes images and drops data: blobs to their alt text", () => {
+        expect(md("<p><img src='pic.png' alt='Pic'> <img src='data:image/png;base64,x' alt='blob'></p>", "https://ex.com/d/")).toBe(
+            "![Pic](https://ex.com/d/pic.png) blob\n",
+        );
+    });
+
+    it("prefixes every blockquote line, including inner blank ones", () => {
+        expect(md("<blockquote><p>one</p><p>two</p></blockquote>")).toBe("> one\n>\n> two\n");
+    });
+
+    it("keeps br as a line break inside a paragraph", () => {
+        expect(md("<p>first<br>second</p>")).toBe("first\nsecond\n");
+    });
+
+    it("collapses whitespace runs and decodes entities", () => {
+        expect(md("<p>a\n\t   b &amp; c</p>")).toBe("a b & c\n");
+    });
+
+    it("skips script, style and their text entirely", () => {
+        expect(md("<p>keep</p><script>drop()</script><style>.x{}</style>")).toBe("keep\n");
+    });
+});