@intentic/webq 1.317.0

package/src/lib/prune.test.ts

@@ -0,0 +1,78 @@
+import { bodyOf, type Element, parseHtml, textOf } from "./dom.js";
+import { renderMarkdown } from "./markdown.js";
+import { pruneTree } from "./prune.js";
+
+const ARTICLE = `<main><article>
+<h1>Real article</h1>
+<p>This is the actual content of the page, a long paragraph carrying the substance of the article, dense readable text with many words and few links, exactly what a reader came for.</p>
+<p>Second paragraph, also substantial, explaining details about the topic in complete sentences.</p>
+</article></main>`;
+
+const CHROME = `<nav><a href='/'>Home</a><a href='/about'>About</a><a href='/pricing'>Pricing</a></nav>
+<div class='sidebar'><ul><li><a href='/a'>A</a></li><li><a href='/b'>B</a></li><li><a href='/c'>C</a></li></ul></div>
+<footer><p>Copyright · <a href='/tos'>Terms</a></p></footer>
+<div class='social-share'><a href='https://x.com/share'>Share on X</a></div>`;
+
+const bodyFor = (html: string): Element => {
+    const body = bodyOf(parseHtml(`<html><body>${html}</body></html>`));
+    if (body === undefined) {
+        throw new Error("fixture has no body");
+    }
+    return body;
+};
+
+describe("pruneTree", () => {
+    it("keeps the article and drops navigation, sidebar, footer and share chrome", () => {
+        const body = bodyFor(CHROME + ARTICLE);
+        pruneTree(body);
+        const markdown = renderMarkdown(body, {});
+        expect(markdown).toContain("# Real article");
+        expect(markdown).toContain("Second paragraph");
+        expect(markdown).not.toContain("Pricing");
+        expect(markdown).not.toContain("Copyright");
+        expect(markdown).not.toContain("Share on X");
+    });
+
+    it("reports the share of text mass it removed", () => {
+        const body = bodyFor(CHROME + ARTICLE);
+        const share = pruneTree(body);
+        expect(share).toBeGreaterThan(0.1);
+        expect(share).toBeLessThan(1);
+    });
+
+    it("keeps link-heavy chrome when the threshold is lowered to zero", () => {
+        const body = bodyFor(`<div class='sidebar'><ul><li><a href='/a'>A</a></li></ul></div>${ARTICLE}`);
+        pruneTree(body, { threshold: 0 });
+        expect(textOf(body)).toContain("A");
+        expect(textOf(body)).toContain("Real article");
+    });
+
+    it("always strips excluded tags, whatever the threshold", () => {
+        const body = bodyFor(`<script>evil()</script><noscript>enable js</noscript>${ARTICLE}`);
+        pruneTree(body, { threshold: 0 });
+        expect(textOf(body)).not.toContain("enable js");
+    });
+
+    it("keeps every token of a syntax-highlighted code block", () => {
+        const token = (cls: string, text: string): string => `<span class="token ${cls}">${text}</span>`;
+        const line = [
+            token("keyword", "import"),
+            " ",
+            token("punctuation", "{"),
+            " ",
+            token("plain", "useState"),
+            " ",
+            token("punctuation", "}"),
+            " ",
+            token("keyword", "from"),
+            " ",
+            token("string", "'react'"),
+            token("punctuation", ";"),
+        ].join("");
+        const body = bodyFor(
+            `${CHROME}<main><article><h1>Hooks</h1><p>Import the hook before you call it at the top level of your component, like this example shows in full.</p><div class="code-block"><pre><code>${line}</code></pre></div></article></main>`,
+        );
+        pruneTree(body);
+        expect(renderMarkdown(body, {})).toContain("import { useState } from 'react';");
+    });
+});

package/src/lib/prune.ts

@@ -0,0 +1,133 @@
+// Fit-content pruning: scores every element on text/link density, tag kind, class/id smell and text mass; drops
+// subtrees under the threshold. Ports crawl4ai's PruningContentFilter (Apache-2.0,
+// https://github.com/unclecode/crawl4ai), same weights and threshold. One post-order pass memoizes lengths instead of
+// re-serializing each node.
+import { attr, childElements, type Element, isElement, isText, remove, textOf } from "./dom.js";
+
+// Never content; upstream's excluded_tags plus the invisible tags this parser still keeps.
+const EXCLUDED = new Set(["nav", "footer", "header", "aside", "script", "style", "form", "iframe", "noscript", "template", "svg", "canvas"]);
+
+// Class or id naming page chrome; upstream's negative_patterns, verbatim.
+const NEGATIVE = /nav|footer|header|sidebar|ads|comment|promo|advert|social|share/i;
+
+const TAG_WEIGHTS: Record<string, number> = {
+    div: 0.5,
+    p: 1.0,
+    article: 1.5,
+    section: 1.0,
+    span: 0.3,
+    li: 0.5,
+    ul: 0.5,
+    ol: 0.5,
+    h1: 1.2,
+    h2: 1.1,
+    h3: 1.0,
+    h4: 0.9,
+    h5: 0.8,
+    h6: 0.7,
+};
+
+const WEIGHTS = { textDensity: 0.4, linkDensity: 0.2, tagWeight: 0.2, classIdWeight: 0.1, textLength: 0.1 };
+
+export interface PruneOptions {
+    /** Composite-score floor an element must clear to survive. Upstream default. */
+    readonly threshold?: number;
+    /** Fewer words than this in a subtree is an automatic drop (off by default, like upstream). */
+    readonly minWords?: number;
+}
+
+interface Mass {
+    readonly textLen: number;
+    readonly markupLen: number;
+    readonly linkTextLen: number;
+}
+
+/** Prunes in place under `body`; returns the fraction of text mass removed, for the capsule's honesty line. */
+export const pruneTree = (body: Element, options: PruneOptions = {}): number => {
+    const threshold = options.threshold ?? 0.48;
+    const before = textOf(body).length;
+    stripExcluded(body);
+    const masses = new Map<Element, Mass>();
+    weigh(body, masses);
+    for (const child of childElements(body)) {
+        pruneNode(child, threshold, options.minWords, masses);
+    }
+    const after = textOf(body).length;
+    return before === 0 ? 0 : (before - after) / before;
+};
+
+const stripExcluded = (el: Element): void => {
+    for (const child of childElements(el)) {
+        if (EXCLUDED.has(child.tagName)) {
+            remove(child);
+        } else {
+            stripExcluded(child);
+        }
+    }
+};
+
+// One post-order pass computes text mass, approximate markup length, and direct <a> text length per element: the
+// score's three inputs.
+const weigh = (el: Element, memo: Map<Element, Mass>): Mass => {
+    let textLen = 0;
+    let markupLen = 0;
+    let linkTextLen = 0;
+    for (const child of el.childNodes) {
+        if (isText(child)) {
+            const trimmed = child.value.trim();
+            textLen += trimmed.length;
+            markupLen += trimmed.length;
+        } else if (isElement(child)) {
+            const mass = weigh(child, memo);
+            textLen += mass.textLen;
+            markupLen += mass.markupLen;
+            if (child.tagName === "a") {
+                linkTextLen += textOf(child).length;
+            }
+        }
+    }
+    // A code block is all content however many highlighter spans carry it: its inner markup would read as chrome and
+    // sink every wrapper around it, so only the <pre>'s own tags count against it.
+    if (el.tagName === "pre") {
+        markupLen = textLen;
+    }
+    // <tag attrs></tag> overhead: 2 brackets + name, twice, plus each attribute's name="value".
+    markupLen += 2 * (el.tagName.length + 2) + el.attrs.reduce((sum, a) => sum + a.name.length + a.value.length + 4, 0);
+    const mass = { textLen, markupLen, linkTextLen };
+    memo.set(el, mass);
+    return mass;
+};
+
+const pruneNode = (el: Element, threshold: number, minWords: number | undefined, masses: Map<Element, Mass>): void => {
+    if (compositeScore(el, minWords, masses) < threshold) {
+        remove(el);
+        return;
+    }
+    // Inside a code block every token span scores as chrome; dropping one silently rewrites the code.
+    if (el.tagName === "pre") {
+        return;
+    }
+    for (const child of childElements(el)) {
+        pruneNode(child, threshold, minWords, masses);
+    }
+};
+
+const compositeScore = (el: Element, minWords: number | undefined, masses: Map<Element, Mass>): number => {
+    const mass = masses.get(el) ?? { textLen: 0, markupLen: 0, linkTextLen: 0 };
+    if (minWords !== undefined && textOf(el).split(" ").filter(Boolean).length < minWords) {
+        return -1;
+    }
+    const textDensity = mass.markupLen > 0 ? mass.textLen / mass.markupLen : 0;
+    const linkDensity = 1 - (mass.textLen > 0 ? mass.linkTextLen / mass.textLen : 0);
+    const tagWeight = TAG_WEIGHTS[el.tagName] ?? 0.5;
+    const classId = attr(el, "class") ?? "";
+    const id = attr(el, "id") ?? "";
+    const classIdScore = (NEGATIVE.test(classId) ? -0.5 : 0) + (NEGATIVE.test(id) ? -0.5 : 0);
+    const score =
+        WEIGHTS.textDensity * textDensity +
+        WEIGHTS.linkDensity * linkDensity +
+        WEIGHTS.tagWeight * tagWeight +
+        WEIGHTS.classIdWeight * Math.max(0, classIdScore) +
+        WEIGHTS.textLength * Math.log(mass.textLen + 1);
+    return score / (WEIGHTS.textDensity + WEIGHTS.linkDensity + WEIGHTS.tagWeight + WEIGHTS.classIdWeight + WEIGHTS.textLength);
+};

package/src/lib/robots.test.ts

@@ -0,0 +1,52 @@
+import { isAllowed, parseRobots } from "./robots.js";
+
+describe("parseRobots", () => {
+    it("takes the wildcard group and reads sitemaps and crawl-delay", () => {
+        const rules = parseRobots(
+            [
+                "User-agent: googlebot",
+                "Disallow: /google-only",
+                "",
+                "User-agent: *",
+                "Disallow: /private",
+                "Crawl-delay: 2",
+                "",
+                "Sitemap: https://ex.com/sitemap.xml",
+            ].join("\n"),
+        );
+        expect(rules.disallows).toEqual(["/private"]);
+        expect(rules.crawlDelayS).toBe(2);
+        expect(rules.sitemaps).toEqual(["https://ex.com/sitemap.xml"]);
+    });
+
+    it("prefers a group naming webq over the wildcard", () => {
+        const rules = parseRobots(["User-agent: *", "Disallow: /", "", "User-agent: webq", "Disallow: /only-this"].join("\n"));
+        expect(rules.disallows).toEqual(["/only-this"]);
+    });
+
+    it("lets several user-agent lines share one group", () => {
+        const rules = parseRobots(["User-agent: a", "User-agent: *", "Disallow: /x"].join("\n"));
+        expect(rules.disallows).toEqual(["/x"]);
+    });
+});
+
+describe("isAllowed", () => {
+    it("applies longest-match-wins with allow beating disallow on ties", () => {
+        const rules = parseRobots(["User-agent: *", "Disallow: /docs", "Allow: /docs/public"].join("\n"));
+        expect(isAllowed(rules, "/docs/private")).toBe(false);
+        expect(isAllowed(rules, "/docs/public/page")).toBe(true);
+        expect(isAllowed(rules, "/other")).toBe(true);
+    });
+
+    it("supports * wildcards and $ end anchors", () => {
+        const rules = parseRobots(["User-agent: *", "Disallow: /*.pdf$", "Disallow: /tmp/*"].join("\n"));
+        expect(isAllowed(rules, "/report.pdf")).toBe(false);
+        expect(isAllowed(rules, "/report.pdf.html")).toBe(true);
+        expect(isAllowed(rules, "/tmp/x/y")).toBe(false);
+    });
+
+    it("treats an empty disallow as allow-all", () => {
+        const rules = parseRobots(["User-agent: *", "Disallow:"].join("\n"));
+        expect(isAllowed(rules, "/anything")).toBe(true);
+    });
+});

package/src/lib/robots.ts

@@ -0,0 +1,102 @@
+/* robots.txt, parsed the way Google documents it: rules grouped by user-agent (several agent lines can share one group). */
+
+export interface RobotsRules {
+    readonly allows: string[];
+    readonly disallows: string[];
+    readonly crawlDelayS: number | undefined;
+    readonly sitemaps: string[];
+}
+
+export const EVERYTHING_ALLOWED: RobotsRules = { allows: [], disallows: [], crawlDelayS: undefined, sitemaps: [] };
+
+const OUR_NAME = "webq";
+
+export const parseRobots = (text: string): RobotsRules => {
+    interface Group {
+        agents: string[];
+        allows: string[];
+        disallows: string[];
+        crawlDelayS: number | undefined;
+    }
+    const groups: Group[] = [];
+    const sitemaps: string[] = [];
+    let current: Group | undefined;
+    let openForAgents = false;
+    for (const rawLine of text.split("\n")) {
+        const line = (rawLine.split("#")[0] ?? "").trim();
+        const colon = line.indexOf(":");
+        if (colon < 0) {
+            continue;
+        }
+        const field = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (field === "sitemap") {
+            sitemaps.push(value);
+            continue;
+        }
+        if (field === "user-agent") {
+            if (!openForAgents || current === undefined) {
+                current = { agents: [], allows: [], disallows: [], crawlDelayS: undefined };
+                groups.push(current);
+                openForAgents = true;
+            }
+            current.agents.push(value.toLowerCase());
+            continue;
+        }
+        openForAgents = false;
+        if (current === undefined) {
+            continue;
+        }
+        if (field === "allow") {
+            current.allows.push(value);
+        } else if (field === "disallow") {
+            current.disallows.push(value);
+        } else if (field === "crawl-delay") {
+            const delay = Number(value);
+            current.crawlDelayS = Number.isFinite(delay) ? delay : undefined;
+        }
+    }
+    const named = groups.find((group) => group.agents.some((agent) => OUR_NAME.includes(agent) || agent.includes(OUR_NAME)));
+    const wildcard = groups.find((group) => group.agents.includes("*"));
+    const chosen = named ?? wildcard;
+    return {
+        allows: chosen?.allows ?? [],
+        disallows: chosen?.disallows ?? [],
+        crawlDelayS: chosen?.crawlDelayS,
+        sitemaps,
+    };
+};
+
+export const isAllowed = (rules: RobotsRules, path: string): boolean => {
+    const disallow = longestMatch(rules.disallows, path);
+    if (disallow === undefined) {
+        return true;
+    }
+    const allow = longestMatch(rules.allows, path);
+    // Ties go to allow, per the spec's "least restrictive" rule.
+    return allow !== undefined && allow >= disallow;
+};
+
+/** Length of the longest pattern in `patterns` matching `path`, or undefined when none match. */
+const longestMatch = (patterns: string[], path: string): number | undefined => {
+    let longest: number | undefined;
+    for (const pattern of patterns) {
+        if (pattern === "") {
+            continue;
+        }
+        if (matches(pattern, path) && (longest === undefined || pattern.length > longest)) {
+            longest = pattern.length;
+        }
+    }
+    return longest;
+};
+
+const matches = (pattern: string, path: string): boolean => {
+    const escaped = pattern.replaceAll(/[.+?^${}()|[\]\\]/g, "\\$&").replaceAll("*", ".*");
+    const anchored = escaped.endsWith("$") ? `^${escaped.slice(0, -2)}$` : `^${escaped}`;
+    try {
+        return new RegExp(anchored).test(path);
+    } catch {
+        return false;
+    }
+};

package/src/lib/sitemap.test.ts

@@ -0,0 +1,37 @@
+import { fetchSitemapUrls } from "./sitemap.js";
+
+const stub =
+    (map: Record<string, string>) =>
+    (url: string): Promise<{ body: string } | undefined> =>
+        Promise.resolve(map[url] === undefined ? undefined : { body: map[url] });
+
+describe("fetchSitemapUrls", () => {
+    it("reads a urlset, decoding entities and normalizing", async () => {
+        const seed = await fetchSitemapUrls(
+            "https://ex.com/start",
+            [],
+            stub({
+                "https://ex.com/sitemap.xml": `<urlset><url><loc> https://ex.com/a </loc></url><url><loc>https://ex.com/b?x=1&amp;y=2</loc></url></urlset>`,
+            }),
+        );
+        expect(seed.urls).toEqual(["https://ex.com/a", "https://ex.com/b?x=1&y=2"]);
+        expect(seed.capped).toBe(false);
+    });
+
+    it("follows one level of sitemap index and prefers robots-declared sitemaps", async () => {
+        const seed = await fetchSitemapUrls(
+            "https://ex.com/",
+            ["https://ex.com/custom.xml"],
+            stub({
+                "https://ex.com/custom.xml": `<sitemapindex><sitemap><loc>https://ex.com/part1.xml</loc></sitemap></sitemapindex>`,
+                "https://ex.com/part1.xml": `<urlset><url><loc>https://ex.com/deep</loc></url></urlset>`,
+            }),
+        );
+        expect(seed.urls).toEqual(["https://ex.com/deep"]);
+    });
+
+    it("returns empty when no sitemap answers", async () => {
+        const seed = await fetchSitemapUrls("https://ex.com/", [], stub({}));
+        expect(seed.urls).toEqual([]);
+    });
+});

package/src/lib/sitemap.ts

@@ -0,0 +1,74 @@
+/* Sitemap seeding: the URLs a site already published as its own map, so a crawl starts from the real table of contents instead of discovering it hop by hop. */
+import { gunzipSync } from "node:zlib";
+import { normalizeUrl } from "./links.js";
+
+const URL_CAP = 2_000;
+const CHILD_SITEMAP_CAP = 10;
+
+export interface SitemapSeed {
+    readonly urls: string[];
+    /** True when the cap cut discovery short — the crawl capsule reports it instead of implying totality. */
+    readonly capped: boolean;
+}
+
+export const fetchSitemapUrls = async (
+    startUrl: string,
+    robotsSitemaps: string[],
+    fetchText: (url: string) => Promise<{ body: string; bytes?: Uint8Array } | undefined>,
+): Promise<SitemapSeed> => {
+    const origin = new URL(startUrl).origin;
+    const roots = robotsSitemaps.length > 0 ? robotsSitemaps : [`${origin}/sitemap.xml`];
+    const urls = new Set<string>();
+    let capped = false;
+    let childBudget = CHILD_SITEMAP_CAP;
+    const queue = [...roots];
+    while (queue.length > 0 && urls.size < URL_CAP) {
+        const sitemapUrl = queue.shift();
+        if (sitemapUrl === undefined) {
+            break;
+        }
+        const fetched = await fetchText(sitemapUrl);
+        if (fetched === undefined) {
+            continue;
+        }
+        const xml = maybeGunzip(sitemapUrl, fetched);
+        const locs = [...xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/g)].map((m) => m[1] ?? "");
+        if (/<sitemapindex/i.test(xml)) {
+            for (const loc of locs) {
+                if (childBudget > 0) {
+                    queue.push(loc);
+                    childBudget -= 1;
+                } else {
+                    capped = true;
+                }
+            }
+            continue;
+        }
+        for (const loc of locs) {
+            const url = normalizeUrl(decodeXmlEntities(loc));
+            if (url === undefined) {
+                continue;
+            }
+            if (urls.size >= URL_CAP) {
+                capped = true;
+                break;
+            }
+            urls.add(url);
+        }
+    }
+    return { urls: [...urls], capped };
+};
+
+const maybeGunzip = (url: string, fetched: { body: string; bytes?: Uint8Array }): string => {
+    if (!url.endsWith(".gz") || fetched.bytes === undefined) {
+        return fetched.body;
+    }
+    try {
+        return gunzipSync(fetched.bytes).toString("utf8");
+    } catch {
+        return fetched.body;
+    }
+};
+
+const decodeXmlEntities = (text: string): string =>
+    text.replaceAll("&amp;", "&").replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("&quot;", '"').replaceAll("&#39;", "'");