@intentic/webq 1.317.0

package/dist/lib/prune.js

@@ -0,0 +1,100 @@
+import { attr, childElements, isElement, isText, remove, textOf } from "./dom.js";
+const EXCLUDED = new Set(["nav", "footer", "header", "aside", "script", "style", "form", "iframe", "noscript", "template", "svg", "canvas"]);
+const NEGATIVE = /nav|footer|header|sidebar|ads|comment|promo|advert|social|share/i;
+const TAG_WEIGHTS = {
+    div: 0.5,
+    p: 1.0,
+    article: 1.5,
+    section: 1.0,
+    span: 0.3,
+    li: 0.5,
+    ul: 0.5,
+    ol: 0.5,
+    h1: 1.2,
+    h2: 1.1,
+    h3: 1.0,
+    h4: 0.9,
+    h5: 0.8,
+    h6: 0.7,
+};
+const WEIGHTS = { textDensity: 0.4, linkDensity: 0.2, tagWeight: 0.2, classIdWeight: 0.1, textLength: 0.1 };
+export const pruneTree = (body, options = {}) => {
+    const threshold = options.threshold ?? 0.48;
+    const before = textOf(body).length;
+    stripExcluded(body);
+    const masses = new Map();
+    weigh(body, masses);
+    for (const child of childElements(body)) {
+        pruneNode(child, threshold, options.minWords, masses);
+    }
+    const after = textOf(body).length;
+    return before === 0 ? 0 : (before - after) / before;
+};
+const stripExcluded = (el) => {
+    for (const child of childElements(el)) {
+        if (EXCLUDED.has(child.tagName)) {
+            remove(child);
+        }
+        else {
+            stripExcluded(child);
+        }
+    }
+};
+const weigh = (el, memo) => {
+    let textLen = 0;
+    let markupLen = 0;
+    let linkTextLen = 0;
+    for (const child of el.childNodes) {
+        if (isText(child)) {
+            const trimmed = child.value.trim();
+            textLen += trimmed.length;
+            markupLen += trimmed.length;
+        }
+        else if (isElement(child)) {
+            const mass = weigh(child, memo);
+            textLen += mass.textLen;
+            markupLen += mass.markupLen;
+            if (child.tagName === "a") {
+                linkTextLen += textOf(child).length;
+            }
+        }
+    }
+    if (el.tagName === "pre") {
+        markupLen = textLen;
+    }
+    markupLen += 2 * (el.tagName.length + 2) + el.attrs.reduce((sum, a) => sum + a.name.length + a.value.length + 4, 0);
+    const mass = { textLen, markupLen, linkTextLen };
+    memo.set(el, mass);
+    return mass;
+};
+const pruneNode = (el, threshold, minWords, masses) => {
+    if (compositeScore(el, minWords, masses) < threshold) {
+        remove(el);
+        return;
+    }
+    if (el.tagName === "pre") {
+        return;
+    }
+    for (const child of childElements(el)) {
+        pruneNode(child, threshold, minWords, masses);
+    }
+};
+const compositeScore = (el, minWords, masses) => {
+    const mass = masses.get(el) ?? { textLen: 0, markupLen: 0, linkTextLen: 0 };
+    if (minWords !== undefined && textOf(el).split(" ").filter(Boolean).length < minWords) {
+        return -1;
+    }
+    const textDensity = mass.markupLen > 0 ? mass.textLen / mass.markupLen : 0;
+    const linkDensity = 1 - (mass.textLen > 0 ? mass.linkTextLen / mass.textLen : 0);
+    const tagWeight = TAG_WEIGHTS[el.tagName] ?? 0.5;
+    const classId = attr(el, "class") ?? "";
+    const id = attr(el, "id") ?? "";
+    const classIdScore = (NEGATIVE.test(classId) ? -0.5 : 0) + (NEGATIVE.test(id) ? -0.5 : 0);
+    const score = WEIGHTS.textDensity * textDensity +
+        WEIGHTS.linkDensity * linkDensity +
+        WEIGHTS.tagWeight * tagWeight +
+        WEIGHTS.classIdWeight * Math.max(0, classIdScore) +
+        WEIGHTS.textLength * Math.log(mass.textLen + 1);
+    return score / (WEIGHTS.textDensity + WEIGHTS.linkDensity + WEIGHTS.tagWeight + WEIGHTS.classIdWeight + WEIGHTS.textLength);
+};
+//# sourceMappingURL=prune.js.map

package/dist/lib/prune.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prune.js","sourceRoot":"","sources":["../../src/lib/prune.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,IAAI,EAAE,aAAa,EAAgB,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC;AAGhG,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;AAG7I,MAAM,QAAQ,GAAG,kEAAkE,CAAC;AAEpF,MAAM,WAAW,GAA2B;IACxC,GAAG,EAAE,GAAG;IACR,CAAC,EAAE,GAAG;IACN,OAAO,EAAE,GAAG;IACZ,OAAO,EAAE,GAAG;IACZ,IAAI,EAAE,GAAG;IACT,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;CACV,CAAC;AAEF,MAAM,OAAO,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;AAgB5G,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAa,EAAE,OAAO,GAAiB,EAAE,EAAU,EAAE;IAC3E,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC;IAC5C,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACnC,aAAa,CAAC,IAAI,CAAC,CAAC;IACpB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAC;IACxC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACpB,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QACtC,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC1D,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC;AACxD,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,EAAW,EAAQ,EAAE;IACxC,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;QACpC,IAAI,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;aAAM,CAAC;YACJ,aAAa,CAAC,KAAK,CAAC,CAAC;QACzB,CAAC;IACL,CAAC;AACL,CAAC,CAAC;AAIF,MAAM,KAAK,GAAG,CAAC,EAAW,EAAE,IAAwB,EAAQ,EAAE;IAC1D,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,UAAU,EAAE,CAAC;QAChC,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;YACnC,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;YAC1B,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;QAChC,CAAC;aAAM,IAAI,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;YAChC,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;YACxB,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;YAC5B,IAAI,KAAK,CAAC,OAAO,KAAK,GAAG,EAAE,CAAC;gBACxB,WAAW,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;YACxC,CAAC;QACL,CAAC;IACL,CAAC;IAGD,IAAI,EAAE,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QACvB,SAAS,GAAG,OAAO,CAAC;IACxB,CAAC;IAED,SAAS,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IACpH,MAAM,IAAI,GAAG,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;IACjD,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IACnB,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,EAAW,EAAE,SAAiB,EAAE,QAA4B,EAAE,MAA0B,EAAQ,EAAE;IACjH,IAAI,cAAc,CAAC,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;QACnD,MAAM,CAAC,EAAE,CAAC,CAAC;QACX,OAAO;IACX,CAAC;IAED,IAAI,EAAE,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QACvB,OAAO;IACX,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,EAAE,CAAC,EAAE,CAAC;QACpC,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,EAAW,EAAE,QAA4B,EAAE,MAA0B,EAAU,EAAE;IACrG,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC;IAC5E,IAAI,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;QACpF,OAAO,CAAC,CAAC,CAAC;IACd,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACjF,MAAM,SAAS,GAAG,WAAW,CAAC,EAAE,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC;IACjD,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;IACxC,MAAM,EAAE,GAAG,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;IAChC,MAAM,YAAY,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,KAAK,GACP,OAAO,CAAC,WAAW,GAAG,WAAW;QACjC,OAAO,CAAC,WAAW,GAAG,WAAW;QACjC,OAAO,CAAC,SAAS,GAAG,SAAS;QAC7B,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,CAAC;QACjD,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;IACpD,OAAO,KAAK,GAAG,CAAC,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,GAAG,OAAO,CAAC,SAAS,GAAG,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;AAChI,CAAC,CAAC"}

package/dist/lib/robots.d.ts

@@ -0,0 +1,10 @@
+export interface RobotsRules {
+    readonly allows: string[];
+    readonly disallows: string[];
+    readonly crawlDelayS: number | undefined;
+    readonly sitemaps: string[];
+}
+export declare const EVERYTHING_ALLOWED: RobotsRules;
+export declare const parseRobots: (text: string) => RobotsRules;
+export declare const isAllowed: (rules: RobotsRules, path: string) => boolean;
+//# sourceMappingURL=robots.d.ts.map

package/dist/lib/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../../src/lib/robots.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC;IAC7B,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC;CAC/B;AAED,eAAO,MAAM,kBAAkB,EAAE,WAAiF,CAAC;AAInH,eAAO,MAAM,WAAW,SAAU,MAAM,KAAG,WAsD1C,CAAC;AAEF,eAAO,MAAM,SAAS,UAAW,WAAW,QAAQ,MAAM,KAAG,OAQ5D,CAAC"}

package/dist/lib/robots.js

@@ -0,0 +1,84 @@
+export const EVERYTHING_ALLOWED = { allows: [], disallows: [], crawlDelayS: undefined, sitemaps: [] };
+const OUR_NAME = "webq";
+export const parseRobots = (text) => {
+    const groups = [];
+    const sitemaps = [];
+    let current;
+    let openForAgents = false;
+    for (const rawLine of text.split("\n")) {
+        const line = (rawLine.split("#")[0] ?? "").trim();
+        const colon = line.indexOf(":");
+        if (colon < 0) {
+            continue;
+        }
+        const field = line.slice(0, colon).trim().toLowerCase();
+        const value = line.slice(colon + 1).trim();
+        if (field === "sitemap") {
+            sitemaps.push(value);
+            continue;
+        }
+        if (field === "user-agent") {
+            if (!openForAgents || current === undefined) {
+                current = { agents: [], allows: [], disallows: [], crawlDelayS: undefined };
+                groups.push(current);
+                openForAgents = true;
+            }
+            current.agents.push(value.toLowerCase());
+            continue;
+        }
+        openForAgents = false;
+        if (current === undefined) {
+            continue;
+        }
+        if (field === "allow") {
+            current.allows.push(value);
+        }
+        else if (field === "disallow") {
+            current.disallows.push(value);
+        }
+        else if (field === "crawl-delay") {
+            const delay = Number(value);
+            current.crawlDelayS = Number.isFinite(delay) ? delay : undefined;
+        }
+    }
+    const named = groups.find((group) => group.agents.some((agent) => OUR_NAME.includes(agent) || agent.includes(OUR_NAME)));
+    const wildcard = groups.find((group) => group.agents.includes("*"));
+    const chosen = named ?? wildcard;
+    return {
+        allows: chosen?.allows ?? [],
+        disallows: chosen?.disallows ?? [],
+        crawlDelayS: chosen?.crawlDelayS,
+        sitemaps,
+    };
+};
+export const isAllowed = (rules, path) => {
+    const disallow = longestMatch(rules.disallows, path);
+    if (disallow === undefined) {
+        return true;
+    }
+    const allow = longestMatch(rules.allows, path);
+    return allow !== undefined && allow >= disallow;
+};
+const longestMatch = (patterns, path) => {
+    let longest;
+    for (const pattern of patterns) {
+        if (pattern === "") {
+            continue;
+        }
+        if (matches(pattern, path) && (longest === undefined || pattern.length > longest)) {
+            longest = pattern.length;
+        }
+    }
+    return longest;
+};
+const matches = (pattern, path) => {
+    const escaped = pattern.replaceAll(/[.+?^${}()|[\]\\]/g, "\\$&").replaceAll("*", ".*");
+    const anchored = escaped.endsWith("$") ? `^${escaped.slice(0, -2)}$` : `^${escaped}`;
+    try {
+        return new RegExp(anchored).test(path);
+    }
+    catch {
+        return false;
+    }
+};
+//# sourceMappingURL=robots.js.map

package/dist/lib/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../../src/lib/robots.ts"],"names":[],"mappings":"AASA,MAAM,CAAC,MAAM,kBAAkB,GAAgB,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;AAEnH,MAAM,QAAQ,GAAG,MAAM,CAAC;AAExB,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,IAAY,EAAe,EAAE;IAOrD,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,OAA0B,CAAC;IAC/B,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACZ,SAAS;QACb,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,SAAS;QACb,CAAC;QACD,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YACzB,IAAI,CAAC,aAAa,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1C,OAAO,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC;gBAC5E,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,aAAa,GAAG,IAAI,CAAC;YACzB,CAAC;YACD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;YACzC,SAAS;QACb,CAAC;QACD,aAAa,GAAG,KAAK,CAAC;QACtB,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACxB,SAAS;QACb,CAAC;QACD,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YACpB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YAC9B,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,KAAK,KAAK,aAAa,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5B,OAAO,CAAC,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACrE,CAAC;IACL,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACzH,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,MAAM,MAAM,GAAG,KAAK,IAAI,QAAQ,CAAC;IACjC,OAAO;QACH,MAAM,EAAE,MAAM,EAAE,MAAM,IAAI,EAAE;QAC5B,SAAS,EAAE,MAAM,EAAE,SAAS,IAAI,EAAE;QAClC,WAAW,EAAE,MAAM,EAAE,WAAW;QAChC,QAAQ;KACX,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,KAAkB,EAAE,IAAY,EAAW,EAAE;IACnE,MAAM,QAAQ,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACrD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAE/C,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,IAAI,QAAQ,CAAC;AACpD,CAAC,CAAC;AAGF,MAAM,YAAY,GAAG,CAAC,QAAkB,EAAE,IAAY,EAAsB,EAAE;IAC1E,IAAI,OAA2B,CAAC;IAChC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC7B,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;YACjB,SAAS;QACb,CAAC;QACD,IAAI,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,OAAO,CAAC,EAAE,CAAC;YAChF,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,CAAC;IACL,CAAC;IACD,OAAO,OAAO,CAAC;AACnB,CAAC,CAAC;AAEF,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,IAAY,EAAW,EAAE;IACvD,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACvF,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;IACrF,IAAI,CAAC;QACD,OAAO,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC,CAAC"}

package/dist/lib/sitemap.d.ts

@@ -0,0 +1,9 @@
+export interface SitemapSeed {
+    readonly urls: string[];
+    readonly capped: boolean;
+}
+export declare const fetchSitemapUrls: (startUrl: string, robotsSitemaps: string[], fetchText: (url: string) => Promise<{
+    body: string;
+    bytes?: Uint8Array;
+} | undefined>) => Promise<SitemapSeed>;
+//# sourceMappingURL=sitemap.d.ts.map

package/dist/lib/sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../../src/lib/sitemap.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;IAExB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;CAC5B;AAED,eAAO,MAAM,gBAAgB,aACf,MAAM,kBACA,MAAM,EAAE,aACb,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,UAAU,CAAA;CAAE,GAAG,SAAS,CAAC,KACtF,OAAO,CAAC,WAAW,CA0CrB,CAAC"}

package/dist/lib/sitemap.js

@@ -0,0 +1,61 @@
+import { gunzipSync } from "node:zlib";
+import { normalizeUrl } from "./links.js";
+const URL_CAP = 2_000;
+const CHILD_SITEMAP_CAP = 10;
+export const fetchSitemapUrls = async (startUrl, robotsSitemaps, fetchText) => {
+    const origin = new URL(startUrl).origin;
+    const roots = robotsSitemaps.length > 0 ? robotsSitemaps : [`${origin}/sitemap.xml`];
+    const urls = new Set();
+    let capped = false;
+    let childBudget = CHILD_SITEMAP_CAP;
+    const queue = [...roots];
+    while (queue.length > 0 && urls.size < URL_CAP) {
+        const sitemapUrl = queue.shift();
+        if (sitemapUrl === undefined) {
+            break;
+        }
+        const fetched = await fetchText(sitemapUrl);
+        if (fetched === undefined) {
+            continue;
+        }
+        const xml = maybeGunzip(sitemapUrl, fetched);
+        const locs = [...xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/g)].map((m) => m[1] ?? "");
+        if (/<sitemapindex/i.test(xml)) {
+            for (const loc of locs) {
+                if (childBudget > 0) {
+                    queue.push(loc);
+                    childBudget -= 1;
+                }
+                else {
+                    capped = true;
+                }
+            }
+            continue;
+        }
+        for (const loc of locs) {
+            const url = normalizeUrl(decodeXmlEntities(loc));
+            if (url === undefined) {
+                continue;
+            }
+            if (urls.size >= URL_CAP) {
+                capped = true;
+                break;
+            }
+            urls.add(url);
+        }
+    }
+    return { urls: [...urls], capped };
+};
+const maybeGunzip = (url, fetched) => {
+    if (!url.endsWith(".gz") || fetched.bytes === undefined) {
+        return fetched.body;
+    }
+    try {
+        return gunzipSync(fetched.bytes).toString("utf8");
+    }
+    catch {
+        return fetched.body;
+    }
+};
+const decodeXmlEntities = (text) => text.replaceAll("&amp;", "&").replaceAll("&lt;", "<").replaceAll("&gt;", ">").replaceAll("&quot;", '"').replaceAll("&#39;", "'");
+//# sourceMappingURL=sitemap.js.map

package/dist/lib/sitemap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../../src/lib/sitemap.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AACvC,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,OAAO,GAAG,KAAK,CAAC;AACtB,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAQ7B,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACjC,QAAgB,EAChB,cAAwB,EACxB,SAAqF,EACjE,EAAE;IACtB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IACxC,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,CAAC;IACrF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,IAAI,WAAW,GAAG,iBAAiB,CAAC;IACpC,MAAM,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACzB,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,IAAI,GAAG,OAAO,EAAE,CAAC;QAC7C,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;QACjC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM;QACV,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACxB,SAAS;QACb,CAAC;QACD,MAAM,GAAG,GAAG,WAAW,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,6BAA6B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrF,IAAI,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACrB,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;oBAClB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAChB,WAAW,IAAI,CAAC,CAAC;gBACrB,CAAC;qBAAM,CAAC;oBACJ,MAAM,GAAG,IAAI,CAAC;gBAClB,CAAC;YACL,CAAC;YACD,SAAS;QACb,CAAC;QACD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,GAAG,GAAG,YAAY,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;YACjD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;gBACpB,SAAS;YACb,CAAC;YACD,IAAI,IAAI,CAAC,IAAI,IAAI,OAAO,EAAE,CAAC;gBACvB,MAAM,GAAG,IAAI,CAAC;gBACd,MAAM;YACV,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClB,CAAC;IACL,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;AACvC,CAAC,CAAC;AAEF,MAAM,WAAW,GAAG,CAAC,GAAW,EAAE,OAA6C,EAAU,EAAE;IACvF,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACtD,OAAO,OAAO,CAAC,IAAI,CAAC;IACxB,CAAC;IACD,IAAI,CAAC;QACD,OAAO,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,OAAO,CAAC,IAAI,CAAC;IACxB,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,IAAY,EAAU,EAAE,CAC/C,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC"}

package/dist/lib/version.d.ts

@@ -0,0 +1,2 @@
+export declare const version: string;
+//# sourceMappingURL=version.d.ts.map

package/dist/lib/version.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"version.d.ts","sourceRoot":"","sources":["../../src/lib/version.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA8F,CAAC"}

package/dist/lib/version.js

@@ -0,0 +1,3 @@
+import { createRequire } from "node:module";
+export const version = createRequire(import.meta.url)("../../package.json").version;
+//# sourceMappingURL=version.js.map

package/dist/lib/version.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"version.js","sourceRoot":"","sources":["../../src/lib/version.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAI5C,MAAM,CAAC,MAAM,OAAO,GAAY,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,oBAAoB,CAAyB,CAAC,OAAO,CAAC"}

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "@intentic/webq",
+  "version": "1.317.0",
+  "description": "webq, agent-native web fetch CLI: any URL as clean budgeted markdown, and bounded same-site crawls, over one shared cache",
+  "license": "MIT",
+  "type": "module",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/intentic/intentic.git",
+    "directory": "_sandbox/webq"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "bin": {
+    "webq": "./dist/cli.js"
+  },
+  "exports": {
+    "./markdown": {
+      "types": "./dist/lib/markdown.d.ts",
+      "import": {
+        "@intentic/src": "./src/lib/markdown.ts",
+        "default": "./dist/lib/markdown.js"
+      },
+      "require": {
+        "@intentic/src": "./src/lib/markdown.ts",
+        "default": "./dist/lib/markdown.js"
+      }
+    },
+    "./dom": {
+      "types": "./dist/lib/dom.d.ts",
+      "import": {
+        "@intentic/src": "./src/lib/dom.ts",
+        "default": "./dist/lib/dom.js"
+      },
+      "require": {
+        "@intentic/src": "./src/lib/dom.ts",
+        "default": "./dist/lib/dom.js"
+      }
+    }
+  },
+  "dependencies": {
+    "@intentic/agent-cli": "1.317.0",
+    "@intentic/base": "1.317.0",
+    "@stricli/core": "1.3.0",
+    "parse5": "8.0.1",
+    "playwright": "1.63.0",
+    "tslib": "2.8.1"
+  },
+  "devDependencies": {
+    "@intentic/testing": "0.0.0",
+    "@intentic/tsconfig": "0.0.0",
+    "@types/bun": "1.4.2",
+    "@types/node": "24.13.6",
+    "@typescript/native-preview": "7.0.0-dev.20260707.2"
+  },
+  "scripts": {
+    "build": "tsgo",
+    "typecheck": "tsgo --noEmit -p tsconfig.test.json",
+    "watch": "tsgo --build --watch",
+    "test": "suites",
+    "test:watch": "suites --watch"
+  }
+}

package/src/app.ts

@@ -0,0 +1,43 @@
+import { packageVersion } from "@intentic/agent-cli/version";
+import { buildApplication, buildRouteMap } from "@stricli/core";
+import { cacheCommand } from "./commands/cache.command.js";
+import { crawlCommand } from "./commands/crawl.command.js";
+import { fetchCommand } from "./commands/fetch.command.js";
+
+// The agent-facing contract, kept small — this is what `webq --help` prints.
+const HELP = `Web pages as clean markdown, built for an agent's context window.
+
+  webq fetch <url>                        one page: capsule + budgeted markdown on stdout, whole page saved to a file
+  webq fetch <url> --query "rate limits"  keep only the blocks relevant to a question
+  webq fetch <url> --raw                  whole page, no chrome-pruning
+  webq crawl <url> --max-pages 30         bounded same-site crawl: a file per page + index.md
+  webq crawl <url> --query "webhooks"     best-first: links about the query are visited first
+  webq cache [--clear]                    the shared fetch cache
+
+Every fetch is cached for 15 minutes (--fresh bypasses, --max-age tunes), so parallel agents and repeated
+runs stop paying the network. JS-heavy pages render in the image's Chromium automatically when the static
+HTML is an empty app shell (--browser never|force overrides); without the browser pack webq says so and
+serves the static HTML. Crawls obey robots.txt (--ignore-robots is on you), stay on the start origin
+(--external follows out), and report every skipped-URL count — a capped crawl never poses as a complete one.
+
+Read the capsule first: title · final URL · token cost · fit/raw · cache|network|browser. The saved file
+carries front matter (url, title, fetched_at) so it stays self-describing when found later.
+Exit codes: 0 content, 1 none (HTTP error / empty crawl), 2 broken invocation.`;
+
+export const app = buildApplication(
+    buildRouteMap({
+        routes: {
+            fetch: fetchCommand,
+            crawl: crawlCommand,
+            cache: cacheCommand,
+        },
+        defaultCommand: "fetch",
+        docs: { brief: "webq, agent-native web fetch: URLs as clean budgeted markdown", fullDescription: HELP },
+    }),
+    {
+        name: "webq",
+        versionInfo: { currentVersion: packageVersion(import.meta.url) },
+        scanner: { caseStyle: "allow-kebab-for-camel" },
+        determineExitCode: () => 2,
+    },
+);

package/src/cli.integration.test.ts

@@ -0,0 +1,191 @@
+import { requires } from "@intentic/testing/requires";
+/* The CLI end to end against a loopback fixture site: fetch, cache, query filtering, crawl with robots and caps, budget clipping. */
+import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs";
+import { createServer, type Server } from "node:http";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import type { AddressInfo } from "node:net";
+import { captureCli, type CliOutcome } from "@intentic/agent-cli/testing";
+import { app } from "./app.js";
+import { chromiumAvailable } from "./lib/browser.js";
+
+const page = (title: string, body: string): string =>
+    `<html><head><title>${title}</title></head><body>
+    <nav><a href="/">Home</a><a href="/a">A</a></nav>
+    <main><article>${body}</article></main>
+    <footer><a href="/tos">Terms</a> Copyright</footer></body></html>`;
+
+const paragraphs = (label: string): string =>
+    Array.from(
+        { length: 12 },
+        (_, i) => `<p>${label} paragraph ${i} with plenty of dense readable words in a complete sentence about topics agents research.</p>`,
+    ).join("");
+
+let server: Server;
+let base = "";
+let home = "";
+
+const routes = (): Record<string, { body: string; type?: string }> => ({
+    "/": {
+        body: page(
+            "Home",
+            `<h1>Docs home</h1>${paragraphs("home")}<p><a href="/a">Alpha guide</a> <a href="/sub/c">Webhooks retry reference</a> <a href="/blocked">Secret</a> <a href="https://offsite.example/x">Offsite</a> <a href="/gone">Gone</a></p>`,
+        ),
+    },
+    "/a": { body: page("Alpha", `<h1>Alpha</h1>${paragraphs("alpha")}`) },
+    "/sub/c": { body: page("Webhooks", `<h1>Webhook retries</h1><p>webhook retry policy with backoff.</p>${paragraphs("webhook retry")}`) },
+    "/blocked": { body: page("Blocked", "<h1>Never fetched</h1>") },
+    "/robots.txt": { body: "User-agent: *\nDisallow: /blocked\n", type: "text/plain" },
+    "/plain.json": { body: JSON.stringify({ hello: "world" }), type: "application/json" },
+});
+
+beforeAll(async () => {
+    server = createServer((req, res) => {
+        // A connection cut before any answer: the one failure a crawl can only name, never count as a status.
+        if (req.url === "/cut") {
+            req.socket.destroy();
+            return;
+        }
+        const route = routes()[(req.url ?? "/").split("?")[0] ?? "/"];
+        if (route === undefined) {
+            res.writeHead(404, { "content-type": "text/html" });
+            res.end("<html><body>nope</body></html>");
+            return;
+        }
+        res.writeHead(200, { "content-type": route.type ?? "text/html; charset=utf-8" });
+        res.end(route.body);
+    });
+    await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+    base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`;
+    home = mkdtempSync(join(tmpdir(), "webq-test-"));
+    process.env["WEBQ_HOME"] = home;
+});
+
+afterAll(async () => {
+    await new Promise((resolve) => server.close(resolve));
+    delete process.env["WEBQ_HOME"];
+    rmSync(home, { recursive: true, force: true });
+});
+
+const webq = (...args: string[]): Promise<CliOutcome> => captureCli(app, args);
+
+describe("webq fetch", () => {
+    it("prints a capsule, clips to the budget, and saves the whole page", async () => {
+        const { out, exitCode } = await webq("fetch", `${base}/a`, "--budget", "60");
+        expect(exitCode).toBe(0);
+        expect(out).toContain("webq: Alpha");
+        expect(out).toContain("· network");
+        expect(out).toContain("[cut at 60 of");
+        const path = /saved: (\S+)/.exec(out)?.[1] ?? "";
+        const file = readFileSync(path, "utf8");
+        expect(file).toContain(`url: ${base}/a`);
+        expect(file).toContain("alpha paragraph 11");
+    });
+
+    it("serves the second fetch from cache", async () => {
+        const { out } = await webq("fetch", `${base}/a`, "--budget", "0");
+        expect(out).toContain("· cache");
+    });
+
+    it("bypasses the cache under --fresh", async () => {
+        const { out } = await webq("fetch", `${base}/a`, "--budget", "0", "--fresh");
+        expect(out).toContain("· network");
+    });
+
+    it("keeps only query-relevant blocks and says what it kept", async () => {
+        const { out } = await webq("fetch", `${base}/sub/c`, "--query", "backoff policy", "--budget", "200", "--fresh");
+        expect(out).toContain("webhook retry policy with backoff");
+        expect(out).toMatch(/query (kept|matched)/);
+    });
+
+    it("fences non-HTML text content instead of parsing it", async () => {
+        const { out } = await webq("fetch", `${base}/plain.json`, "--budget", "100");
+        expect(out).toContain("```json");
+        expect(out).toContain('"hello"');
+    });
+
+    it("exits 1 on an HTTP error page", async () => {
+        const { exitCode } = await webq("fetch", `${base}/gone`, "--budget", "0");
+        expect(exitCode).toBe(1);
+    });
+
+    it("emits machine-readable output under --json", async () => {
+        const { out } = await webq("fetch", `${base}/a`, "--json");
+        const parsed = JSON.parse(out) as { title: string; path: string; source: string };
+        expect(parsed.title).toBe("Alpha");
+        expect(parsed.path).toContain(home);
+    });
+});
+
+describe("webq crawl", () => {
+    it("crawls the site under its caps, obeying robots and staying on-origin", async () => {
+        const outDir = join(home, "crawl-basic");
+        const { out, exitCode } = await webq("crawl", `${base}/`, "--max-pages", "10", "--out", outDir, "--fresh");
+        expect(exitCode).toBe(0);
+        expect(out).toContain("3 pages");
+        expect(out).toContain("robots 1");
+        expect(out).toContain("offsite 1");
+        // Two links lied: /gone from the home page and /tos from every footer.
+        expect(out).toContain("http-errors 2");
+        const files = readdirSync(outDir);
+        expect(files).toContain("index.md");
+        expect(files).toContain("index.json");
+        expect(files.join(" ")).not.toContain("blocked");
+        expect(readFileSync(join(outDir, "index.md"), "utf8")).toContain("| Alpha |");
+    });
+
+    it("visits query-relevant links first under a tight cap", async () => {
+        const outDir = join(home, "crawl-query");
+        const { out } = await webq("crawl", `${base}/`, "--query", "webhook retries", "--max-pages", "2", "--out", outDir, "--fresh");
+        expect(out).toContain("2 pages");
+        expect(out).toContain("/sub/c");
+        expect(out).not.toContain(`${base}/a →`);
+        expect(out).toContain("beyond-cap");
+    });
+
+    it("exits 1 when nothing could be crawled", async () => {
+        const { exitCode } = await webq("crawl", `${base}/gone`, "--max-pages", "2", "--out", join(home, "crawl-empty"), "--fresh");
+        expect(exitCode).toBe(1);
+    });
+
+    it("names each page that failed, and why, beside the count", async () => {
+        const { out, exitCode } = await webq("crawl", `${base}/cut`, "--max-pages", "1", "--out", join(home, "crawl-cut"), "--fresh");
+        expect(out).toContain("skipped: errors 1");
+        expect(out).toMatch(new RegExp(`failed: ${base}/cut: \\S`));
+        expect(exitCode).toBe(1);
+    });
+});
+
+describe("webq cache", () => {
+    it("reports entries and clears them", async () => {
+        const stats = await webq("cache");
+        expect(stats.out).toMatch(/cache: \d+ entries/);
+        const cleared = await webq("cache", "--clear");
+        expect(cleared.out).toContain("cache cleared");
+        const after = await webq("cache");
+        expect(after.out).toContain("cache: 0 entries");
+    });
+});
+
+// CI's verify-machine job installs it (ci.yml); ci-base carries no browser, so a stray one cannot wake other suites.
+const chromium = requires(await chromiumAvailable(), "Playwright's Chromium (playwright install chromium)", { lane: "machine" });
+
+describe("browser fallback", () => {
+    it.skipIf(!chromium.runs)(chromium.title("renders an app-shell page through Chromium"), async () => {
+        const shellServer = createServer((_req, res) => {
+            res.writeHead(200, { "content-type": "text/html" });
+            res.end(
+                `<html><head><title>Shell</title></head><body><div id="root"></div><script>document.getElementById("root").innerHTML = "<h1>Hydrated</h1><p>Content that only exists after JavaScript ran in a real browser engine, long enough to clear the app-shell text threshold when rendered, with plenty of words to be sure the heuristic sees a real page.</p>";</script></body></html>`,
+            );
+        });
+        await new Promise<void>((resolve) => shellServer.listen(0, "127.0.0.1", resolve));
+        const shellBase = `http://127.0.0.1:${(shellServer.address() as AddressInfo).port}`;
+        try {
+            const { out } = await webq("fetch", `${shellBase}/`, "--budget", "100", "--fresh");
+            expect(out).toContain("# Hydrated");
+            expect(out).toContain("· browser");
+        } finally {
+            await new Promise((resolve) => shellServer.close(resolve));
+        }
+    });
+});

package/src/cli.ts

@@ -0,0 +1,6 @@
+#!/usr/bin/env node
+// The shared agent-CLI shell (@intentic/agent-cli/run) is the ONE thing this file may import at runtime: it
+// pulls nothing in itself, so webq's own module graph still loads inside a catch that can report a bad install.
+import { runAgentCli } from "@intentic/agent-cli/run";
+
+await runAgentCli({ name: "webq", noun: "page", load: async () => ({ app: (await import("./app.js")).app }) });