@intentic/webq 1.317.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Artur Kurowski
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,46 @@
+# webq
+
+A CLI that fetches web pages for an agent as pruned, token-budgeted markdown and crawls a site into files, over one shared cache.
+
+```mermaid
+flowchart LR
+    agent["Agent<br/>webq fetch · crawl"] --> webq(["webq"])
+    webq --> cache["Shared cache<br/>per URL and render mode"]
+    webq --> http["Static HTTP"]
+    webq --> chromium["Headless Chromium<br/>empty app shells"]
+    webq --> out["Capsule on stdout<br/>whole page saved"]
+```
+
+- A raw page is mostly navigation and script, and one docs page can fill a context window. `webq fetch` prints a
+  capsule (title, final URL, token cost, cache, network or browser), then the page with its chrome pruned and clipped
+  to `--budget`, and saves the whole page as markdown with front matter.
+- `--query` keeps only the blocks relevant to a question (BM25). On a crawl it also visits matching links first.
+- When the static HTML is an empty app shell, the page renders in the image's Chromium if that feature pack is
+  installed; without it webq says so and serves the static HTML.
+- Crawls stay on the start origin, obey `robots.txt` and report every skipped URL by reason, so a capped crawl never
+  reads as complete.
+- The sandbox image puts `webq` on PATH and loads [plugin/skills/webq/SKILL.md](plugin/skills/webq/SKILL.md), which
+  tells agents when to use it over WebFetch. It does not handle pages behind a sign-in. [fileq](../fileq) reuses the
+  `./dom` and `./markdown` exports for HTML-based documents.
+
+## Usage
+
+```sh
+webq fetch https://docs.example.com/guide --query "rate limits"
+webq crawl https://docs.example.com --max-pages 30
+```
+
+## Key files
+
+- [src/app.ts](src/app.ts) — the commands and the `--help` text an agent reads.
+- [src/lib/page.ts](src/lib/page.ts) — one URL to one `PageResult`: cache, static fetch, browser fallback, notes.
+- [src/lib/prune.ts](src/lib/prune.ts) — scores elements and drops navigation and page chrome.
+- [src/lib/markdown.ts](src/lib/markdown.ts) — DOM to markdown with absolute links.
+- [src/lib/crawl.ts](src/lib/crawl.ts) — bounded crawl, breadth-first or query-steered, with skip counts.
+- [src/cli.integration.test.ts](src/cli.integration.test.ts) — the CLI end to end against a loopback fixture site.
+
+## Commands
+
+```sh
+pnpm --filter @intentic/webq test
+```

package/dist/app.d.ts

@@ -0,0 +1,2 @@
+export declare const app: import("@stricli/core").Application<import("@stricli/core").CommandContext>;
+//# sourceMappingURL=app.d.ts.map

package/dist/app.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"app.d.ts","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AA0BA,eAAO,MAAM,GAAG,6EAgBf,CAAC"}

package/dist/app.js

@@ -0,0 +1,38 @@
+import { packageVersion } from "@intentic/agent-cli/version";
+import { buildApplication, buildRouteMap } from "@stricli/core";
+import { cacheCommand } from "./commands/cache.command.js";
+import { crawlCommand } from "./commands/crawl.command.js";
+import { fetchCommand } from "./commands/fetch.command.js";
+const HELP = `Web pages as clean markdown, built for an agent's context window.
+
+  webq fetch <url>                        one page: capsule + budgeted markdown on stdout, whole page saved to a file
+  webq fetch <url> --query "rate limits"  keep only the blocks relevant to a question
+  webq fetch <url> --raw                  whole page, no chrome-pruning
+  webq crawl <url> --max-pages 30         bounded same-site crawl: a file per page + index.md
+  webq crawl <url> --query "webhooks"     best-first: links about the query are visited first
+  webq cache [--clear]                    the shared fetch cache
+
+Every fetch is cached for 15 minutes (--fresh bypasses, --max-age tunes), so parallel agents and repeated
+runs stop paying the network. JS-heavy pages render in the image's Chromium automatically when the static
+HTML is an empty app shell (--browser never|force overrides); without the browser pack webq says so and
+serves the static HTML. Crawls obey robots.txt (--ignore-robots is on you), stay on the start origin
+(--external follows out), and report every skipped-URL count — a capped crawl never poses as a complete one.
+
+Read the capsule first: title · final URL · token cost · fit/raw · cache|network|browser. The saved file
+carries front matter (url, title, fetched_at) so it stays self-describing when found later.
+Exit codes: 0 content, 1 none (HTTP error / empty crawl), 2 broken invocation.`;
+export const app = buildApplication(buildRouteMap({
+    routes: {
+        fetch: fetchCommand,
+        crawl: crawlCommand,
+        cache: cacheCommand,
+    },
+    defaultCommand: "fetch",
+    docs: { brief: "webq, agent-native web fetch: URLs as clean budgeted markdown", fullDescription: HELP },
+}), {
+    name: "webq",
+    versionInfo: { currentVersion: packageVersion(import.meta.url) },
+    scanner: { caseStyle: "allow-kebab-for-camel" },
+    determineExitCode: () => 2,
+});
+//# sourceMappingURL=app.js.map

package/dist/app.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"app.js","sourceRoot":"","sources":["../src/app.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAG3D,MAAM,IAAI,GAAG;;;;;;;;;;;;;;;;;+EAiBkE,CAAC;AAEhF,MAAM,CAAC,MAAM,GAAG,GAAG,gBAAgB,CAC/B,aAAa,CAAC;IACV,MAAM,EAAE;QACJ,KAAK,EAAE,YAAY;QACnB,KAAK,EAAE,YAAY;QACnB,KAAK,EAAE,YAAY;KACtB;IACD,cAAc,EAAE,OAAO;IACvB,IAAI,EAAE,EAAE,KAAK,EAAE,+DAA+D,EAAE,eAAe,EAAE,IAAI,EAAE;CAC1G,CAAC,EACF;IACI,IAAI,EAAE,MAAM;IACZ,WAAW,EAAE,EAAE,cAAc,EAAE,cAAc,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,EAAE;IAChE,OAAO,EAAE,EAAE,SAAS,EAAE,uBAAuB,EAAE;IAC/C,iBAAiB,EAAE,GAAG,EAAE,CAAC,CAAC;CAC7B,CACJ,CAAC"}

package/dist/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}

package/dist/cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { runAgentCli } from "@intentic/agent-cli/run";
+await runAgentCli({ name: "webq", noun: "page", load: async () => ({ app: (await import("./app.js")).app }) });
+//# sourceMappingURL=cli.js.map

package/dist/cli.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAGA,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AAEtD,MAAM,WAAW,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC"}

package/dist/commands/cache.command.d.ts

@@ -0,0 +1,3 @@
+import { type CommandContext } from "@stricli/core";
+export declare const cacheCommand: import("@stricli/core").Command<CommandContext>;
+//# sourceMappingURL=cache.command.d.ts.map

package/dist/commands/cache.command.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.command.d.ts","sourceRoot":"","sources":["../../src/commands/cache.command.ts"],"names":[],"mappings":"AAIA,OAAO,EAAgB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAIlE,eAAO,MAAM,YAAY,iDAiBvB,CAAC"}

package/dist/commands/cache.command.js

@@ -0,0 +1,54 @@
+import { readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { toolHome } from "@intentic/agent-cli/env";
+import { buildCommand } from "@stricli/core";
+import { cacheClear } from "../lib/cache.js";
+import { cacheDir } from "../lib/env.js";
+export const cacheCommand = buildCommand({
+    docs: { brief: "Show the shared fetch cache; --clear empties it" },
+    parameters: {
+        flags: {
+            clear: { kind: "boolean", default: false, brief: "Delete every cached fetch" },
+        },
+        positional: { kind: "tuple", parameters: [] },
+    },
+    async func(flags) {
+        if (flags.clear) {
+            await cacheClear();
+            this.process.stdout.write("cache cleared\n");
+            return;
+        }
+        const { entries, bytes } = await measure(cacheDir());
+        this.process.stdout.write(`webq home: ${toolHome("webq")}\ncache: ${entries} entries, ${(bytes / 1024 / 1024).toFixed(1)} MiB\n`);
+    },
+});
+const measure = async (dir) => {
+    let entries = 0;
+    let bytes = 0;
+    const walkDir = async (current) => {
+        let names;
+        try {
+            names = await readdir(current);
+        }
+        catch {
+            return;
+        }
+        for (const name of names) {
+            const path = join(current, name);
+            const info = await stat(path).catch(() => undefined);
+            if (info === undefined) {
+                continue;
+            }
+            if (info.isDirectory()) {
+                await walkDir(path);
+            }
+            else {
+                entries += 1;
+                bytes += info.size;
+            }
+        }
+    };
+    await walkDir(dir);
+    return { entries, bytes };
+};
+//# sourceMappingURL=cache.command.js.map

package/dist/commands/cache.command.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.command.js","sourceRoot":"","sources":["../../src/commands/cache.command.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAE,YAAY,EAAuB,MAAM,eAAe,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,MAAM,CAAC,MAAM,YAAY,GAAG,YAAY,CAAC;IACrC,IAAI,EAAE,EAAE,KAAK,EAAE,iDAAiD,EAAE;IAClE,UAAU,EAAE;QACR,KAAK,EAAE;YACH,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,2BAA2B,EAAE;SACjF;QACD,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE;KAChD;IACD,KAAK,CAAC,IAAI,CAAuB,KAAkC;QAC/D,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YACd,MAAM,UAAU,EAAE,CAAC;YACnB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;YAC7C,OAAO;QACX,CAAC;QACD,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,MAAM,CAAC,YAAY,OAAO,aAAa,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACtI,CAAC;CACJ,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,KAAK,EAAE,GAAW,EAA+C,EAAE;IAC/E,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,OAAO,GAAG,KAAK,EAAE,OAAe,EAAiB,EAAE;QACrD,IAAI,KAAe,CAAC;QACpB,IAAI,CAAC;YACD,KAAK,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACL,OAAO;QACX,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACrB,SAAS;YACb,CAAC;YACD,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACrB,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACJ,OAAO,IAAI,CAAC,CAAC;gBACb,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC;YACvB,CAAC;QACL,CAAC;IACL,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;IACnB,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;AAC9B,CAAC,CAAC"}

package/dist/commands/crawl.command.d.ts

@@ -0,0 +1,3 @@
+import { type CommandContext } from "@stricli/core";
+export declare const crawlCommand: import("@stricli/core").Command<CommandContext>;
+//# sourceMappingURL=crawl.command.d.ts.map

package/dist/commands/crawl.command.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.command.d.ts","sourceRoot":"","sources":["../../src/commands/crawl.command.ts"],"names":[],"mappings":"AAIA,OAAO,EAAgB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AAqBlE,eAAO,MAAM,YAAY,iDAoEvB,CAAC"}

package/dist/commands/crawl.command.js

@@ -0,0 +1,77 @@
+import { join } from "node:path";
+import { toolOutDir } from "@intentic/agent-cli/env";
+import { countParser } from "@intentic/agent-cli/flags";
+import { buildCommand } from "@stricli/core";
+import { DEFAULT_MAX_AGE_S } from "../lib/cache.js";
+import { crawl } from "../lib/crawl.js";
+import { sharedFlagParameters, urlParser } from "../lib/flags.js";
+import { slugFor } from "../lib/output.js";
+const FAILURES_SHOWN = 10;
+export const crawlCommand = buildCommand({
+    docs: { brief: "Bounded same-site crawl: one markdown file per page, plus an index" },
+    parameters: {
+        flags: {
+            ...sharedFlagParameters,
+            maxPages: { kind: "parsed", parse: countParser, default: "20", brief: "Hard page cap" },
+            depth: { kind: "parsed", parse: countParser, default: "2", brief: "Link hops from the start URL" },
+            concurrency: { kind: "parsed", parse: countParser, default: "4", brief: "Pages in flight at once" },
+            sitemap: { kind: "boolean", default: false, brief: "Seed the frontier from the site's sitemap" },
+            external: { kind: "boolean", default: false, brief: "Follow links off the start origin too" },
+            ignoreRobots: { kind: "boolean", default: false, brief: "Crawl paths robots.txt disallows (your responsibility)" },
+            include: { kind: "parsed", parse: String, variadic: true, optional: true, brief: "Only URLs containing this (or *-glob); repeatable" },
+            exclude: { kind: "parsed", parse: String, variadic: true, optional: true, brief: "Skip URLs containing this (or *-glob); repeatable" },
+            delay: { kind: "parsed", parse: countParser, default: "0", brief: "Milliseconds between requests (robots crawl-delay still respected)" },
+        },
+        positional: { kind: "tuple", parameters: [{ parse: urlParser, brief: "Where the crawl starts", placeholder: "url" }] },
+    },
+    async func(flags, url) {
+        const outDir = flags.out ?? join(toolOutDir("webq"), slugFor(url).replace(/\.md$/, ""));
+        const report = await crawl(url, {
+            outDir,
+            maxPages: flags.maxPages,
+            maxDepth: flags.depth,
+            concurrency: flags.concurrency,
+            robots: !flags.ignoreRobots,
+            sitemap: flags.sitemap,
+            external: flags.external,
+            ...(flags.include === undefined ? {} : { include: flags.include }),
+            ...(flags.exclude === undefined ? {} : { exclude: flags.exclude }),
+            delayMs: flags.delay,
+            raw: flags.raw,
+            query: flags.query,
+            browser: flags.browser,
+            maxAgeS: flags.fresh ? 0 : (flags.maxAge ?? DEFAULT_MAX_AGE_S),
+            timeoutMs: flags.timeout * 1000,
+            threshold: flags.threshold,
+        });
+        if (flags.json) {
+            this.process.stdout.write(`${JSON.stringify(report)}\n`);
+            process.exitCode = report.pages.length > 0 ? 0 : 1;
+            return;
+        }
+        const totalTokens = report.pages.reduce((sum, page) => sum + page.tokens, 0);
+        const skipLine = Object.entries(report.skipped)
+            .filter(([, count]) => count > 0)
+            .map(([reason, count]) => `${reason} ${count}`)
+            .join(", ");
+        this.process.stdout.write(`webq crawl: ${url} · ${report.pages.length} pages · ${totalTokens} tokens${skipLine === "" ? "" : ` · skipped: ${skipLine}`}\n`);
+        for (const { url: failed, reason } of report.failures.slice(0, FAILURES_SHOWN)) {
+            this.process.stdout.write(`failed: ${failed}: ${reason}\n`);
+        }
+        if (report.failures.length > FAILURES_SHOWN) {
+            this.process.stdout.write(`…and ${report.failures.length - FAILURES_SHOWN} more failed pages (--json lists every one)\n`);
+        }
+        if (report.sitemapCapped) {
+            this.process.stdout.write("note: sitemap larger than the seed cap, frontier seeded from a prefix of it\n");
+        }
+        this.process.stdout.write(`index: ${report.indexMarkdown}\n`);
+        for (const page of report.pages) {
+            this.process.stdout.write(`  ${page.tokens}t d${page.depth} ${page.url} → ${page.path}\n`);
+            for (const note of page.notes) {
+                this.process.stdout.write(`    note: ${note}\n`);
+            }
+        }
+        process.exitCode = report.pages.length > 0 ? 0 : 1;
+    },
+});
+//# sourceMappingURL=crawl.command.js.map

package/dist/commands/crawl.command.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.command.js","sourceRoot":"","sources":["../../src/commands/crawl.command.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAuB,MAAM,eAAe,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAC;AACxC,OAAO,EAAE,oBAAoB,EAAoB,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAG3C,MAAM,cAAc,GAAG,EAAE,CAAC;AAc1B,MAAM,CAAC,MAAM,YAAY,GAAG,YAAY,CAAC;IACrC,IAAI,EAAE,EAAE,KAAK,EAAE,oEAAoE,EAAE;IACrF,UAAU,EAAE;QACR,KAAK,EAAE;YACH,GAAG,oBAAoB;YACvB,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE;YACvF,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,8BAA8B,EAAE;YAClG,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,yBAAyB,EAAE;YACnG,OAAO,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,2CAA2C,EAAE;YAChG,QAAQ,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,uCAAuC,EAAE;YAC7F,YAAY,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,wDAAwD,EAAE;YAClH,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,mDAAmD,EAAE;YACtI,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,mDAAmD,EAAE;YACtI,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,oEAAoE,EAAE;SAC3I;QACD,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,wBAAwB,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE;KACzH;IACD,KAAK,CAAC,IAAI,CAAuB,KAAiB,EAAE,GAAW;QAC3D,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC5B,MAAM;YACN,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,QAAQ,EAAE,KAAK,CAAC,KAAK;YACrB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,MAAM,EAAE,CAAC,KAAK,CAAC,YAAY;YAC3B,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;YAClE,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;YAClE,OAAO,EAAE,KAAK,CAAC,KAAK;YACpB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,iBAAiB,CAAC;YAC9D,SAAS,EAAE,KAAK,CAAC,OAAO,GAAG,IAAI;YAC/B,SAAS,EAAE,KAAK,CAAC,SAAS;SAC7B,CAAC,CAAC;QACH,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACb,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACzD,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,OAAO;QACX,CAAC;QACD,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC7E,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;aAC1C,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC;aAChC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,MAAM,IAAI,KAAK,EAAE,CAAC;aAC9C,IAAI,CAAC,IAAI,CAAC,CAAC;QAChB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CACrB,eAAe,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,YAAY,WAAW,UAAU,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,QAAQ,EAAE,IAAI,CACnI,CAAC;QACF,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,EAAE,CAAC;YAC7E,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;YAC1C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,cAAc,+CAA+C,CAAC,CAAC;QAC9H,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;QAC/G,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;QAC9D,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;YAC3F,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAC5B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC;YACrD,CAAC;QACL,CAAC;QACD,OAAO,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvD,CAAC;CACJ,CAAC,CAAC"}

package/dist/commands/fetch.command.d.ts

@@ -0,0 +1,3 @@
+import { type CommandContext } from "@stricli/core";
+export declare const fetchCommand: import("@stricli/core").Command<CommandContext>;
+//# sourceMappingURL=fetch.command.d.ts.map

package/dist/commands/fetch.command.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.command.d.ts","sourceRoot":"","sources":["../../src/commands/fetch.command.ts"],"names":[],"mappings":"AAIA,OAAO,EAAgB,KAAK,cAAc,EAAE,MAAM,eAAe,CAAC;AASlE,eAAO,MAAM,YAAY,iDAqCvB,CAAC"}

package/dist/commands/fetch.command.js

@@ -0,0 +1,55 @@
+import { toolOutDir } from "@intentic/agent-cli/env";
+import { countParser } from "@intentic/agent-cli/flags";
+import { capsule, clip } from "@intentic/agent-cli/output";
+import { buildCommand } from "@stricli/core";
+import { closeBrowser } from "../lib/browser.js";
+import { DEFAULT_MAX_AGE_S } from "../lib/cache.js";
+import { sharedFlagParameters, urlParser } from "../lib/flags.js";
+import { savePage } from "../lib/output.js";
+import { fetchPage } from "../lib/page.js";
+export const fetchCommand = buildCommand({
+    docs: { brief: "One URL as clean markdown: budgeted on stdout, whole in a saved file" },
+    parameters: {
+        flags: {
+            ...sharedFlagParameters,
+            budget: { kind: "parsed", parse: countParser, default: "4000", brief: "Max stdout tokens; 0 prints only the capsule" },
+        },
+        positional: { kind: "tuple", parameters: [{ parse: urlParser, brief: "The page to fetch", placeholder: "url" }] },
+    },
+    async func(flags, url) {
+        try {
+            const page = await fetchPage(url, {
+                raw: flags.raw,
+                query: flags.query,
+                browser: flags.browser,
+                maxAgeS: flags.fresh ? 0 : (flags.maxAge ?? DEFAULT_MAX_AGE_S),
+                timeoutMs: flags.timeout * 1000,
+                threshold: flags.threshold,
+            });
+            const saved = await savePage(flags.out ?? toolOutDir("webq"), page, new Date());
+            if (flags.json) {
+                this.process.stdout.write(`${JSON.stringify({ url, finalUrl: page.finalUrl, status: page.status, title: page.meta.title, tokens: saved.tokens, path: saved.path, source: page.source, notes: page.notes })}\n`);
+            }
+            else {
+                this.process.stdout.write(capsule("webq", capsuleFields(page, saved.tokens, flags.raw), page.notes));
+                this.process.stdout.write(`saved: ${saved.path}\n`);
+                if (flags.budget > 0 && page.markdown !== "") {
+                    this.process.stdout.write("---\n");
+                    this.process.stdout.write(clip(page.markdown, flags.budget, saved.path, "page"));
+                }
+            }
+            process.exitCode = page.status >= 400 ? 1 : 0;
+        }
+        finally {
+            await closeBrowser();
+        }
+    },
+});
+const capsuleFields = (page, tokens, raw) => [
+    page.meta.title === "" ? page.finalUrl : page.meta.title,
+    page.finalUrl,
+    `${tokens} tokens`,
+    raw ? "raw" : page.prunedShare === undefined ? "fit" : `fit (pruned ${Math.round(page.prunedShare * 100)}% of text mass)`,
+    page.source,
+];
+//# sourceMappingURL=fetch.command.js.map

package/dist/commands/fetch.command.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fetch.command.js","sourceRoot":"","sources":["../../src/commands/fetch.command.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAuB,MAAM,eAAe,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,oBAAoB,EAAoB,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAmB,MAAM,gBAAgB,CAAC;AAI5D,MAAM,CAAC,MAAM,YAAY,GAAG,YAAY,CAAC;IACrC,IAAI,EAAE,EAAE,KAAK,EAAE,sEAAsE,EAAE;IACvF,UAAU,EAAE;QACR,KAAK,EAAE;YACH,GAAG,oBAAoB;YACvB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,8CAA8C,EAAE;SACzH;QACD,UAAU,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,mBAAmB,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC,EAAE;KACpH;IACD,KAAK,CAAC,IAAI,CAAuB,KAAiB,EAAE,GAAW;QAC3D,IAAI,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;gBAC9B,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,IAAI,iBAAiB,CAAC;gBAC9D,SAAS,EAAE,KAAK,CAAC,OAAO,GAAG,IAAI;gBAC/B,SAAS,EAAE,KAAK,CAAC,SAAS;aAC7B,CAAC,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC;YAChF,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACb,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CACrB,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,CACvL,CAAC;YACN,CAAC;iBAAM,CAAC;gBACJ,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;gBACrG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;gBACpD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;oBAC3C,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBACnC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;gBACrF,CAAC;YACL,CAAC;YACD,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClD,CAAC;gBAAS,CAAC;YACP,MAAM,YAAY,EAAE,CAAC;QACzB,CAAC;IACL,CAAC;CACJ,CAAC,CAAC;AAIH,MAAM,aAAa,GAAG,CAAC,IAAgB,EAAE,MAAc,EAAE,GAAY,EAAY,EAAE,CAAC;IAChF,IAAI,CAAC,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK;IACxD,IAAI,CAAC,QAAQ;IACb,GAAG,MAAM,SAAS;IAClB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,GAAG,CAAC,iBAAiB;IACzH,IAAI,CAAC,MAAM;CACd,CAAC"}

package/dist/lib/bm25.d.ts

@@ -0,0 +1,7 @@
+export interface ScoredBlock<T> {
+    readonly block: T;
+    readonly score: number;
+}
+export declare const tokenize: (text: string) => string[];
+export declare const bm25Rank: <T>(blocks: readonly T[], textOf: (block: T) => string, query: string) => ScoredBlock<T>[];
+//# sourceMappingURL=bm25.d.ts.map

package/dist/lib/bm25.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/lib/bm25.ts"],"names":[],"mappings":"AAIA,MAAM,WAAW,WAAW,CAAC,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAClB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CAC1B;AAED,eAAO,MAAM,QAAQ,SAAU,MAAM,KAAG,MAAM,EAKtB,CAAC;AAyBzB,eAAO,MAAM,QAAQ,GAAI,CAAC,UAAU,SAAS,CAAC,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC,KAAK,MAAM,SAAS,MAAM,KAAG,WAAW,CAAC,CAAC,CAAC,EA8B7G,CAAC"}

package/dist/lib/bm25.js

@@ -0,0 +1,51 @@
+export const tokenize = (text) => text
+    .toLowerCase()
+    .split(/[^\p{L}\p{N}]+/u)
+    .filter((token) => token.length >= 2)
+    .map(foldPlural);
+const foldPlural = (token) => {
+    if (token.length <= 3) {
+        return token;
+    }
+    if (token.endsWith("ies")) {
+        return `${token.slice(0, -3)}y`;
+    }
+    if (/(?:[xz]|ch|sh|o)es$/.test(token)) {
+        return token.slice(0, -2);
+    }
+    return token.endsWith("s") && !token.endsWith("ss") ? token.slice(0, -1) : token;
+};
+const K1 = 1.2;
+const B = 0.75;
+export const bm25Rank = (blocks, textOf, query) => {
+    const queryTerms = [...new Set(tokenize(query))];
+    if (queryTerms.length === 0 || blocks.length === 0) {
+        return [];
+    }
+    const docs = blocks.map((block) => tokenize(textOf(block)));
+    const avgLen = docs.reduce((sum, doc) => sum + doc.length, 0) / docs.length || 1;
+    const docFrequency = new Map();
+    for (const term of queryTerms) {
+        docFrequency.set(term, docs.filter((doc) => doc.includes(term)).length);
+    }
+    const scored = blocks.map((block, index) => {
+        const doc = docs[index] ?? [];
+        const counts = new Map();
+        for (const token of doc) {
+            counts.set(token, (counts.get(token) ?? 0) + 1);
+        }
+        let score = 0;
+        for (const term of queryTerms) {
+            const df = docFrequency.get(term) ?? 0;
+            if (df === 0) {
+                continue;
+            }
+            const idf = Math.log((docs.length - df + 0.5) / (df + 0.5) + 1);
+            const tf = counts.get(term) ?? 0;
+            score += idf * ((tf * (K1 + 1)) / (tf + K1 * (1 - B + B * (doc.length / avgLen))));
+        }
+        return { block, score };
+    });
+    return scored.filter((entry) => entry.score > 0).toSorted((a, b) => b.score - a.score);
+};
+//# sourceMappingURL=bm25.js.map

package/dist/lib/bm25.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.js","sourceRoot":"","sources":["../../src/lib/bm25.ts"],"names":[],"mappings":"AASA,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAC,IAAY,EAAY,EAAE,CAC/C,IAAI;KACC,WAAW,EAAE;KACb,KAAK,CAAC,iBAAiB,CAAC;KACxB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;KACpC,GAAG,CAAC,UAAU,CAAC,CAAC;AAIzB,MAAM,UAAU,GAAG,CAAC,KAAa,EAAU,EAAE;IACzC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IACpC,CAAC;IAED,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AACrF,CAAC,CAAC;AAEF,MAAM,EAAE,GAAG,GAAG,CAAC;AACf,MAAM,CAAC,GAAG,IAAI,CAAC;AAMf,MAAM,CAAC,MAAM,QAAQ,GAAG,CAAI,MAAoB,EAAE,MAA4B,EAAE,KAAa,EAAoB,EAAE;IAC/G,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACjD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjD,OAAO,EAAE,CAAC;IACd,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5D,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;IACjF,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC5B,YAAY,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5E,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACvC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;YACtB,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACpD,CAAC;QACD,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC5B,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;gBACX,SAAS;YACb,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjC,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACvF,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAC5B,CAAC,CAAC,CAAC;IACH,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;AAC3F,CAAC,CAAC"}

package/dist/lib/browser.d.ts

@@ -0,0 +1,8 @@
+export interface RenderedPage {
+    readonly finalUrl: string;
+    readonly html: string;
+}
+export declare const chromiumAvailable: () => Promise<boolean>;
+export declare const renderPage: (url: string, timeoutMs: number) => Promise<RenderedPage>;
+export declare const closeBrowser: () => Promise<void>;
+//# sourceMappingURL=browser.d.ts.map

package/dist/lib/browser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.d.ts","sourceRoot":"","sources":["../../src/lib/browser.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,YAAY;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACzB;AAID,eAAO,MAAM,iBAAiB,QAAa,OAAO,CAAC,OAAO,CAGzD,CAAC;AAEF,eAAO,MAAM,UAAU,QAAe,MAAM,aAAa,MAAM,KAAG,OAAO,CAAC,YAAY,CAkBrF,CAAC;AAEF,eAAO,MAAM,YAAY,QAAa,OAAO,CAAC,IAAI,CAGjD,CAAC"}

package/dist/lib/browser.js

@@ -0,0 +1,29 @@
+import { existsSync } from "node:fs";
+let browser;
+export const chromiumAvailable = async () => {
+    const { chromium } = await import("playwright");
+    return existsSync(chromium.executablePath());
+};
+export const renderPage = async (url, timeoutMs) => {
+    const { chromium } = await import("playwright");
+    browser ??= await chromium.launch({ executablePath: chromium.executablePath(), headless: true, args: ["--no-sandbox"] });
+    const context = await browser.newContext({ javaScriptEnabled: true, viewport: { width: 1280, height: 900 } });
+    try {
+        await context.route("**/*", (route) => {
+            const kind = route.request().resourceType();
+            return kind === "image" || kind === "media" || kind === "font" ? route.abort() : route.continue();
+        });
+        const page = await context.newPage();
+        await page.goto(url, { waitUntil: "domcontentloaded", timeout: timeoutMs });
+        await page.waitForLoadState("networkidle", { timeout: Math.min(5_000, timeoutMs) }).catch(() => undefined);
+        return { finalUrl: page.url(), html: await page.content() };
+    }
+    finally {
+        await context.close();
+    }
+};
+export const closeBrowser = async () => {
+    await browser?.close();
+    browser = undefined;
+};
+//# sourceMappingURL=browser.js.map

package/dist/lib/browser.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"browser.js","sourceRoot":"","sources":["../../src/lib/browser.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAQrC,IAAI,OAA4B,CAAC;AAEjC,MAAM,CAAC,MAAM,iBAAiB,GAAG,KAAK,IAAsB,EAAE;IAC1D,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,cAAc,EAAE,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,GAAW,EAAE,SAAiB,EAAyB,EAAE;IACtF,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAEhD,OAAO,KAAK,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,cAAc,EAAE,QAAQ,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IACzH,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,EAAE,iBAAiB,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAC9G,IAAI,CAAC;QACD,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YAClC,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;YAC5C,OAAO,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,OAAO,IAAI,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACtG,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;QACrC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;QAE5E,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC3G,OAAO,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,IAAI,CAAC,OAAO,EAAE,EAAE,CAAC;IAChE,CAAC;YAAS,CAAC;QACP,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,KAAK,IAAmB,EAAE;IAClD,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC;IACvB,OAAO,GAAG,SAAS,CAAC;AACxB,CAAC,CAAC"}

package/dist/lib/cache.d.ts

@@ -0,0 +1,15 @@
+export type RenderMode = "static" | "browser";
+export interface CacheEntry {
+    readonly url: string;
+    readonly finalUrl: string;
+    readonly status: number;
+    readonly contentType: string;
+    readonly body: string;
+    readonly mode: RenderMode;
+    readonly fetchedAt: number;
+}
+export declare const DEFAULT_MAX_AGE_S = 900;
+export declare const cacheRead: (url: string, mode: RenderMode, maxAgeS: number) => Promise<CacheEntry | undefined>;
+export declare const cacheWrite: (entry: CacheEntry) => Promise<void>;
+export declare const cacheClear: () => Promise<void>;
+//# sourceMappingURL=cache.d.ts.map

package/dist/lib/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../src/lib/cache.ts"],"names":[],"mappings":"AAMA,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE9C,MAAM,WAAW,UAAU;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC9B;AAED,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAOrC,eAAO,MAAM,SAAS,QAAe,MAAM,QAAQ,UAAU,WAAW,MAAM,KAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAQ9G,CAAC;AAEF,eAAO,MAAM,UAAU,UAAiB,UAAU,KAAG,OAAO,CAAC,IAAI,CAIhE,CAAC;AAEF,eAAO,MAAM,UAAU,QAAa,OAAO,CAAC,IAAI,CAE/C,CAAC"}

package/dist/lib/cache.js

@@ -0,0 +1,28 @@
+import { createHash } from "node:crypto";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { cacheDir } from "./env.js";
+export const DEFAULT_MAX_AGE_S = 900;
+const entryPath = (url, mode) => {
+    const hash = createHash("sha256").update(`${mode}\n${url}`).digest("hex");
+    return join(cacheDir(), hash.slice(0, 2), `${hash}.json`);
+};
+export const cacheRead = async (url, mode, maxAgeS) => {
+    try {
+        const raw = await readFile(entryPath(url, mode), "utf8");
+        const entry = JSON.parse(raw);
+        return Date.now() - entry.fetchedAt <= maxAgeS * 1000 ? entry : undefined;
+    }
+    catch {
+        return undefined;
+    }
+};
+export const cacheWrite = async (entry) => {
+    const path = entryPath(entry.url, entry.mode);
+    await mkdir(dirname(path), { recursive: true });
+    await writeFile(path, JSON.stringify(entry));
+};
+export const cacheClear = async () => {
+    await rm(cacheDir(), { recursive: true, force: true });
+};
+//# sourceMappingURL=cache.js.map

package/dist/lib/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/lib/cache.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAcpC,MAAM,CAAC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAErC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,IAAgB,EAAU,EAAE;IACxD,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC1E,OAAO,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;AAC9D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAAE,GAAW,EAAE,IAAgB,EAAE,OAAe,EAAmC,EAAE;IAC/G,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;QACzD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAe,CAAC;QAC5C,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,IAAI,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,EAAE,KAAiB,EAAiB,EAAE;IACjE,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9C,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,MAAM,SAAS,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,KAAK,IAAmB,EAAE;IAChD,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC3D,CAAC,CAAC"}

package/dist/lib/crawl.d.ts

@@ -0,0 +1,31 @@
+import { type IndexEntry } from "./output.js";
+import { type FetchPageOptions } from "./page.js";
+export interface CrawlOptions extends FetchPageOptions {
+    readonly maxPages?: number;
+    readonly maxDepth?: number;
+    readonly concurrency?: number;
+    readonly robots?: boolean;
+    readonly sitemap?: boolean;
+    readonly external?: boolean;
+    readonly include?: string[];
+    readonly exclude?: string[];
+    readonly delayMs?: number;
+    readonly outDir: string;
+}
+export interface CrawlPageReport extends IndexEntry {
+    readonly source: string;
+    readonly notes: string[];
+}
+export interface CrawlReport {
+    readonly pages: CrawlPageReport[];
+    readonly skipped: Record<string, number>;
+    readonly failures: ReadonlyArray<{
+        readonly url: string;
+        readonly reason: string;
+    }>;
+    readonly indexMarkdown: string;
+    readonly indexJson: string;
+    readonly sitemapCapped: boolean;
+}
+export declare const crawl: (startUrl: string, options: CrawlOptions) => Promise<CrawlReport>;
+//# sourceMappingURL=crawl.d.ts.map

package/dist/lib/crawl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.d.ts","sourceRoot":"","sources":["../../src/lib/crawl.ts"],"names":[],"mappings":"AASA,OAAO,EAAuB,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,EAAa,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAI7D,MAAM,WAAW,YAAa,SAAQ,gBAAgB;IAClD,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAE9B,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;IAE5B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAC5B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,eAAgB,SAAQ,UAAU;IAC/C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,KAAK,EAAE,eAAe,EAAE,CAAC;IAClC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAEzC,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC;QAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACpF,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;CACnC;AAWD,eAAO,MAAM,KAAK,aAAoB,MAAM,WAAW,YAAY,KAAG,OAAO,CAAC,WAAW,CAyHxF,CAAC"}