@intentic/webq 1.317.0

package/dist/lib/crawl.js

@@ -0,0 +1,188 @@
+import { setTimeout as sleep } from "node:timers/promises";
+import { errorMessage } from "@intentic/base/errors";
+import { tokenize } from "./bm25.js";
+import { closeBrowser } from "./browser.js";
+import { httpFetch } from "./http.js";
+import { normalizeUrl, sameOrigin } from "./links.js";
+import { saveIndex, savePage } from "./output.js";
+import { fetchPage } from "./page.js";
+import { EVERYTHING_ALLOWED, isAllowed, parseRobots } from "./robots.js";
+import { fetchSitemapUrls } from "./sitemap.js";
+const MAX_CRAWL_DELAY_MS = 10_000;
+export const crawl = async (startUrl, options) => {
+    const maxPages = options.maxPages ?? 20;
+    const maxDepth = options.maxDepth ?? 2;
+    const concurrency = Math.max(1, options.concurrency ?? 4);
+    const queryTerms = options.query === undefined ? [] : [...new Set(tokenize(options.query))];
+    const skipped = { robots: 0, offsite: 0, filtered: 0, "beyond-depth": 0, "beyond-cap": 0, "http-errors": 0, errors: 0 };
+    const robots = options.robots === false ? EVERYTHING_ALLOWED : await loadRobots(startUrl, options.timeoutMs);
+    const delayMs = Math.max(options.delayMs ?? 0, Math.min((robots.crawlDelayS ?? 0) * 1000, MAX_CRAWL_DELAY_MS));
+    const taken = new Set();
+    const pending = new Map();
+    let order = 0;
+    const admit = (raw, depth, anchorText) => {
+        const url = normalizeUrl(raw);
+        if (url === undefined || taken.has(url)) {
+            return;
+        }
+        const queued = pending.get(url);
+        if (queued !== undefined) {
+            const score = scoreCandidate(url, anchorText, depth, queryTerms);
+            if (score > queued.score) {
+                queued.score = score;
+                queued.depth = Math.min(queued.depth, depth);
+            }
+            return;
+        }
+        if (options.external !== true && !sameOrigin(url, startUrl)) {
+            skipped.offsite += 1;
+            return;
+        }
+        if (!passesFilters(url, options.include, options.exclude)) {
+            skipped.filtered += 1;
+            return;
+        }
+        if (options.robots !== false && !isAllowed(robots, pathOf(url))) {
+            skipped.robots += 1;
+            return;
+        }
+        if (depth > maxDepth) {
+            skipped["beyond-depth"] += 1;
+            return;
+        }
+        pending.set(url, { url, depth, score: scoreCandidate(url, anchorText, depth, queryTerms), order: order++ });
+    };
+    admit(startUrl, 0, "");
+    let sitemapCapped = false;
+    if (options.sitemap === true) {
+        const seed = await fetchSitemapUrls(startUrl, robots.sitemaps, sitemapFetcher(options.timeoutMs));
+        sitemapCapped = seed.capped;
+        for (const url of seed.urls) {
+            admit(url, 1, "");
+        }
+    }
+    const pages = [];
+    const failures = [];
+    const fetchedAt = new Date();
+    let inFlight = 0;
+    let landed = 0;
+    await new Promise((resolve) => {
+        const pump = () => {
+            if (landed >= maxPages || (pending.size === 0 && inFlight === 0)) {
+                if (inFlight === 0) {
+                    skipped["beyond-cap"] += landed >= maxPages ? pending.size : 0;
+                    resolve();
+                }
+                return;
+            }
+            const width = queryTerms.length > 0 && landed === 0 ? 1 : concurrency;
+            while (inFlight < width && pending.size > 0 && landed + inFlight < maxPages) {
+                const next = takeBest(pending);
+                taken.add(next.url);
+                inFlight += 1;
+                void workOne(next).finally(() => {
+                    inFlight -= 1;
+                    pump();
+                });
+            }
+        };
+        const workOne = async (candidate) => {
+            try {
+                if (delayMs > 0) {
+                    await sleep(delayMs);
+                }
+                const page = await fetchPage(candidate.url, options);
+                if (page.status >= 400) {
+                    skipped["http-errors"] += 1;
+                    return;
+                }
+                const saved = await savePage(options.outDir, page, fetchedAt);
+                landed += 1;
+                pages.push({
+                    url: candidate.url,
+                    title: page.meta.title,
+                    path: saved.path,
+                    tokens: saved.tokens,
+                    depth: candidate.depth,
+                    source: page.source,
+                    notes: page.notes,
+                });
+                for (const link of page.links) {
+                    admit(link.url, candidate.depth + 1, link.text);
+                }
+            }
+            catch (error) {
+                skipped.errors += 1;
+                failures.push({ url: candidate.url, reason: errorMessage(error).split("\n")[0] ?? "" });
+            }
+        };
+        pump();
+    });
+    await closeBrowser();
+    const index = await saveIndex(options.outDir, startUrl, pages, skipped);
+    return { pages, skipped, failures, indexMarkdown: index.markdownPath, indexJson: index.jsonPath, sitemapCapped };
+};
+const takeBest = (pending) => {
+    let best;
+    for (const candidate of pending.values()) {
+        if (best === undefined || candidate.score > best.score || (candidate.score === best.score && candidate.order < best.order)) {
+            best = candidate;
+        }
+    }
+    pending.delete(best.url);
+    return best;
+};
+const scoreCandidate = (url, anchorText, depth, queryTerms) => {
+    if (queryTerms.length === 0) {
+        return -depth;
+    }
+    const haystack = new Set(tokenize(`${anchorText} ${pathOf(url).replaceAll(/[/\-_.]/g, " ")}`));
+    const overlap = queryTerms.filter((term) => haystack.has(term)).length / queryTerms.length;
+    return overlap - depth * 0.05;
+};
+const passesFilters = (url, include, exclude) => {
+    const test = (pattern) => pattern.includes("*")
+        ? new RegExp(pattern.replaceAll(/[.+?^${}()|[\]\\]/g, "\\$&").replaceAll("\\*", ".*")).test(url)
+        : url.includes(pattern);
+    if (include !== undefined && include.length > 0 && !include.some(test)) {
+        return false;
+    }
+    return !(exclude ?? []).some(test);
+};
+const pathOf = (url) => {
+    try {
+        const parsed = new URL(url);
+        return `${parsed.pathname}${parsed.search}`;
+    }
+    catch {
+        return url;
+    }
+};
+const loadRobots = async (startUrl, timeoutMs) => {
+    try {
+        const origin = new URL(startUrl).origin;
+        const response = await httpFetch(`${origin}/robots.txt`, { timeoutMs: Math.min(timeoutMs ?? 10_000, 10_000) });
+        return response.status >= 200 && response.status < 300 ? parseRobots(response.body) : EVERYTHING_ALLOWED;
+    }
+    catch {
+        return EVERYTHING_ALLOWED;
+    }
+};
+const sitemapFetcher = (timeoutMs) => async (url) => {
+    try {
+        if (url.endsWith(".gz")) {
+            const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs ?? 10_000) });
+            if (!response.ok) {
+                return undefined;
+            }
+            const bytes = new Uint8Array(await response.arrayBuffer());
+            return { body: "", bytes };
+        }
+        const response = await httpFetch(url, { timeoutMs: timeoutMs ?? 10_000 });
+        return response.status >= 200 && response.status < 300 ? { body: response.body } : undefined;
+    }
+    catch {
+        return undefined;
+    }
+};
+//# sourceMappingURL=crawl.js.map

package/dist/lib/crawl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl.js","sourceRoot":"","sources":["../../src/lib/crawl.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAmB,MAAM,aAAa,CAAC;AACnE,OAAO,EAAE,SAAS,EAAyB,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,kBAAkB,EAAE,SAAS,EAAE,WAAW,EAAoB,MAAM,aAAa,CAAC;AAC3F,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAuChD,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,MAAM,CAAC,MAAM,KAAK,GAAG,KAAK,EAAE,QAAgB,EAAE,OAAqB,EAAwB,EAAE;IACzF,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;IACvC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;IAC1D,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC5F,MAAM,OAAO,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAExH,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;IAC7G,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,GAAG,IAAI,EAAE,kBAAkB,CAAC,CAAC,CAAC;IAE/G,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,KAAK,GAAG,CAAC,GAAuB,EAAE,KAAa,EAAE,UAAkB,EAAQ,EAAE;QAC/E,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,GAAG,KAAK,SAAS,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtC,OAAO;QACX,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YAEvB,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;YACjE,IAAI,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;gBACvB,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;gBACrB,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACjD,CAAC;YACD,OAAO;QACX,CAAC;QACD,IAAI,OAAO,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC1D,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;YACrB,OAAO;QACX,CAAC;QACD,IAAI,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACxD,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;YACtB,OAAO;QACX,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;YAC9D,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;YACpB,OAAO;QACX,CAAC;QACD,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;YACnB,OAAO,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;YAC7B,OAAO;QACX,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,cAAc,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;IAChH,CAAC,CAAC;IAEF,KAAK,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;IACvB,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAClG,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC;QAC5B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC1B,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACtB,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAsB,EAAE,CAAC;IACpC,MAAM,QAAQ,GAAsC,EAAE,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC;IAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,GAAG,CAAC,CAAC;IAGf,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;QAChC,MAAM,IAAI,GAAG,GAAS,EAAE;YACpB,IAAI,MAAM,IAAI,QAAQ,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;gBAC/D,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;oBACjB,OAAO,CAAC,YAAY,CAAC,IAAI,MAAM,IAAI,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/D,OAAO,EAAE,CAAC;gBACd,CAAC;gBACD,OAAO;YACX,CAAC;YAED,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;YACtE,OAAO,QAAQ,GAAG,KAAK,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,IAAI,MAAM,GAAG,QAAQ,GAAG,QAAQ,EAAE,CAAC;gBAC1E,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAC/B,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACpB,QAAQ,IAAI,CAAC,CAAC;gBACd,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;oBAC5B,QAAQ,IAAI,CAAC,CAAC;oBACd,IAAI,EAAE,CAAC;gBACX,CAAC,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,KAAK,EAAE,SAAoB,EAAiB,EAAE;YAC1D,IAAI,CAAC;gBACD,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBACd,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;gBACzB,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;gBAErD,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;oBACrB,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;oBAC5B,OAAO;gBACX,CAAC;gBACD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC;gBAC9D,MAAM,IAAI,CAAC,CAAC;gBACZ,KAAK,CAAC,IAAI,CAAC;oBACP,GAAG,EAAE,SAAS,CAAC,GAAG;oBAClB,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;oBACtB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,KAAK,EAAE,SAAS,CAAC,KAAK;oBACtB,MAAM,EAAE,IAAI,CAAC,MAAM;oBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;iBACpB,CAAC,CAAC;gBACH,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,KAAK,GAAG,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpD,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC;gBACpB,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC5F,CAAC;QACL,CAAC,CAAC;QACF,IAAI,EAAE,CAAC;IACX,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,EAAE,CAAC;IACrB,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IACxE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,YAAY,EAAE,SAAS,EAAE,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,CAAC;AACrH,CAAC,CAAC;AAEF,MAAM,QAAQ,GAAG,CAAC,OAA+B,EAAa,EAAE;IAC5D,IAAI,IAA2B,CAAC;IAChC,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;QACvC,IAAI,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACzH,IAAI,GAAG,SAAS,CAAC;QACrB,CAAC;IACL,CAAC;IACD,OAAO,CAAC,MAAM,CAAE,IAAkB,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,IAAiB,CAAC;AAC7B,CAAC,CAAC;AAIF,MAAM,cAAc,GAAG,CAAC,GAAW,EAAE,UAAkB,EAAE,KAAa,EAAE,UAAoB,EAAU,EAAE;IACpG,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,CAAC,KAAK,CAAC;IAClB,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,UAAU,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/F,MAAM,OAAO,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IAC3F,OAAO,OAAO,GAAG,KAAK,GAAG,IAAI,CAAC;AAClC,CAAC,CAAC;AAEF,MAAM,aAAa,GAAG,CAAC,GAAW,EAAE,OAA6B,EAAE,OAA6B,EAAW,EAAE;IACzG,MAAM,IAAI,GAAG,CAAC,OAAe,EAAW,EAAE,CACtC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC;QACjB,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,oBAAoB,EAAE,MAAM,CAAC,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAChG,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAChC,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACrE,OAAO,KAAK,CAAC;IACjB,CAAC;IACD,OAAO,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACvC,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,GAAW,EAAU,EAAE;IACnC,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,GAAG,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,GAAG,CAAC;IACf,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,KAAK,EAAE,QAAgB,EAAE,SAA6B,EAAwB,EAAE;IAC/F,IAAI,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;QACxC,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,MAAM,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAE/G,OAAO,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC;IAC7G,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,kBAAkB,CAAC;IAC9B,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,cAAc,GAChB,CAAC,SAA6B,EAAE,EAAE,CAClC,KAAK,EAAE,GAAW,EAA6D,EAAE;IAC7E,IAAI,CAAC;QACD,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC,EAAE,CAAC,CAAC;YACxF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACf,OAAO,SAAS,CAAC;YACrB,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YAC3D,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC;QAC/B,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,SAAS,IAAI,MAAM,EAAE,CAAC,CAAC;QAC1E,OAAO,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;IACjG,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC,CAAC"}

package/dist/lib/dom.d.ts

@@ -0,0 +1,23 @@
+import { type DefaultTreeAdapterMap } from "parse5";
+export type Document = DefaultTreeAdapterMap["document"];
+export type Node = DefaultTreeAdapterMap["node"];
+export type Element = DefaultTreeAdapterMap["element"];
+export type TextNode = DefaultTreeAdapterMap["textNode"];
+export declare const isElement: (node: Node) => node is Element;
+export declare const isText: (node: Node) => node is TextNode;
+export declare const parseHtml: (html: string) => Document;
+export declare const attr: (el: Element, name: string) => string | undefined;
+export declare const childElements: (el: Element) => Element[];
+export declare const textOf: (node: Node) => string;
+export declare const rawTextOf: (node: Node) => string;
+export declare const elementsByTag: (root: Node, tag: string) => Element[];
+export declare const walk: (node: Node, visit: (el: Element) => void) => void;
+export declare const bodyOf: (doc: Document) => Element | undefined;
+export declare const remove: (el: Element) => void;
+export interface PageMeta {
+    readonly title: string;
+    readonly description: string | undefined;
+    readonly lang: string | undefined;
+}
+export declare const metaOf: (doc: Document) => PageMeta;
+//# sourceMappingURL=dom.d.ts.map

package/dist/lib/dom.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dom.d.ts","sourceRoot":"","sources":["../../src/lib/dom.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,KAAK,qBAAqB,EAAS,MAAM,QAAQ,CAAC;AAE3D,MAAM,MAAM,QAAQ,GAAG,qBAAqB,CAAC,UAAU,CAAC,CAAC;AACzD,MAAM,MAAM,IAAI,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;AACjD,MAAM,MAAM,OAAO,GAAG,qBAAqB,CAAC,SAAS,CAAC,CAAC;AACvD,MAAM,MAAM,QAAQ,GAAG,qBAAqB,CAAC,UAAU,CAAC,CAAC;AAEzD,eAAO,MAAM,SAAS,SAAU,IAAI,KAAG,IAAI,IAAI,OAA4B,CAAC;AAC5E,eAAO,MAAM,MAAM,SAAU,IAAI,KAAG,IAAI,IAAI,QAAqC,CAAC;AAElF,eAAO,MAAM,SAAS,SAAU,MAAM,KAAG,QAAuB,CAAC;AAEjE,eAAO,MAAM,IAAI,OAAQ,OAAO,QAAQ,MAAM,KAAG,MAAM,GAAG,SAAyD,CAAC;AAKpH,eAAO,MAAM,aAAa,OAAQ,OAAO,KAAG,OAAO,EAAqC,CAAC;AAGzF,eAAO,MAAM,MAAM,SAAU,IAAI,KAAG,MAAoE,CAAC;AAGzG,eAAO,MAAM,SAAS,SAAU,IAAI,KAAG,MAAoC,CAAC;AAa5E,eAAO,MAAM,aAAa,SAAU,IAAI,OAAO,MAAM,KAAG,OAAO,EAQ9D,CAAC;AAEF,eAAO,MAAM,IAAI,SAAU,IAAI,SAAS,CAAC,EAAE,EAAE,OAAO,KAAK,IAAI,KAAG,IAS/D,CAAC;AAmBF,eAAO,MAAM,MAAM,QAAS,QAAQ,KAAG,OAAO,GAAG,SAA0D,CAAC;AAG5G,eAAO,MAAM,MAAM,OAAQ,OAAO,KAAG,IASpC,CAAC;AAGF,MAAM,WAAW,QAAQ;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;CACrC;AAED,eAAO,MAAM,MAAM,QAAS,QAAQ,KAAG,QAStC,CAAC"}

package/dist/lib/dom.js

@@ -0,0 +1,74 @@
+import { parse } from "parse5";
+export const isElement = (node) => "tagName" in node;
+export const isText = (node) => node.nodeName === "#text";
+export const parseHtml = (html) => parse(html);
+export const attr = (el, name) => el.attrs.find((a) => a.name === name)?.value;
+const NON_CONTENT = new Set(["script", "style", "noscript", "template", "svg", "canvas"]);
+export const childElements = (el) => el.childNodes.filter(isElement);
+export const textOf = (node) => collectText(node).join(" ").replaceAll(/\s+/g, " ").trim();
+export const rawTextOf = (node) => collectText(node).join("");
+const collectText = (node) => {
+    if (isText(node)) {
+        return [node.value];
+    }
+    if (!isElement(node) || NON_CONTENT.has(node.tagName)) {
+        return [];
+    }
+    return node.childNodes.flatMap((child) => collectText(child));
+};
+export const elementsByTag = (root, tag) => {
+    const found = [];
+    walk(root, (el) => {
+        if (el.tagName === tag) {
+            found.push(el);
+        }
+    });
+    return found;
+};
+export const walk = (node, visit) => {
+    if (isElement(node)) {
+        visit(node);
+    }
+    if (isElement(node) || node.nodeName === "#document" || node.nodeName === "#document-fragment") {
+        for (const child of node.childNodes) {
+            walk(child, visit);
+        }
+    }
+};
+const findFirst = (root, match) => {
+    if (isElement(root) && match(root)) {
+        return root;
+    }
+    if (!("childNodes" in root)) {
+        return undefined;
+    }
+    for (const child of root.childNodes) {
+        const hit = findFirst(child, match);
+        if (hit !== undefined) {
+            return hit;
+        }
+    }
+    return undefined;
+};
+export const bodyOf = (doc) => findFirst(doc, (el) => el.tagName === "body");
+export const remove = (el) => {
+    const parent = el.parentNode;
+    if (parent === null || parent === undefined) {
+        return;
+    }
+    const index = parent.childNodes.indexOf(el);
+    if (index >= 0) {
+        parent.childNodes.splice(index, 1);
+    }
+};
+export const metaOf = (doc) => {
+    const title = textOf(findFirst(doc, (el) => el.tagName === "title") ?? { nodeName: "#text", value: "" });
+    const description = findFirst(doc, (el) => el.tagName === "meta" && attr(el, "name") === "description");
+    const html = findFirst(doc, (el) => el.tagName === "html");
+    return {
+        title,
+        description: description === undefined ? undefined : attr(description, "content"),
+        lang: html === undefined ? undefined : attr(html, "lang"),
+    };
+};
+//# sourceMappingURL=dom.js.map

package/dist/lib/dom.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dom.js","sourceRoot":"","sources":["../../src/lib/dom.ts"],"names":[],"mappings":"AAEA,OAAO,EAA8B,KAAK,EAAE,MAAM,QAAQ,CAAC;AAO3D,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAU,EAAmB,EAAE,CAAC,SAAS,IAAI,IAAI,CAAC;AAC5E,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,IAAU,EAAoB,EAAE,CAAC,IAAI,CAAC,QAAQ,KAAK,OAAO,CAAC;AAElF,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAY,EAAY,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAEjE,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,EAAW,EAAE,IAAY,EAAsB,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,KAAK,CAAC;AAGpH,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE1F,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,EAAW,EAAa,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;AAGzF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,IAAU,EAAU,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAGzG,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,IAAU,EAAU,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AAE5E,MAAM,WAAW,GAAG,CAAC,IAAU,EAAY,EAAE;IACzC,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC;IACD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,OAAO,EAAE,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;AAClE,CAAC,CAAC;AAGF,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,IAAU,EAAE,GAAW,EAAa,EAAE;IAChE,MAAM,KAAK,GAAc,EAAE,CAAC;IAC5B,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,EAAE;QACd,IAAI,EAAE,CAAC,OAAO,KAAK,GAAG,EAAE,CAAC;YACrB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnB,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACjB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,IAAU,EAAE,KAA4B,EAAQ,EAAE;IACnE,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IACD,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ,KAAK,WAAW,IAAI,IAAI,CAAC,QAAQ,KAAK,oBAAoB,EAAE,CAAC;QAC7F,KAAK,MAAM,KAAK,IAAK,IAA+B,CAAC,UAAU,EAAE,CAAC;YAC9D,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACvB,CAAC;IACL,CAAC;AACL,CAAC,CAAC;AAGF,MAAM,SAAS,GAAG,CAAC,IAAU,EAAE,KAA+B,EAAuB,EAAE;IACnF,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACjC,OAAO,IAAI,CAAC;IAChB,CAAC;IACD,IAAI,CAAC,CAAC,YAAY,IAAI,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,KAAK,MAAM,KAAK,IAAK,IAA+B,CAAC,UAAU,EAAE,CAAC;QAC9D,MAAM,GAAG,GAAG,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACpC,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACpB,OAAO,GAAG,CAAC;QACf,CAAC;IACL,CAAC;IACD,OAAO,SAAS,CAAC;AACrB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,GAAa,EAAuB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;AAG5G,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,EAAW,EAAQ,EAAE;IACxC,MAAM,MAAM,GAAG,EAAE,CAAC,UAAU,CAAC;IAC7B,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QAC1C,OAAO;IACX,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5C,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QACb,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACvC,CAAC;AACL,CAAC,CAAC;AASF,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,GAAa,EAAY,EAAE;IAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,KAAK,OAAO,CAAC,IAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAW,CAAC,CAAC;IACnH,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,KAAK,MAAM,IAAI,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,KAAK,aAAa,CAAC,CAAC;IACxG,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;IAC3D,OAAO;QACH,KAAK;QACL,WAAW,EAAE,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,SAAS,CAAC;QACjF,IAAI,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;KAC5D,CAAC;AACN,CAAC,CAAC"}

package/dist/lib/env.d.ts

@@ -0,0 +1,2 @@
+export declare const cacheDir: () => string;
+//# sourceMappingURL=env.d.ts.map

package/dist/lib/env.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"env.d.ts","sourceRoot":"","sources":["../../src/lib/env.ts"],"names":[],"mappings":"AAKA,eAAO,MAAM,QAAQ,QAAO,MAAyC,CAAC"}

package/dist/lib/env.js

@@ -0,0 +1,4 @@
+import { join } from "node:path";
+import { toolHome } from "@intentic/agent-cli/env";
+export const cacheDir = () => join(toolHome("webq"), "cache");
+//# sourceMappingURL=env.js.map

package/dist/lib/env.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"env.js","sourceRoot":"","sources":["../../src/lib/env.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AAGnD,MAAM,CAAC,MAAM,QAAQ,GAAG,GAAW,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC"}

package/dist/lib/flags.d.ts

@@ -0,0 +1,69 @@
+import type { BrowserMode } from "./page.js";
+declare const browserModeParser: (raw: string) => BrowserMode;
+export declare const urlParser: (raw: string) => string;
+export declare const sharedFlagParameters: {
+    readonly raw: {
+        readonly kind: "boolean";
+        readonly default: false;
+        readonly brief: "Whole page, skip fit-pruning of chrome";
+    };
+    readonly query: {
+        readonly kind: "parsed";
+        readonly parse: StringConstructor;
+        readonly optional: true;
+        readonly brief: "Keep only blocks relevant to this (BM25); crawl: also steer link order";
+    };
+    readonly browser: {
+        readonly kind: "parsed";
+        readonly parse: typeof browserModeParser;
+        readonly default: "auto";
+        readonly brief: "JS rendering: auto|never|force";
+    };
+    readonly out: {
+        readonly kind: "parsed";
+        readonly parse: StringConstructor;
+        readonly optional: true;
+        readonly brief: "Directory for the markdown files";
+    };
+    readonly fresh: {
+        readonly kind: "boolean";
+        readonly default: false;
+        readonly brief: "Bypass the cache for this run";
+    };
+    readonly maxAge: {
+        readonly kind: "parsed";
+        readonly parse: (raw: string) => number;
+        readonly optional: true;
+        readonly brief: "Cache freshness window in seconds (default 900)";
+    };
+    readonly timeout: {
+        readonly kind: "parsed";
+        readonly parse: (raw: string) => number;
+        readonly default: "20";
+        readonly brief: "Per-page deadline in seconds";
+    };
+    readonly threshold: {
+        readonly kind: "parsed";
+        readonly parse: (raw: string) => number;
+        readonly optional: true;
+        readonly brief: "Fit-pruning score floor (default 0.48; lower keeps more)";
+    };
+    readonly json: {
+        readonly kind: "boolean";
+        readonly default: false;
+        readonly brief: "Machine-readable result on stdout";
+    };
+};
+export interface SharedFlags {
+    readonly raw: boolean;
+    readonly query?: string;
+    readonly browser: BrowserMode;
+    readonly out?: string;
+    readonly fresh: boolean;
+    readonly maxAge?: number;
+    readonly timeout: number;
+    readonly threshold?: number;
+    readonly json: boolean;
+}
+export {};
+//# sourceMappingURL=flags.d.ts.map

package/dist/lib/flags.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"flags.d.ts","sourceRoot":"","sources":["../../src/lib/flags.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAE7C,QAAA,MAAM,iBAAiB,QAAS,MAAM,KAAG,WAKxC,CAAC;AAGF,eAAO,MAAM,SAAS,QAAS,MAAM,KAAG,MAGvC,CAAC;AAGF,eAAO,MAAM,oBAAoB;aAC7B,GAAG;iBAAI,IAAI,EAAE,SAAS;iBAAE,OAAO;iBAAS,KAAK,EAAE,wCAAwC;;aACvF,KAAK;iBAAI,IAAI,EAAE,QAAQ;iBAAE,KAAK;iBAAU,QAAQ;iBAAQ,KAAK,EAAE,wEAAwE;;aACvI,OAAO;iBAAI,IAAI,EAAE,QAAQ;iBAAE,KAAK;iBAAqB,OAAO,EAAE,MAAM;iBAAE,KAAK,EAAE,gCAAgC;;aAC7G,GAAG;iBAAI,IAAI,EAAE,QAAQ;iBAAE,KAAK;iBAAU,QAAQ;iBAAQ,KAAK,EAAE,kCAAkC;;aAC/F,KAAK;iBAAI,IAAI,EAAE,SAAS;iBAAE,OAAO;iBAAS,KAAK,EAAE,+BAA+B;;aAChF,MAAM;iBAAI,IAAI,EAAE,QAAQ;iBAAE,KAAK;iBAAe,QAAQ;iBAAQ,KAAK,EAAE,iDAAiD;;aACtH,OAAO;iBAAI,IAAI,EAAE,QAAQ;iBAAE,KAAK;iBAAe,OAAO,EAAE,IAAI;iBAAE,KAAK,EAAE,8BAA8B;;aACnG,SAAS;iBAAI,IAAI,EAAE,QAAQ;iBAAE,KAAK;iBAAe,QAAQ;iBAAQ,KAAK,EAAE,0DAA0D;;aAClI,IAAI;iBAAI,IAAI,EAAE,SAAS;iBAAE,OAAO;iBAAS,KAAK,EAAE,mCAAmC;;CAC7E,CAAC;AAEX,MAAM,WAAW,WAAW;IACxB,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;IACtB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;CAC1B"}

package/dist/lib/flags.js

@@ -0,0 +1,23 @@
+import { countParser } from "@intentic/agent-cli/flags";
+const browserModeParser = (raw) => {
+    if (raw === "auto" || raw === "never" || raw === "force") {
+        return raw;
+    }
+    throw new Error(`--browser takes auto|never|force, got "${raw}"`);
+};
+export const urlParser = (raw) => {
+    const trimmed = raw.trim();
+    return /^https?:\/\//i.test(trimmed) ? trimmed : `https://${trimmed}`;
+};
+export const sharedFlagParameters = {
+    raw: { kind: "boolean", default: false, brief: "Whole page, skip fit-pruning of chrome" },
+    query: { kind: "parsed", parse: String, optional: true, brief: "Keep only blocks relevant to this (BM25); crawl: also steer link order" },
+    browser: { kind: "parsed", parse: browserModeParser, default: "auto", brief: "JS rendering: auto|never|force" },
+    out: { kind: "parsed", parse: String, optional: true, brief: "Directory for the markdown files" },
+    fresh: { kind: "boolean", default: false, brief: "Bypass the cache for this run" },
+    maxAge: { kind: "parsed", parse: countParser, optional: true, brief: "Cache freshness window in seconds (default 900)" },
+    timeout: { kind: "parsed", parse: countParser, default: "20", brief: "Per-page deadline in seconds" },
+    threshold: { kind: "parsed", parse: countParser, optional: true, brief: "Fit-pruning score floor (default 0.48; lower keeps more)" },
+    json: { kind: "boolean", default: false, brief: "Machine-readable result on stdout" },
+};
+//# sourceMappingURL=flags.js.map

package/dist/lib/flags.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"flags.js","sourceRoot":"","sources":["../../src/lib/flags.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAGxD,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAe,EAAE;IACnD,IAAI,GAAG,KAAK,MAAM,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,OAAO,EAAE,CAAC;QACvD,OAAO,GAAG,CAAC;IACf,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,GAAG,CAAC,CAAC;AACtE,CAAC,CAAC;AAGF,MAAM,CAAC,MAAM,SAAS,GAAG,CAAC,GAAW,EAAU,EAAE;IAC7C,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IAC3B,OAAO,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,OAAO,EAAE,CAAC;AAC1E,CAAC,CAAC;AAGF,MAAM,CAAC,MAAM,oBAAoB,GAAG;IAChC,GAAG,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,wCAAwC,EAAE;IACzF,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,wEAAwE,EAAE;IACzI,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,gCAAgC,EAAE;IAC/G,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,kCAAkC,EAAE;IACjG,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,+BAA+B,EAAE;IAClF,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,iDAAiD,EAAE;IACxH,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,8BAA8B,EAAE;IACrG,SAAS,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,0DAA0D,EAAE;IACpI,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,mCAAmC,EAAE;CAC/E,CAAC"}

package/dist/lib/http.d.ts

@@ -0,0 +1,16 @@
+export interface HttpFetchOptions {
+    readonly timeoutMs?: number;
+    readonly maxBytes?: number;
+    readonly userAgent?: string;
+}
+export interface HttpPage {
+    readonly url: string;
+    readonly finalUrl: string;
+    readonly status: number;
+    readonly contentType: string;
+    readonly body: string;
+    readonly truncated: boolean;
+}
+export declare const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 webq";
+export declare const httpFetch: (url: string, options?: HttpFetchOptions) => Promise<HttpPage>;
+//# sourceMappingURL=http.d.ts.map

package/dist/lib/http.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,gBAAgB;IAC7B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,QAAQ;IACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAErB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC;CAC/B;AAID,eAAO,MAAM,UAAU,+GAA+G,CAAC;AAKvI,eAAO,MAAM,SAAS,QAAe,MAAM,YAAW,gBAAgB,KAAQ,OAAO,CAAC,QAAQ,CAsB7F,CAAC"}

package/dist/lib/http.js

@@ -0,0 +1,75 @@
+export const USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 webq";
+const DEFAULT_TIMEOUT_MS = 20_000;
+const DEFAULT_MAX_BYTES = 10 * 1024 * 1024;
+export const httpFetch = async (url, options = {}) => {
+    const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
+    const response = await fetch(url, {
+        redirect: "follow",
+        signal: AbortSignal.timeout(timeoutMs),
+        headers: {
+            "user-agent": options.userAgent ?? USER_AGENT,
+            accept: "text/html,application/xhtml+xml,application/xml;q=0.9,text/plain;q=0.8,*/*;q=0.5",
+            "accept-language": "en-US,en;q=0.9",
+        },
+    });
+    const contentType = response.headers.get("content-type") ?? "";
+    const { bytes, truncated } = await readCapped(response, maxBytes, timeoutMs);
+    return {
+        url,
+        finalUrl: response.url === "" ? url : response.url,
+        status: response.status,
+        contentType,
+        body: decode(bytes, contentType),
+        truncated,
+    };
+};
+const readCapped = async (response, maxBytes, timeoutMs) => {
+    if (response.body === null) {
+        return { bytes: new Uint8Array(0), truncated: false };
+    }
+    const deadline = Date.now() + timeoutMs;
+    const chunks = [];
+    let total = 0;
+    const reader = response.body.getReader();
+    for (;;) {
+        if (Date.now() > deadline) {
+            await reader.cancel();
+            return { bytes: concat(chunks, total), truncated: true };
+        }
+        const { done, value } = await reader.read();
+        if (done) {
+            return { bytes: concat(chunks, total), truncated: false };
+        }
+        chunks.push(value);
+        total += value.byteLength;
+        if (total >= maxBytes) {
+            await reader.cancel();
+            return { bytes: concat(chunks, total).slice(0, maxBytes), truncated: true };
+        }
+    }
+};
+const concat = (chunks, total) => {
+    const merged = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        merged.set(chunk, offset);
+        offset += chunk.byteLength;
+    }
+    return merged;
+};
+const decode = (bytes, contentType) => {
+    const headerCharset = /charset=["']?([\w-]+)/i.exec(contentType)?.[1];
+    const sniffed = headerCharset ?? sniffMetaCharset(bytes);
+    try {
+        return new TextDecoder(sniffed ?? "utf8", { fatal: false }).decode(bytes);
+    }
+    catch {
+        return new TextDecoder("utf-8", { fatal: false }).decode(bytes);
+    }
+};
+const sniffMetaCharset = (bytes) => {
+    const head = new TextDecoder("latin1").decode(bytes.slice(0, 4096));
+    return (/<meta[^>]+charset=["']?([\w-]+)/i.exec(head) ?? /<\?xml[^>]+encoding=["']([\w-]+)/i.exec(head))?.[1];
+};
+//# sourceMappingURL=http.js.map

package/dist/lib/http.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"http.js","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"AAqBA,MAAM,CAAC,MAAM,UAAU,GAAG,4GAA4G,CAAC;AAEvI,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,SAAS,GAAG,KAAK,EAAE,GAAW,EAAE,OAAO,GAAqB,EAAE,EAAqB,EAAE;IAC9F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC;IAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACvD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAC9B,QAAQ,EAAE,QAAQ;QAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;QACtC,OAAO,EAAE;YACL,YAAY,EAAE,OAAO,CAAC,SAAS,IAAI,UAAU;YAC7C,MAAM,EAAE,kFAAkF;YAC1F,iBAAiB,EAAE,gBAAgB;SACtC;KACJ,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC7E,OAAO;QACH,GAAG;QACH,QAAQ,EAAE,QAAQ,CAAC,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG;QAClD,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,WAAW;QACX,IAAI,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC;QAChC,SAAS;KACZ,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,UAAU,GAAG,KAAK,EAAE,QAAkB,EAAE,QAAgB,EAAE,SAAiB,EAAsD,EAAE;IACrI,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QACzB,OAAO,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAC1D,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IACxC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACzC,SAAS,CAAC;QACN,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;YACxB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAC7D,CAAC;QACD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,IAAI,EAAE,CAAC;YACP,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QAC9D,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;QAC1B,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;YACtB,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QAChF,CAAC;IACL,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,MAAoB,EAAE,KAAa,EAAc,EAAE;IAC/D,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IACrC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;IAC/B,CAAC;IACD,OAAO,MAAM,CAAC;AAClB,CAAC,CAAC;AAEF,MAAM,MAAM,GAAG,CAAC,KAAiB,EAAE,WAAmB,EAAU,EAAE;IAC9D,MAAM,aAAa,GAAG,wBAAwB,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,aAAa,IAAI,gBAAgB,CAAC,KAAK,CAAC,CAAC;IACzD,IAAI,CAAC;QACD,OAAO,IAAI,WAAW,CAAC,OAAO,IAAI,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpE,CAAC;AACL,CAAC,CAAC;AAGF,MAAM,gBAAgB,GAAG,CAAC,KAAiB,EAAsB,EAAE;IAC/D,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IACpE,OAAO,CAAC,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AAClH,CAAC,CAAC"}

package/dist/lib/links.d.ts

@@ -0,0 +1,9 @@
+import { type Node } from "./dom.js";
+export interface Link {
+    readonly url: string;
+    readonly text: string;
+}
+export declare const extractLinks: (root: Node, baseUrl: string) => Link[];
+export declare const normalizeUrl: (raw: string | undefined, base?: string) => string | undefined;
+export declare const sameOrigin: (a: string, b: string) => boolean;
+//# sourceMappingURL=links.d.ts.map

package/dist/lib/links.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"links.d.ts","sourceRoot":"","sources":["../../src/lib/links.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,KAAK,IAAI,EAAU,MAAM,UAAU,CAAC;AAElE,MAAM,WAAW,IAAI;IACjB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACzB;AAED,eAAO,MAAM,YAAY,SAAU,IAAI,WAAW,MAAM,KAAG,IAAI,EAc9D,CAAC;AAEF,eAAO,MAAM,YAAY,QAAS,MAAM,GAAG,SAAS,SAAS,MAAM,KAAG,MAAM,GAAG,SAc9E,CAAC;AAEF,eAAO,MAAM,UAAU,MAAO,MAAM,KAAK,MAAM,KAAG,OAMjD,CAAC"}

package/dist/lib/links.js

@@ -0,0 +1,41 @@
+import { attr, elementsByTag, textOf } from "./dom.js";
+export const extractLinks = (root, baseUrl) => {
+    const seen = new Map();
+    for (const anchor of elementsByTag(root, "a")) {
+        const url = normalizeUrl(attr(anchor, "href"), baseUrl);
+        if (url === undefined) {
+            continue;
+        }
+        const existing = seen.get(url);
+        const text = textOf(anchor).slice(0, 200);
+        if (existing === undefined || (existing.text === "" && text !== "")) {
+            seen.set(url, { url, text });
+        }
+    }
+    return [...seen.values()];
+};
+export const normalizeUrl = (raw, base) => {
+    if (raw === undefined || raw.trim() === "") {
+        return undefined;
+    }
+    try {
+        const url = base === undefined ? new URL(raw.trim()) : new URL(raw.trim(), base);
+        if (url.protocol !== "http:" && url.protocol !== "https:") {
+            return undefined;
+        }
+        url.hash = "";
+        return url.href;
+    }
+    catch {
+        return undefined;
+    }
+};
+export const sameOrigin = (a, b) => {
+    try {
+        return new URL(a).origin === new URL(b).origin;
+    }
+    catch {
+        return false;
+    }
+};
+//# sourceMappingURL=links.js.map

package/dist/lib/links.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"links.js","sourceRoot":"","sources":["../../src/lib/links.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,IAAI,EAAE,aAAa,EAAa,MAAM,EAAE,MAAM,UAAU,CAAC;AAOlE,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,IAAU,EAAE,OAAe,EAAU,EAAE;IAChE,MAAM,IAAI,GAAG,IAAI,GAAG,EAAgB,CAAC;IACrC,KAAK,MAAM,MAAM,IAAI,aAAa,CAAC,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;QACxD,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACpB,SAAS;QACb,CAAC;QACD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1C,IAAI,QAAQ,KAAK,SAAS,IAAI,CAAC,QAAQ,CAAC,IAAI,KAAK,EAAE,IAAI,IAAI,KAAK,EAAE,CAAC,EAAE,CAAC;YAClE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACjC,CAAC;IACL,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;AAC9B,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,CAAC,GAAuB,EAAE,IAAa,EAAsB,EAAE;IACvF,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACzC,OAAO,SAAS,CAAC;IACrB,CAAC;IACD,IAAI,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC;QACjF,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACxD,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,GAAG,CAAC,IAAI,GAAG,EAAE,CAAC;QACd,OAAO,GAAG,CAAC,IAAI,CAAC;IACpB,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,SAAS,CAAC;IACrB,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,CAAS,EAAE,CAAS,EAAW,EAAE;IACxD,IAAI,CAAC;QACD,OAAO,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC,CAAC"}

package/dist/lib/markdown.d.ts

@@ -0,0 +1,6 @@
+import { type Element } from "./dom.js";
+export interface MarkdownOptions {
+    readonly baseUrl?: string | undefined;
+}
+export declare const renderMarkdown: (root: Element, options?: MarkdownOptions) => string;
+//# sourceMappingURL=markdown.d.ts.map

package/dist/lib/markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/lib/markdown.ts"],"names":[],"mappings":"AAGA,OAAO,EAAQ,KAAK,OAAO,EAA2C,MAAM,UAAU,CAAC;AAiBvF,MAAM,WAAW,eAAe;IAE5B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CACzC;AAED,eAAO,MAAM,cAAc,SAAU,OAAO,YAAW,eAAe,KAAQ,MAO7E,CAAC"}