@chatcops/ingest 0.0.0-stage → 0.1.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 CoderCops
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,37 @@
-# Temporary Holding Version
+# @chatcops/ingest
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Turns a website into a knowledge index a ChatCops bot can answer from, with citations. It discovers pages (sitemaps, URL lists, folders of Markdown or HTML), fetches and extracts the readable content, splits it into chunks along headings, embeds them, and writes them to a vector store.
+
+```ts
+import { memoryStorage, memoryVectors, openaiEmbedder } from '@chatcops/core';
+import { ingest } from '@chatcops/ingest';
+
+const embedder = openaiEmbedder({ apiKey: process.env.OPENAI_API_KEY! });
+const stats = await ingest({
+  sources: [{ type: 'sitemap', url: 'https://example.com/sitemap.xml' }],
+  embedder,
+  store: memoryVectors({ dimensions: embedder.dimensions }),
+  manifest: memoryStorage().kv!,
+});
+console.log(stats);
+```
+
+Use a persistent vector store (`@chatcops/store-postgres`, `store-sqlite`, `store-cloudflare` or `store-upstash-vector`) and the same storage's key-value store as the manifest, so later runs only embed what changed.
+
+- Re-running is cheap: unchanged pages (same ETag, or the same extracted text) aren't fetched again or embedded again.
+- Hidden elements, scripts, styles and HTML comments are removed before extraction, since that's where injected instructions usually hide.
+- `prune: true` removes pages that are no longer discovered, but never after a sitemap failed to load.
+- An index remembers its embedder and refuses another one; build a new index with a different `index` name instead.
+- `dryRun: true` discovers, extracts and chunks without embedding or writing anything.
+
+Node 22.13 or later.
+
+## robots.txt
+
+Network sources respect `robots.txt` by default. Each run fetches it once per origin, selecting `ChatCopsBot` rules (or `*`), with the longest matching rule winning and `allow` winning ties. Wildcards and `$` end anchors are supported. Disallowed pages are skipped with the reason `blocked by robots.txt`.
+
+A 4xx response allows crawling; a 5xx response or an unreachable robots server blocks it. Up to five redirects are followed. `crawl-delay` (seconds) sets a minimum interval between requests to an origin, including concurrent workers. Directory sources read local files and never check robots. Set `respectRobots: false` in the ingest config to opt out.
+
+`robotsFor(origin, fetch)` also exposes `Sitemap:` entries as `sitemaps`; ingestion still uses the sources you configure.
+
+When robots.txt is unavailable, affected pages are skipped with a reason such as `robots.txt unavailable (HTTP 503)`. The run reports one error per origin and skips pruning, preserving existing indexed pages until the rules can be read again.

package/dist/chunk.d.ts

@@ -0,0 +1,21 @@
+import type { Chunk, ExtractedDoc } from './types.js';
+export interface ChunkOptions {
+    /** About this many tokens per chunk. Default 500. */
+    targetTokens?: number;
+    /** Tokens repeated from the end of the previous chunk in the same section. Default 60. */
+    overlapTokens?: number;
+}
+/**
+ * A rough token count: about four characters per token, except CJK, where
+ * each character is about one. Good enough to size chunks; never billed on.
+ */
+export declare function estimateTokens(text: string): number;
+/**
+ * Splits a page into chunks: first at h1 to h3, then by packing blocks
+ * (paragraphs, lists, code blocks, tables) up to `targetTokens`. Code blocks
+ * and tables stay whole. Each chunk starts with its heading path
+ * (`Page title > Section > Subsection`), so a chunk read alone still says
+ * what it's about.
+ */
+export declare function chunkDoc(doc: ExtractedDoc, options?: ChunkOptions): Chunk[];
+//# sourceMappingURL=chunk.d.ts.map

package/dist/chunk.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../src/chunk.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAEtD,MAAM,WAAW,YAAY;IAC3B,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,0FAA0F;IAC1F,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAeD;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED;;;;;;GAMG;AACH,wBAAgB,QAAQ,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,GAAE,YAAiB,GAAG,KAAK,EAAE,CAkB/E"}

package/dist/chunk.js

@@ -0,0 +1,162 @@
+const CJK = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/gu;
+/**
+ * A rough token count: about four characters per token, except CJK, where
+ * each character is about one. Good enough to size chunks; never billed on.
+ */
+export function estimateTokens(text) {
+    const cjk = text.match(CJK)?.length ?? 0;
+    return cjk + Math.ceil((text.length - cjk) / 4);
+}
+/**
+ * Splits a page into chunks: first at h1 to h3, then by packing blocks
+ * (paragraphs, lists, code blocks, tables) up to `targetTokens`. Code blocks
+ * and tables stay whole. Each chunk starts with its heading path
+ * (`Page title > Section > Subsection`), so a chunk read alone still says
+ * what it's about.
+ */
+export function chunkDoc(doc, options = {}) {
+    const target = Math.max(50, options.targetTokens ?? 500);
+    const overlap = Math.max(0, Math.min(options.overlapTokens ?? 60, Math.floor(target / 2)));
+    const chunks = [];
+    for (const section of sections(doc.markdown)) {
+        const crumbs = [doc.title, ...section.path].filter((part) => !!part);
+        const breadcrumb = dedupe(crumbs).join(' > ');
+        const section_ = section.path.length ? section.path.join(' > ') : undefined;
+        for (const body of pack(section.blocks.flatMap((block) => split(block, target)), target, overlap)) {
+            chunks.push({
+                index: chunks.length,
+                text: breadcrumb ? `${breadcrumb}\n\n${body}` : body,
+                ...(section_ ? { section: section_ } : {}),
+            });
+        }
+    }
+    return chunks;
+}
+function sections(markdown) {
+    const out = [];
+    // The current heading at each level (1 to 3); a heading clears the ones below it.
+    const levels = [];
+    let current = { path: [], blocks: [] };
+    const lines = markdown.split('\n');
+    const flush = () => {
+        if (current.blocks.length)
+            out.push(current);
+    };
+    for (let i = 0; i < lines.length;) {
+        const line = lines[i];
+        const heading = /^(#{1,3})\s+(.+?)\s*#*\s*$/.exec(line);
+        if (heading) {
+            flush();
+            const level = heading[1].length;
+            levels[level] = heading[2];
+            levels.length = level + 1;
+            current = { path: levels.filter((text) => !!text), blocks: [] };
+            i++;
+            continue;
+        }
+        if (/^\s*(```|~~~)/.test(line)) {
+            const fence = line.trim().slice(0, 3);
+            const start = i;
+            i++;
+            while (i < lines.length && !lines[i].trim().startsWith(fence))
+                i++;
+            i = Math.min(i + 1, lines.length);
+            current.blocks.push({ text: lines.slice(start, i).join('\n'), atomic: true });
+            continue;
+        }
+        if (/^\s*\|/.test(line)) {
+            const start = i;
+            while (i < lines.length && /^\s*\|/.test(lines[i]))
+                i++;
+            current.blocks.push({ text: lines.slice(start, i).join('\n'), atomic: true });
+            continue;
+        }
+        if (!line.trim()) {
+            i++;
+            continue;
+        }
+        const start = i;
+        while (i < lines.length && lines[i].trim() && !/^(#{1,3})\s/.test(lines[i]) && !/^\s*(```|~~~|\|)/.test(lines[i]))
+            i++;
+        current.blocks.push({ text: lines.slice(start, i).join('\n'), atomic: false });
+    }
+    flush();
+    return out;
+}
+/** A paragraph longer than a chunk, split at sentence ends (or, failing that, words). */
+function split(block, target) {
+    if (block.atomic || estimateTokens(block.text) <= target)
+        return [block];
+    const sentences = block.text.match(/[^.!?。！？\n]+(?:[.!?。！？]+|\n|$)\s*/gu) ?? [block.text];
+    const pieces = [];
+    let piece = '';
+    for (const sentence of sentences) {
+        if (piece && estimateTokens(piece + sentence) > target) {
+            pieces.push({ text: piece.trim(), atomic: false });
+            piece = '';
+        }
+        if (estimateTokens(sentence) > target) {
+            for (const part of byLength(sentence, target))
+                pieces.push({ text: part, atomic: false });
+        }
+        else {
+            piece += sentence;
+        }
+    }
+    if (piece.trim())
+        pieces.push({ text: piece.trim(), atomic: false });
+    return pieces;
+}
+function byLength(text, target) {
+    const parts = [];
+    let part = '';
+    for (const char of text) {
+        part += char;
+        if (estimateTokens(part) >= target) {
+            parts.push(part.trim());
+            part = '';
+        }
+    }
+    if (part.trim())
+        parts.push(part.trim());
+    return parts;
+}
+/** Blocks packed into chunk bodies, each starting with the previous chunk's last blocks up to `overlap`. */
+function pack(blocks, target, overlap) {
+    const bodies = [];
+    let current = [];
+    let tokens = 0;
+    let fresh = false;
+    const close = () => {
+        if (fresh)
+            bodies.push(current.map((block) => block.text).join('\n\n'));
+        const tail = [];
+        let tailTokens = 0;
+        for (let i = current.length - 1; i >= 0; i--) {
+            const block = current[i];
+            const size = estimateTokens(block.text);
+            if (block.atomic || tailTokens + size > overlap)
+                break;
+            tail.unshift(block);
+            tailTokens += size;
+        }
+        current = tail;
+        tokens = tailTokens;
+        fresh = false;
+    };
+    for (const block of blocks) {
+        const size = estimateTokens(block.text);
+        if (fresh && tokens + size > target)
+            close();
+        current.push(block);
+        tokens += size;
+        fresh = true;
+    }
+    if (fresh)
+        bodies.push(current.map((block) => block.text).join('\n\n'));
+    return bodies;
+}
+function dedupe(parts) {
+    return parts.filter((part, i) => i === 0 || part !== parts[i - 1]);
+}
+//# sourceMappingURL=chunk.js.map

package/dist/chunk.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../src/chunk.ts"],"names":[],"mappings":"AAoBA,MAAM,GAAG,GAAG,2EAA2E,CAAC;AAExF;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;IACzC,OAAO,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;AAClD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAiB,EAAE,UAAwB,EAAE;IACpE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC,CAAC;IACzD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,MAAM,MAAM,GAAY,EAAE,CAAC;IAE3B,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrF,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC5E,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;YAClG,MAAM,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,MAAM,CAAC,MAAM;gBACpB,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI;gBACpD,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC3C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,QAAgB;IAChC,MAAM,GAAG,GAAc,EAAE,CAAC;IAC1B,kFAAkF;IAClF,MAAM,MAAM,GAA8B,EAAE,CAAC;IAC7C,IAAI,OAAO,GAAY,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;IAChD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM;YAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC/C,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAI,CAAC;QACnC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,OAAO,GAAG,4BAA4B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,EAAE,CAAC;YACR,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YAChC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC3B,MAAM,CAAC,MAAM,GAAG,KAAK,GAAG,CAAC,CAAC;YAC1B,OAAO,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YAChF,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACtC,MAAM,KAAK,GAAG,CAAC,CAAC;YAChB,CAAC,EAAE,CAAC;YACJ,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,CAAC,EAAE,CAAC;YACnE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;YAClC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9E,SAAS;QACX,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,CAAC,CAAC;YAChB,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAAE,CAAC,EAAE,CAAC;YACxD,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9E,SAAS;QACX,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACjB,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QACD,MAAM,KAAK,GAAG,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,CAAC,EAAE,CAAC;QACvH,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,GAAG,CAAC;AACb,CAAC;AAED,yFAAyF;AACzF,SAAS,KAAK,CAAC,KAAY,EAAE,MAAc;IACzC,IAAI,KAAK,CAAC,MAAM,IAAI,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM;QAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IACzE,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,qCAAqC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC1F,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,EAAE,CAAC;IACf,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,KAAK,IAAI,cAAc,CAAC,KAAK,GAAG,QAAQ,CAAC,GAAG,MAAM,EAAE,CAAC;YACvD,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;YACnD,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;QACD,IAAI,cAAc,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,CAAC;YACtC,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5F,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,QAAQ,CAAC;QACpB,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,EAAE;QAAE,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACrE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,MAAc;IAC5C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,IAAI,CAAC;QACb,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACxB,IAAI,GAAG,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;IACzC,OAAO,KAAK,CAAC;AACf,CAAC;AAED,4GAA4G;AAC5G,SAAS,IAAI,CAAC,MAAe,EAAE,MAAc,EAAE,OAAe;IAC5D,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAY,EAAE,CAAC;IAC1B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,KAAK,GAAG,KAAK,CAAC;IAElB,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,KAAK;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QACxE,MAAM,IAAI,GAAY,EAAE,CAAC;QACzB,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YACzB,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,GAAG,IAAI,GAAG,OAAO;gBAAE,MAAM;YACvD,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACpB,UAAU,IAAI,IAAI,CAAC;QACrB,CAAC;QACD,OAAO,GAAG,IAAI,CAAC;QACf,MAAM,GAAG,UAAU,CAAC;QACpB,KAAK,GAAG,KAAK,CAAC;IAChB,CAAC,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,KAAK,IAAI,MAAM,GAAG,IAAI,GAAG,MAAM;YAAE,KAAK,EAAE,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpB,MAAM,IAAI,IAAI,CAAC;QACf,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IACD,IAAI,KAAK;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IACxE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,KAAe;IAC7B,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC"}

package/dist/discover.d.ts

@@ -0,0 +1,25 @@
+import type { DiscoveredPage, IngestSource } from './types.js';
+export interface DiscoverOptions {
+    fetch: typeof fetch;
+    userAgent: string;
+    timeoutMs: number;
+    include?: string[];
+    exclude?: string[];
+}
+export interface Discovery {
+    pages: DiscoveredPage[];
+    /** One line per problem. Any entry means pruning isn't safe. */
+    errors: string[];
+}
+/** Every page the sources list, once each, filtered by `include` and `exclude`. */
+export declare function discover(sources: IngestSource[], options: DiscoverOptions): Promise<Discovery>;
+/** The URL without its fragment, or undefined when it isn't http(s). */
+export declare function normalizeUrl(url: string): string | undefined;
+/**
+ * `*` matches within a path segment, `**` across segments, `?` one character.
+ * A double star followed by a slash also matches no segments at all, so
+ * "docs, any folders, *.md" matches docs/a.md as well as docs/x/a.md.
+ */
+export declare function globToRegExp(glob: string): RegExp;
+export declare function matches(url: string, include?: string[], exclude?: string[]): boolean;
+//# sourceMappingURL=discover.d.ts.map

package/dist/discover.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"discover.d.ts","sourceRoot":"","sources":["../src/discover.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAM/D,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,OAAO,KAAK,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,gEAAgE;IAChE,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,mFAAmF;AACnF,wBAAsB,QAAQ,CAAC,OAAO,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,CAAC,CAsBpG;AA2DD,wEAAwE;AACxE,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAS5D;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAmBjD;AAED,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAGpF"}

package/dist/discover.js

@@ -0,0 +1,148 @@
+import { readdir } from 'node:fs/promises';
+import { join, relative, sep } from 'node:path';
+import { gunzipSync } from 'node:zlib';
+/** Sitemap indexes can point at more indexes; this many levels is plenty. */
+const MAX_SITEMAP_DEPTH = 3;
+const PAGE_FILES = /\.(md|mdx|html?)$/i;
+/** Every page the sources list, once each, filtered by `include` and `exclude`. */
+export async function discover(sources, options) {
+    const pages = new Map();
+    const errors = [];
+    const add = (page) => {
+        const url = normalizeUrl(page.url);
+        if (url && !pages.has(url) && matches(url, options.include, options.exclude))
+            pages.set(url, { ...page, url });
+    };
+    for (const source of sources) {
+        if (source.type === 'urls') {
+            for (const url of source.urls)
+                add({ url });
+        }
+        else if (source.type === 'sitemap') {
+            await readSitemap(source.url, options, 0, new Set(), add, errors);
+        }
+        else {
+            try {
+                for (const page of await readDirectory(source.path, source.baseUrl))
+                    add(page);
+            }
+            catch (error) {
+                errors.push(`${source.path}: ${message(error)}`);
+            }
+        }
+    }
+    return { pages: [...pages.values()], errors };
+}
+async function readSitemap(url, options, depth, seen, add, errors) {
+    if (seen.has(url))
+        return;
+    seen.add(url);
+    if (depth > MAX_SITEMAP_DEPTH) {
+        errors.push(`${url}: sitemap indexes nested more than ${MAX_SITEMAP_DEPTH} deep`);
+        return;
+    }
+    let xml;
+    try {
+        const response = await options.fetch(url, {
+            headers: { 'user-agent': options.userAgent },
+            signal: AbortSignal.timeout(options.timeoutMs),
+        });
+        if (!response.ok)
+            throw new Error(`HTTP ${response.status}`);
+        const bytes = new Uint8Array(await response.arrayBuffer());
+        // gzip's magic number; some servers send .xml.gz without saying so.
+        xml = (bytes[0] === 0x1f && bytes[1] === 0x8b ? gunzipSync(bytes) : Buffer.from(bytes)).toString('utf8');
+    }
+    catch (error) {
+        errors.push(`${url}: ${message(error)}`);
+        return;
+    }
+    const locations = [...xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)].map((match) => decodeXml(match[1]));
+    if (/<sitemapindex[\s>]/i.test(xml)) {
+        for (const child of locations)
+            await readSitemap(child, options, depth + 1, seen, add, errors);
+    }
+    else if (/<urlset[\s>]/i.test(xml)) {
+        for (const location of locations)
+            add({ url: location });
+    }
+    else {
+        errors.push(`${url}: not a sitemap (no <urlset> or <sitemapindex>)`);
+    }
+}
+/** Markdown and HTML files under `path`, with URLs under `baseUrl` (`docs/intro.md` is `<baseUrl>/docs/intro`). */
+async function readDirectory(path, baseUrl) {
+    const entries = await readdir(path, { recursive: true, withFileTypes: true });
+    const base = baseUrl.endsWith('/') ? baseUrl : `${baseUrl}/`;
+    return entries
+        .filter((entry) => entry.isFile() && PAGE_FILES.test(entry.name))
+        .map((entry) => {
+        const file = join(entry.parentPath, entry.name);
+        const route = relative(path, file)
+            .split(sep)
+            .join('/')
+            .replace(PAGE_FILES, '')
+            .replace(/(^|\/)index$/, '$1');
+        return { url: new URL(route, base).href, file };
+    })
+        .sort((a, b) => a.url.localeCompare(b.url));
+}
+/** The URL without its fragment, or undefined when it isn't http(s). */
+export function normalizeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:')
+            return undefined;
+        parsed.hash = '';
+        return parsed.href;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * `*` matches within a path segment, `**` across segments, `?` one character.
+ * A double star followed by a slash also matches no segments at all, so
+ * "docs, any folders, *.md" matches docs/a.md as well as docs/x/a.md.
+ */
+export function globToRegExp(glob) {
+    let source = '';
+    for (let i = 0; i < glob.length; i++) {
+        const char = glob[i];
+        if (char === '*' && glob[i + 1] === '*' && glob[i + 2] === '/') {
+            source += '(?:.*/)?';
+            i += 2;
+        }
+        else if (char === '*' && glob[i + 1] === '*') {
+            source += '.*';
+            i++;
+        }
+        else if (char === '*') {
+            source += '[^/]*';
+        }
+        else if (char === '?') {
+            source += '[^/]';
+        }
+        else {
+            source += char.replace(/[.+^${}()|[\]\\]/g, '\\$&');
+        }
+    }
+    return new RegExp(`^${source}$`);
+}
+export function matches(url, include, exclude) {
+    if (include?.length && !include.some((glob) => globToRegExp(glob).test(url)))
+        return false;
+    return !exclude?.some((glob) => globToRegExp(glob).test(url));
+}
+function decodeXml(text) {
+    return text
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&apos;/g, "'")
+        .replace(/&amp;/g, '&');
+}
+function message(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+//# sourceMappingURL=discover.js.map

package/dist/discover.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"discover.js","sourceRoot":"","sources":["../src/discover.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAGvC,6EAA6E;AAC7E,MAAM,iBAAiB,GAAG,CAAC,CAAC;AAC5B,MAAM,UAAU,GAAG,oBAAoB,CAAC;AAgBxC,mFAAmF;AACnF,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAuB,EAAE,OAAwB;IAC9E,MAAM,KAAK,GAAG,IAAI,GAAG,EAA0B,CAAC;IAChD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,GAAG,GAAG,CAAC,IAAoB,EAAE,EAAE;QACnC,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC;YAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACjH,CAAC,CAAC;IAEF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC3B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI;gBAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAC9C,CAAC;aAAM,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YACrC,MAAM,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACpE,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,KAAK,MAAM,IAAI,IAAI,MAAM,aAAa,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC;oBAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YACjF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,GAAW,EACX,OAAwB,EACxB,KAAa,EACb,IAAiB,EACjB,GAAmC,EACnC,MAAgB;IAEhB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO;IAC1B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,iBAAiB,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,sCAAsC,iBAAiB,OAAO,CAAC,CAAC;QAClF,OAAO;IACT,CAAC;IACD,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE;YACxC,OAAO,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,SAAS,EAAE;YAC5C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC;SAC/C,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QAC3D,oEAAoE;QACpE,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC3G,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,KAAK,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzC,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,8BAA8B,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxG,IAAI,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACpC,KAAK,MAAM,KAAK,IAAI,SAAS;YAAE,MAAM,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IACjG,CAAC;SAAM,IAAI,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACrC,KAAK,MAAM,QAAQ,IAAI,SAAS;YAAE,GAAG,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC3D,CAAC;SAAM,CAAC;QACN,MAAM,CAAC,IAAI,CAAC,GAAG,GAAG,iDAAiD,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,mHAAmH;AACnH,KAAK,UAAU,aAAa,CAAC,IAAY,EAAE,OAAe;IACxD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9E,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,GAAG,CAAC;IAC7D,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SAChE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACb,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;aAC/B,KAAK,CAAC,GAAG,CAAC;aACV,IAAI,CAAC,GAAG,CAAC;aACT,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;aACvB,OAAO,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;QACjC,OAAO,EAAE,GAAG,EAAE,IAAI,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IAClD,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AAChD,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,SAAS,CAAC;QAClF,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACjB,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,MAAM,GAAG,EAAE,CAAC;IAChB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC/D,MAAM,IAAI,UAAU,CAAC;YACrB,CAAC,IAAI,CAAC,CAAC;QACT,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;YAC/C,MAAM,IAAI,IAAI,CAAC;YACf,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,OAAO,CAAC;QACpB,CAAC;aAAM,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,MAAM,CAAC;QACnB,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,mBAAmB,EAAE,MAAM,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IACD,OAAO,IAAI,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,GAAW,EAAE,OAAkB,EAAE,OAAkB;IACzE,IAAI,OAAO,EAAE,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAC3F,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI;SACR,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC;SACrB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC;SACvB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,OAAO,CAAC,KAAc;IAC7B,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC"}

package/dist/extract.d.ts

@@ -0,0 +1,15 @@
+import type { ExtractedDoc } from './types.js';
+/**
+ * The readable content of an HTML page as Markdown. Hidden elements and
+ * comments go first. Then `selector`, when given, picks the content;
+ * otherwise Readability does, falling back to `<main>`, `<article>` or the
+ * body without its navigation, header, footer and sidebars.
+ */
+export declare function extractHtml(html: string, url: string, selector?: string): ExtractedDoc;
+/**
+ * A Markdown or MDX file as a document: front matter gives the title and
+ * description, and MDX `import`/`export` lines and component tags are
+ * removed, keeping the text between tags.
+ */
+export declare function extractMarkdown(source: string, url: string): ExtractedDoc;
+//# sourceMappingURL=extract.d.ts.map

package/dist/extract.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../src/extract.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAkC/C;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,YAAY,CAqCtF;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,YAAY,CAqBzE"}

package/dist/extract.js

@@ -0,0 +1,138 @@
+import { Readability, isProbablyReaderable } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
+/**
+ * Where text that a visitor never sees sits in a page. Prompt injection
+ * usually hides in exactly these places, so they're removed before anything
+ * is read.
+ */
+const INVISIBLE = [
+    'script',
+    'style',
+    'noscript',
+    'template',
+    'iframe',
+    'svg',
+    'canvas',
+    'object',
+    'embed',
+    '[hidden]',
+    '[aria-hidden="true"]',
+    '[style*="display:none"]',
+    '[style*="display: none"]',
+    '[style*="visibility:hidden"]',
+    '[style*="visibility: hidden"]',
+].join(',');
+/** Page chrome, dropped when Readability can't find the article. */
+const CHROME = 'nav, header, footer, aside, form, [role="navigation"], [role="banner"], [role="contentinfo"]';
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced', bulletListMarker: '-' });
+turndown.use(gfm);
+// Images and link targets are noise for search; the link text stays.
+turndown.addRule('no-images', { filter: 'img', replacement: () => '' });
+turndown.addRule('link-text', { filter: 'a', replacement: (content) => content });
+/**
+ * The readable content of an HTML page as Markdown. Hidden elements and
+ * comments go first. Then `selector`, when given, picks the content;
+ * otherwise Readability does, falling back to `<main>`, `<article>` or the
+ * body without its navigation, header, footer and sidebars.
+ */
+export function extractHtml(html, url, selector) {
+    const { document } = parseHTML(html);
+    const lang = document.documentElement?.getAttribute('lang')?.trim() || undefined;
+    const description = document.querySelector('meta[name="description"]')?.getAttribute('content')?.trim() || undefined;
+    const pageTitle = document.querySelector('meta[property="og:title"]')?.getAttribute('content')?.trim() ||
+        document.querySelector('title')?.textContent?.trim() ||
+        undefined;
+    for (const element of [...document.querySelectorAll(INVISIBLE)])
+        element.remove();
+    removeComments(document);
+    let contentHtml;
+    let title = pageTitle;
+    if (selector) {
+        contentHtml = document.querySelector(selector)?.outerHTML;
+    }
+    if (contentHtml === undefined && isProbablyReaderable(document)) {
+        const article = new Readability(document.cloneNode(true)).parse();
+        if (article?.content) {
+            contentHtml = article.content;
+            title = article.title?.trim() || title;
+        }
+    }
+    if (contentHtml === undefined) {
+        const root = document.querySelector('main') ?? document.querySelector('article') ?? document.body;
+        for (const element of [...(root?.querySelectorAll(CHROME) ?? [])])
+            element.remove();
+        contentHtml = root?.innerHTML ?? '';
+    }
+    return {
+        url,
+        ...(title ? { title } : {}),
+        ...(lang ? { lang } : {}),
+        ...(description ? { description } : {}),
+        markdown: tidy(turndown.turndown(contentHtml)),
+    };
+}
+/**
+ * A Markdown or MDX file as a document: front matter gives the title and
+ * description, and MDX `import`/`export` lines and component tags are
+ * removed, keeping the text between tags.
+ */
+export function extractMarkdown(source, url) {
+    let body = source.replace(/^\uFEFF/, '');
+    const frontMatter = {};
+    const front = /^---\r?\n([\s\S]*?)\r?\n---\r?\n?/.exec(body);
+    if (front) {
+        for (const line of front[1].split(/\r?\n/)) {
+            const field = /^(\w+):\s*(.*)$/.exec(line);
+            if (field)
+                frontMatter[field[1]] = field[2].trim().replace(/^(['"])(.*)\1$/, '$2');
+        }
+        body = body.slice(front[0].length);
+    }
+    body = withoutMdx(body);
+    const heading = /^#\s+(.+)$/m.exec(body)?.[1]?.trim();
+    const title = frontMatter.title || heading;
+    return {
+        url,
+        ...(title ? { title } : {}),
+        ...(frontMatter.description ? { description: frontMatter.description } : {}),
+        ...(frontMatter.lang ? { lang: frontMatter.lang } : {}),
+        markdown: tidy(body),
+    };
+}
+/** MDX syntax that isn't content, outside code fences. */
+function withoutMdx(markdown) {
+    const out = [];
+    let inFence = false;
+    for (const line of markdown.split('\n')) {
+        const fence = /^\s*(```|~~~)/.test(line);
+        if (inFence || fence) {
+            out.push(line);
+            if (fence)
+                inFence = !inFence;
+            continue;
+        }
+        if (/^\s*(import|export)\s/.test(line))
+            continue;
+        out.push(line.replace(/<\/?[A-Z][\w.]*[^>]*>/g, ''));
+    }
+    return out.join('\n');
+}
+function removeComments(document) {
+    const all = [document, ...document.querySelectorAll('*')];
+    for (const node of all) {
+        for (const child of [...Array.from(node.childNodes ?? [])]) {
+            if (child.nodeType === 8)
+                child.remove();
+        }
+    }
+}
+/** At most one blank line in a row, no trailing spaces. */
+function tidy(markdown) {
+    return markdown
+        .replace(/[ \t]+$/gm, '')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+//# sourceMappingURL=extract.js.map

package/dist/extract.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"extract.js","sourceRoot":"","sources":["../src/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,GAAG,EAAE,MAAM,qBAAqB,CAAC;AAG1C;;;;GAIG;AACH,MAAM,SAAS,GAAG;IAChB,QAAQ;IACR,OAAO;IACP,UAAU;IACV,UAAU;IACV,QAAQ;IACR,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,UAAU;IACV,sBAAsB;IACtB,yBAAyB;IACzB,0BAA0B;IAC1B,8BAA8B;IAC9B,+BAA+B;CAChC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAEZ,oEAAoE;AACpE,MAAM,MAAM,GAAG,8FAA8F,CAAC;AAE9G,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,gBAAgB,EAAE,GAAG,EAAE,CAAC,CAAC;AAC/G,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAClB,qEAAqE;AACrE,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;AACxE,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;AAElF;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,GAAW,EAAE,QAAiB;IACtE,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,eAAe,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;IACjF,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,0BAA0B,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;IACrH,MAAM,SAAS,GACb,QAAQ,CAAC,aAAa,CAAC,2BAA2B,CAAC,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE;QACpF,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE;QACpD,SAAS,CAAC;IAEZ,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,QAAQ,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;QAAE,OAAO,CAAC,MAAM,EAAE,CAAC;IAClF,cAAc,CAAC,QAAQ,CAAC,CAAC;IAEzB,IAAI,WAA+B,CAAC;IACpC,IAAI,KAAK,GAAG,SAAS,CAAC;IACtB,IAAI,QAAQ,EAAE,CAAC;QACb,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC;IAC5D,CAAC;IACD,IAAI,WAAW,KAAK,SAAS,IAAI,oBAAoB,CAAC,QAA+B,CAAC,EAAE,CAAC;QACvF,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAwB,CAAC,CAAC,KAAK,EAAE,CAAC;QACzF,IAAI,OAAO,EAAE,OAAO,EAAE,CAAC;YACrB,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;YAC9B,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,CAAC;QACzC,CAAC;IACH,CAAC;IACD,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC;QAClG,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YAAE,OAAO,CAAC,MAAM,EAAE,CAAC;QACpF,WAAW,GAAG,IAAI,EAAE,SAAS,IAAI,EAAE,CAAC;IACtC,CAAC;IAED,OAAO;QACL,GAAG;QACH,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3B,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACzB,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;KAC/C,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAc,EAAE,GAAW;IACzD,IAAI,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACzC,MAAM,WAAW,GAA2B,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,mCAAmC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7D,IAAI,KAAK,EAAE,CAAC;QACV,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YAC3C,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC3C,IAAI,KAAK;gBAAE,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACrF,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IACxB,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,IAAI,OAAO,CAAC;IAC3C,OAAO;QACL,GAAG;QACH,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3B,GAAG,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5E,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvD,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC;KACrB,CAAC;AACJ,CAAC;AAED,0DAA0D;AAC1D,SAAS,UAAU,CAAC,QAAgB;IAClC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YACrB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,IAAI,KAAK;gBAAE,OAAO,GAAG,CAAC,OAAO,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,IAAI,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QACjD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC;IACD,OAAO,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,cAAc,CAAC,QAA4F;IAClH,MAAM,GAAG,GAAG,CAAC,QAAQ,EAAE,GAAI,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAA+E,CAAC,CAAC;IACzI,KAAK,MAAM,IAAI,IAAI,GAA8E,EAAE,CAAC;QAClG,KAAK,MAAM,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,QAAQ,KAAK,CAAC;gBAAE,KAAK,CAAC,MAAM,EAAE,CAAC;QAC3C,CAAC;IACH,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,SAAS,IAAI,CAAC,QAAgB;IAC5B,OAAO,QAAQ;SACZ,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,7 @@
+export { ingest, chunkId, manifestKeys, DEFAULT_USER_AGENT } from './ingest.js';
+export { discover, globToRegExp, normalizeUrl, type Discovery, type DiscoverOptions } from './discover.js';
+export { extractHtml, extractMarkdown } from './extract.js';
+export { chunkDoc, estimateTokens, type ChunkOptions } from './chunk.js';
+export { robotsFor, parseRobots, type Robots } from './robots.js';
+export type { IngestConfig, IngestSource, IngestEvent, IngestStats, ExtractedDoc, Chunk, DiscoveredPage } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAChF,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,KAAK,SAAS,EAAE,KAAK,eAAe,EAAE,MAAM,eAAe,CAAC;AAC3G,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,KAAK,MAAM,EAAE,MAAM,aAAa,CAAC;AAClE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/index.js

@@ -0,0 +1,6 @@
+export { ingest, chunkId, manifestKeys, DEFAULT_USER_AGENT } from './ingest.js';
+export { discover, globToRegExp, normalizeUrl } from './discover.js';
+export { extractHtml, extractMarkdown } from './extract.js';
+export { chunkDoc, estimateTokens } from './chunk.js';
+export { robotsFor, parseRobots } from './robots.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAChF,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAwC,MAAM,eAAe,CAAC;AAC3G,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAqB,MAAM,YAAY,CAAC;AACzE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAe,MAAM,aAAa,CAAC"}