@chatcops/ingest 0.0.0-stage → 0.1.0

package/dist/ingest.d.ts

@@ -0,0 +1,19 @@
+import type { IngestConfig, IngestStats } from './types.js';
+export { DEFAULT_USER_AGENT } from './user-agent.js';
+/**
+ * Builds or refreshes a knowledge index: discovers pages, fetches and
+ * extracts each one, splits it into chunks, embeds them and writes them to
+ * the store. Pages that haven't changed since the last run (same ETag, or
+ * the same extracted text) aren't embedded again, so re-running costs
+ * nothing until the site changes. Refuses to mix embedders in one index.
+ */
+export declare function ingest(config: IngestConfig): Promise<IngestStats>;
+/** Where the manifest keeps an index's records, under one prefix per index. */
+export declare function manifestKeys(index: string): {
+    meta: string;
+    pages: string;
+    page: (url: string) => string;
+};
+/** Stable per page and position, so re-indexing a page replaces its chunks. */
+export declare function chunkId(url: string, index: number): string;
+//# sourceMappingURL=ingest.d.ts.map

package/dist/ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAgC,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG1F,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAoBrD;;;;;;GAMG;AACH,wBAAsB,MAAM,CAAC,MAAM,EAAE,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC,CAqNvE;AAED,+EAA+E;AAC/E,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM;;;gBAK1B,MAAM;EAErB;AAED,+EAA+E;AAC/E,wBAAgB,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAE1D"}

package/dist/ingest.js

@@ -0,0 +1,255 @@
+import { createHash } from 'node:crypto';
+import { readFile } from 'node:fs/promises';
+import { chunkDoc } from './chunk.js';
+import { discover, globToRegExp } from './discover.js';
+import { extractHtml, extractMarkdown } from './extract.js';
+import { robotsFor } from './robots.js';
+import { DEFAULT_USER_AGENT } from './user-agent.js';
+export { DEFAULT_USER_AGENT } from './user-agent.js';
+/**
+ * Builds or refreshes a knowledge index: discovers pages, fetches and
+ * extracts each one, splits it into chunks, embeds them and writes them to
+ * the store. Pages that haven't changed since the last run (same ETag, or
+ * the same extracted text) aren't embedded again, so re-running costs
+ * nothing until the site changes. Refuses to mix embedders in one index.
+ */
+export async function ingest(config) {
+    const index = config.index ?? 'default';
+    const rawFetch = config.fetch ?? fetch;
+    const userAgent = config.userAgent ?? DEFAULT_USER_AGENT;
+    const timeoutMs = config.timeoutMs ?? 30_000;
+    const keys = manifestKeys(index);
+    const { embedder, store, manifest } = config;
+    const policies = new Map();
+    const errors = [];
+    const lastRequest = new Map();
+    const queues = new Map();
+    async function scheduled(url, init, delay = 0) {
+        const origin = new URL(url).origin;
+        const start = async () => {
+            const due = (lastRequest.get(origin) ?? -Infinity) + delay * 1000;
+            // Timers can fire early; use a monotonic clock and recheck the deadline.
+            while (performance.now() < due)
+                await new Promise((resolve) => setTimeout(resolve, Math.ceil(due - performance.now())));
+            lastRequest.set(origin, performance.now());
+            return { response: rawFetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs) }) };
+        };
+        const started = delay > 0 ? (queues.get(origin) ?? Promise.resolve()).then(start) : start();
+        queues.set(origin, started.then(() => { }));
+        return (await started).response;
+    }
+    function policy(url) {
+        const origin = new URL(url).origin;
+        let pending = policies.get(origin);
+        if (!pending) {
+            pending = robotsFor(origin, ((input, init) => scheduled(String(input), {
+                ...init, headers: { 'user-agent': userAgent }, signal: AbortSignal.timeout(timeoutMs),
+            }))).then((rules) => {
+                if (rules.unavailable)
+                    errors.push(`${origin}/robots.txt unavailable (${rules.unavailable})`);
+                return rules;
+            });
+            policies.set(origin, pending);
+        }
+        return pending;
+    }
+    const send = async (input, init) => {
+        const url = String(input);
+        const rules = config.respectRobots === false ? undefined : await policy(url);
+        return scheduled(url, init, rules?.crawlDelay);
+    };
+    if (embedder.dimensions !== store.dimensions) {
+        throw new Error(`The embedder (${embedder.id}) makes ${embedder.dimensions}-dimension vectors, but the store holds ${store.dimensions}.`);
+    }
+    const meta = await manifest.get(keys.meta);
+    if (meta && meta.embedder !== embedder.id) {
+        throw new Error(`Index "${index}" was built with ${meta.embedder}, not ${embedder.id}. Vectors from different embedders can't be searched together: use the same embedder, or build a new index with another name.`);
+    }
+    const found = await discover(config.sources, { fetch: send, userAgent, timeoutMs, include: config.include, exclude: config.exclude });
+    if (config.maxPages !== undefined && config.maxPages >= 0 && found.pages.length > config.maxPages) {
+        found.pages = found.pages.slice(0, Math.floor(config.maxPages));
+        // Pages left out aren't removed pages, so pruning a partial run isn't safe either.
+        found.errors.push(`stopped at --max-pages ${config.maxPages}`);
+    }
+    config.onEvent?.({ type: 'discovered', pages: found.pages.length, errors: found.errors });
+    errors.push(...found.errors);
+    const stats = {
+        discovered: found.pages.length,
+        indexed: 0,
+        unchanged: 0,
+        skipped: 0,
+        failed: 0,
+        pruned: 0,
+        chunks: 0,
+        embedded: 0,
+        errors,
+    };
+    if (!config.dryRun)
+        await manifest.set(keys.meta, { embedder: embedder.id, dimensions: embedder.dimensions });
+    const selectors = Object.entries(config.selectors ?? {}).map(([glob, selector]) => ({ test: globToRegExp(glob), selector }));
+    const selectorFor = (url) => selectors.find((entry) => entry.test.test(url))?.selector;
+    async function load(page, previous) {
+        if (page.file) {
+            const source = await readFile(page.file, 'utf8');
+            const doc = /\.html?$/i.test(page.file) ? extractHtml(source, page.url, selectorFor(page.url)) : extractMarkdown(source, page.url);
+            return { kind: 'page', doc };
+        }
+        if (config.respectRobots !== false) {
+            const rules = await policy(page.url);
+            if (rules.unavailable)
+                return { kind: 'skip', reason: `robots.txt unavailable (${rules.unavailable})` };
+            if (!rules.allowed(page.url))
+                return { kind: 'skip', reason: 'blocked by robots.txt' };
+        }
+        const headers = { 'user-agent': userAgent, accept: 'text/html,application/xhtml+xml' };
+        if (previous && !config.force) {
+            if (previous.etag)
+                headers['if-none-match'] = previous.etag;
+            if (previous.lastModified)
+                headers['if-modified-since'] = previous.lastModified;
+        }
+        const response = await send(page.url, { headers, signal: AbortSignal.timeout(timeoutMs), redirect: 'follow' });
+        if (response.status === 304)
+            return { kind: 'not-modified' };
+        if (!response.ok)
+            throw new Error(`HTTP ${response.status}`);
+        const type = response.headers.get('content-type') ?? '';
+        if (type && !/html/i.test(type))
+            return { kind: 'skip', reason: `not HTML (${type.split(';')[0]})` };
+        const doc = extractHtml(await response.text(), page.url, selectorFor(page.url));
+        return {
+            kind: 'page',
+            doc,
+            ...(response.headers.get('etag') ? { etag: response.headers.get('etag') } : {}),
+            ...(response.headers.get('last-modified') ? { lastModified: response.headers.get('last-modified') } : {}),
+        };
+    }
+    async function process(page) {
+        const pageKey = keys.page(page.url);
+        const previous = await manifest.get(pageKey);
+        try {
+            const fetched = await load(page, previous);
+            if (fetched.kind === 'skip') {
+                stats.skipped++;
+                config.onEvent?.({ type: 'page', url: page.url, status: 'skipped', reason: fetched.reason });
+                return;
+            }
+            if (fetched.kind === 'not-modified') {
+                stats.unchanged++;
+                config.onEvent?.({ type: 'page', url: page.url, status: 'unchanged' });
+                return;
+            }
+            const contentHash = sha256(fetched.doc.markdown);
+            if (previous && previous.contentHash === contentHash && !config.force) {
+                stats.unchanged++;
+                if (!config.dryRun && (fetched.etag !== previous.etag || fetched.lastModified !== previous.lastModified)) {
+                    await manifest.set(pageKey, { ...previous, etag: fetched.etag, lastModified: fetched.lastModified });
+                }
+                config.onEvent?.({ type: 'page', url: page.url, status: 'unchanged' });
+                return;
+            }
+            const chunks = chunkDoc(fetched.doc, config.chunk);
+            if (!chunks.length) {
+                stats.skipped++;
+                config.onEvent?.({ type: 'page', url: page.url, status: 'skipped', reason: 'no text' });
+                return;
+            }
+            stats.chunks += chunks.length;
+            if (config.dryRun) {
+                stats.indexed++;
+                config.onEvent?.({ type: 'page', url: page.url, status: 'indexed', chunks: chunks.length });
+                return;
+            }
+            const vectors = await embedder.embed(chunks.map((chunk) => chunk.text), 'document');
+            stats.embedded += chunks.length;
+            const metadata = {
+                ...(fetched.doc.lang ? { locale: fetched.doc.lang } : {}),
+                ...config.metadata?.(fetched.doc),
+            };
+            const records = chunks.map((chunk, i) => ({
+                id: chunkId(page.url, chunk.index),
+                vector: vectors[i],
+                text: chunk.text,
+                source: {
+                    id: page.url,
+                    url: page.url,
+                    ...(fetched.doc.title ? { title: fetched.doc.title } : {}),
+                    ...(chunk.section ? { section: chunk.section } : {}),
+                },
+                ...(Object.keys(metadata).length ? { metadata } : {}),
+            }));
+            // The new chunks are embedded before the old ones go, so a failed
+            // embedding never leaves the page out of the index.
+            if (previous)
+                await store.deleteBySource(page.url);
+            await store.upsert(records);
+            await manifest.set(pageKey, {
+                contentHash,
+                chunkIds: records.map((record) => record.id),
+                ...(fetched.etag ? { etag: fetched.etag } : {}),
+                ...(fetched.lastModified ? { lastModified: fetched.lastModified } : {}),
+            });
+            stats.indexed++;
+            config.onEvent?.({ type: 'page', url: page.url, status: 'indexed', chunks: chunks.length });
+        }
+        catch (error) {
+            stats.failed++;
+            config.onEvent?.({ type: 'page', url: page.url, status: 'failed', reason: error instanceof Error ? error.message : String(error) });
+        }
+    }
+    await inPool(found.pages, Math.max(1, config.concurrency ?? 4), process);
+    const known = new Set((await manifest.get(keys.pages)) ?? []);
+    const discovered = new Set(found.pages.map((page) => page.url));
+    if (config.prune) {
+        if (stats.errors.length) {
+            stats.pruneSkipped = `${stats.errors.length} error(s): unable to verify all sources; pruning could remove valid pages`;
+        }
+        else if (!config.dryRun) {
+            for (const url of known) {
+                if (discovered.has(url))
+                    continue;
+                await store.deleteBySource(url);
+                await manifest.delete(keys.page(url));
+                known.delete(url);
+                stats.pruned++;
+                config.onEvent?.({ type: 'pruned', url });
+            }
+        }
+        else {
+            stats.pruned = [...known].filter((url) => !discovered.has(url)).length;
+        }
+    }
+    if (!config.dryRun) {
+        for (const url of discovered) {
+            if (await manifest.get(keys.page(url)))
+                known.add(url);
+        }
+        await manifest.set(keys.pages, [...known].sort());
+    }
+    return stats;
+}
+/** Where the manifest keeps an index's records, under one prefix per index. */
+export function manifestKeys(index) {
+    const prefix = `chatcops:ingest:${index}`;
+    return {
+        meta: `${prefix}:meta`,
+        pages: `${prefix}:pages`,
+        page: (url) => `${prefix}:page:${url}`,
+    };
+}
+/** Stable per page and position, so re-indexing a page replaces its chunks. */
+export function chunkId(url, index) {
+    return sha256(`${url}#${index}`);
+}
+function sha256(text) {
+    return createHash('sha256').update(text).digest('hex');
+}
+async function inPool(items, size, run) {
+    let next = 0;
+    const worker = async () => {
+        while (next < items.length)
+            await run(items[next++]);
+    };
+    await Promise.all(Array.from({ length: Math.min(size, items.length) }, worker));
+}
+//# sourceMappingURL=ingest.js.map

package/dist/ingest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAe,MAAM,aAAa,CAAC;AAErD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAErD,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAoBrD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAAC,MAAoB;IAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,SAAS,CAAC;IACxC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC;IACvC,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,kBAAkB,CAAC;IACzD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC;IAC7C,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,CAAC;IAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA2B,CAAC;IACpD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAEhD,KAAK,UAAU,SAAS,CAAC,GAAW,EAAE,IAAkB,EAAE,KAAK,GAAG,CAAC;QACjE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnC,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,MAAM,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC;YAClE,yEAAyE;YACzE,OAAO,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG;gBAAE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;YACxH,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAC;YAC3C,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,GAAG,EAAE,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,CAAC;QAC1F,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC;QAC5F,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC;QAC3C,OAAO,CAAC,MAAM,OAAO,CAAC,CAAC,QAAQ,CAAC;IAClC,CAAC;IAED,SAAS,MAAM,CAAC,GAAW;QACzB,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;QACnC,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACnC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;gBACrE,GAAG,IAAI,EAAE,OAAO,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC;aACtF,CAAC,CAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;gBAClC,IAAI,KAAK,CAAC,WAAW;oBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,4BAA4B,KAAK,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC9F,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;YACH,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,MAAM,IAAI,GAAiB,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE;QAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;QAC1B,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;QAC7E,OAAO,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IACjD,CAAC,CAAC;IAEF,IAAI,QAAQ,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;QAC7C,MAAM,IAAI,KAAK,CACb,iBAAiB,QAAQ,CAAC,EAAE,WAAW,QAAQ,CAAC,UAAU,2CAA2C,KAAK,CAAC,UAAU,GAAG,CACzH,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAY,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,UAAU,KAAK,oBAAoB,IAAI,CAAC,QAAQ,SAAS,QAAQ,CAAC,EAAE,+HAA+H,CACpM,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;IACtI,IAAI,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,IAAI,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;QAClG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAChE,mFAAmF;QACnF,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1F,MAAM,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IAE7B,MAAM,KAAK,GAAgB;QACzB,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM;QAC9B,OAAO,EAAE,CAAC;QACV,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,CAAC;QACX,MAAM;KACP,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAsB,CAAC,CAAC;IAElI,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;IAC7H,MAAM,WAAW,GAAG,CAAC,GAAW,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC;IAE/F,KAAK,UAAU,IAAI,CAAC,IAAoB,EAAE,QAA2B;QACnE,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACjD,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YACnI,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;QAC/B,CAAC;QACD,IAAI,MAAM,CAAC,aAAa,KAAK,KAAK,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,IAAI,KAAK,CAAC,WAAW;gBAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,2BAA2B,KAAK,CAAC,WAAW,GAAG,EAAE,CAAC;YACxG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,uBAAuB,EAAE,CAAC;QACzF,CAAC;QACD,MAAM,OAAO,GAA2B,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,EAAE,iCAAiC,EAAE,CAAC;QAC/G,IAAI,QAAQ,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,QAAQ,CAAC,IAAI;gBAAE,OAAO,CAAC,eAAe,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC;YAC5D,IAAI,QAAQ,CAAC,YAAY;gBAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,QAAQ,CAAC,YAAY,CAAC;QAClF,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC/G,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG;YAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC;QAC7D,IAAI,CAAC,QAAQ,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACxD,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;QACrG,MAAM,GAAG,GAAG,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAChF,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,GAAG;YACH,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpH,CAAC;IACJ,CAAC;IAED,KAAK,UAAU,OAAO,CAAC,IAAoB;QACzC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAa,OAAO,CAAC,CAAC;QACzD,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC3C,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC5B,KAAK,CAAC,OAAO,EAAE,CAAC;gBAChB,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC7F,OAAO;YACT,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;gBACpC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAClB,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;gBACvE,OAAO;YACT,CAAC;YACD,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACjD,IAAI,QAAQ,IAAI,QAAQ,CAAC,WAAW,KAAK,WAAW,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACtE,KAAK,CAAC,SAAS,EAAE,CAAC;gBAClB,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,YAAY,KAAK,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACzG,MAAM,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;gBACvG,CAAC;gBACD,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC,CAAC;gBACvE,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;YACnD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnB,KAAK,CAAC,OAAO,EAAE,CAAC;gBAChB,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxF,OAAO;YACT,CAAC;YACD,KAAK,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;YAC9B,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;gBAClB,KAAK,CAAC,OAAO,EAAE,CAAC;gBAChB,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC5F,OAAO;YACT,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;YACpF,KAAK,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,CAAC;YAChC,MAAM,QAAQ,GAAG;gBACf,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACzD,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC;aAClC,CAAC;YACF,MAAM,OAAO,GAAmB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxD,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,CAAC;gBAClC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;gBAClB,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,MAAM,EAAE;oBACN,EAAE,EAAE,IAAI,CAAC,GAAG;oBACZ,GAAG,EAAE,IAAI,CAAC,GAAG;oBACb,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC1D,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACrD;gBACD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACtD,CAAC,CAAC,CAAC;YACJ,kEAAkE;YAClE,oDAAoD;YACpD,IAAI,QAAQ;gBAAE,MAAM,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnD,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC5B,MAAM,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE;gBAC1B,WAAW;gBACX,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5C,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,GAAG,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACnD,CAAC,CAAC;YACxB,KAAK,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QAC9F,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,KAAK,CAAC,MAAM,EAAE,CAAC;YACf,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACtI,CAAC;IACH,CAAC;IAED,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAEzE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,QAAQ,CAAC,GAAG,CAAW,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAChE,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACxB,KAAK,CAAC,YAAY,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,2EAA2E,CAAC;QACzH,CAAC;aAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,IAAI,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC;oBAAE,SAAS;gBAClC,MAAM,KAAK,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;gBAChC,MAAM,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBACtC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBAClB,KAAK,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;YAC5C,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;QACzE,CAAC;IACH,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,IAAI,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,YAAY,CAAC,KAAa;IACxC,MAAM,MAAM,GAAG,mBAAmB,KAAK,EAAE,CAAC;IAC1C,OAAO;QACL,IAAI,EAAE,GAAG,MAAM,OAAO;QACtB,KAAK,EAAE,GAAG,MAAM,QAAQ;QACxB,IAAI,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,GAAG,MAAM,SAAS,GAAG,EAAE;KAC/C,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,MAAM,UAAU,OAAO,CAAC,GAAW,EAAE,KAAa;IAChD,OAAO,MAAM,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,MAAM,CAAC,IAAY;IAC1B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACzD,CAAC;AAED,KAAK,UAAU,MAAM,CAAI,KAAU,EAAE,IAAY,EAAE,GAA+B;IAChF,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;QACxB,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM;YAAE,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACvD,CAAC,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;AAClF,CAAC"}

package/dist/robots.d.ts

@@ -0,0 +1,13 @@
+export interface Robots {
+    allowed(url: string): boolean;
+    sitemaps: string[];
+    /** Why the policy could not be fetched; all pages must remain blocked. */
+    unavailable?: string;
+    /** Minimum interval between requests, in seconds. */
+    crawlDelay?: number;
+}
+/** Parse and combine all groups matching ChatCopsBot, falling back to '*'. */
+export declare function parseRobots(text: string): Robots;
+/** Fetch a policy; callers cache this promise once per origin for each run. */
+export declare function robotsFor(origin: string, send?: typeof fetch): Promise<Robots>;
+//# sourceMappingURL=robots.d.ts.map

package/dist/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,MAAM;IACrB,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IAC9B,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAgBD,8EAA8E;AAC9E,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAwDhD;AAED,+EAA+E;AAC/E,wBAAsB,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,GAAE,OAAO,KAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CA6B3F"}

package/dist/robots.js

@@ -0,0 +1,111 @@
+import { DEFAULT_USER_AGENT } from './user-agent.js';
+/** Normalize UTF-8 and percent-encoded unreserved octets before comparison. */
+function normalize(path) {
+    return path.replace(/\P{ASCII}/gu, (char) => encodeURIComponent(char)).replace(/%[\da-f]{2}/gi, (encoded) => {
+        const char = String.fromCharCode(Number.parseInt(encoded.slice(1), 16));
+        return /[a-z\d._~-]/i.test(char) ? char : encoded.toUpperCase();
+    });
+}
+/** Parse and combine all groups matching ChatCopsBot, falling back to '*'. */
+export function parseRobots(text) {
+    const groups = [];
+    const sitemaps = [];
+    let group;
+    let directives = false;
+    for (const line of text.replace(/^\uFEFF/, '').split(/\r?\n/)) {
+        const clean = line.split('#')[0].trim();
+        const colon = clean.indexOf(':');
+        if (colon < 0)
+            continue;
+        const key = clean.slice(0, colon).trim().toLowerCase();
+        const value = clean.slice(colon + 1).trim();
+        if (key === 'sitemap') {
+            if (value)
+                sitemaps.push(value);
+        }
+        else if (key === 'user-agent') {
+            if (!value)
+                continue;
+            if (!group || directives) {
+                group = { agents: [], rules: [], delays: [] };
+                groups.push(group);
+                directives = false;
+            }
+            group.agents.push(value.toLowerCase());
+        }
+        else if (group && (key === 'allow' || key === 'disallow' || key === 'crawl-delay')) {
+            directives = true;
+            if (key === 'crawl-delay') {
+                const seconds = Number(value);
+                if (value && Number.isFinite(seconds) && seconds >= 0)
+                    group.delays.push(seconds);
+            }
+            else if (value.startsWith('/'))
+                group.rules.push({ allow: key === 'allow', path: normalize(value) });
+        }
+    }
+    const specific = groups.filter((entry) => entry.agents.includes('chatcopsbot'));
+    const selected = specific.length ? specific : groups.filter((entry) => entry.agents.includes('*'));
+    const rules = selected.flatMap((entry) => entry.rules).map((rule) => {
+        const anchored = rule.path.endsWith('$');
+        const path = anchored ? rule.path.slice(0, -1) : rule.path;
+        const pattern = path.split('*').map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('.*');
+        return { ...rule, length: Buffer.byteLength(rule.path), test: new RegExp(`^${pattern}${anchored ? '$' : ''}`) };
+    });
+    const delays = selected.flatMap((entry) => entry.delays);
+    return {
+        sitemaps,
+        ...(delays.length ? { crawlDelay: Math.max(...delays) } : {}),
+        allowed(url) {
+            const parsed = new URL(url);
+            if (parsed.pathname === '/robots.txt')
+                return true;
+            const path = normalize(parsed.pathname + parsed.search);
+            let longest = -1;
+            let allow = true;
+            for (const rule of rules) {
+                if (rule.test.test(path) && (rule.length > longest || (rule.length === longest && rule.allow))) {
+                    longest = rule.length;
+                    allow = rule.allow;
+                }
+            }
+            return allow;
+        },
+    };
+}
+/** Fetch a policy; callers cache this promise once per origin for each run. */
+export async function robotsFor(origin, send = fetch) {
+    const fallback = (allowed, unavailable) => ({
+        allowed: () => allowed,
+        sitemaps: [],
+        ...(unavailable ? { unavailable } : {}),
+    });
+    let url = new URL('/robots.txt', origin).href;
+    try {
+        for (let redirects = 0; redirects <= 5; redirects++) {
+            const response = await send(url, {
+                headers: { 'user-agent': DEFAULT_USER_AGENT },
+                redirect: 'manual',
+                signal: AbortSignal.timeout(30_000),
+            });
+            if (response.status >= 400 && response.status < 500)
+                return fallback(true);
+            if (response.status >= 500)
+                return fallback(false, `HTTP ${response.status}`);
+            if ([301, 302, 303, 307, 308].includes(response.status)) {
+                const location = response.headers.get('location');
+                await response.body?.cancel();
+                if (!location || redirects === 5)
+                    return fallback(false, !location ? 'redirect without location' : 'too many redirects');
+                url = new URL(location, url).href;
+                continue;
+            }
+            return response.ok ? parseRobots(await response.text()) : fallback(false, `HTTP ${response.status}`);
+        }
+    }
+    catch {
+        // Network failures (including timeouts) mean the policy is unreachable.
+    }
+    return fallback(false, 'unreachable');
+}
+//# sourceMappingURL=robots.js.map

package/dist/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAiBrD,+EAA+E;AAC/E,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,OAAO,EAAE,EAAE;QAC1G,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QACxE,OAAO,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,KAAwB,CAAC;IAC7B,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,GAAG,CAAC;YAAE,SAAS;QACxB,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC5C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,IAAI,KAAK;gBAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClC,CAAC;aAAM,IAAI,GAAG,KAAK,YAAY,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,IAAI,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;gBACzB,KAAK,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;gBAC9C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,UAAU,GAAG,KAAK,CAAC;YACrB,CAAC;YACD,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC;QACzC,CAAC;aAAM,IAAI,KAAK,IAAI,CAAC,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,UAAU,IAAI,GAAG,KAAK,aAAa,CAAC,EAAE,CAAC;YACrF,UAAU,GAAG,IAAI,CAAC;YAClB,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC9B,IAAI,KAAK,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC;oBAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACpF,CAAC;iBAAM,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC;gBAAE,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,KAAK,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC;IAChF,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IACnG,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtG,OAAO,EAAE,GAAG,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAClH,CAAC,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACzD,OAAO;QACL,QAAQ;QACR,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,OAAO,CAAC,GAAG;YACT,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,aAAa;gBAAE,OAAO,IAAI,CAAC;YACnD,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YACxD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YACjB,IAAI,KAAK,GAAG,IAAI,CAAC;YACjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBAC/F,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;oBACtB,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;gBACrB,CAAC;YACH,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;KACF,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,MAAc,EAAE,OAAqB,KAAK;IACxE,MAAM,QAAQ,GAAG,CAAC,OAAgB,EAAE,WAAoB,EAAU,EAAE,CAAC,CAAC;QACpE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO;QACtB,QAAQ,EAAE,EAAE;QACZ,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxC,CAAC,CAAC;IACH,IAAI,GAAG,GAAG,IAAI,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,IAAI,CAAC;IAC9C,IAAI,CAAC;QACH,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,IAAI,CAAC,EAAE,SAAS,EAAE,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE;gBAC/B,OAAO,EAAE,EAAE,YAAY,EAAE,kBAAkB,EAAE;gBAC7C,QAAQ,EAAE,QAAQ;gBAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACpC,CAAC,CAAC;YACH,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG;gBAAE,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC3E,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG;gBAAE,OAAO,QAAQ,CAAC,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9E,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAClD,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC;gBAC9B,IAAI,CAAC,QAAQ,IAAI,SAAS,KAAK,CAAC;oBAAE,OAAO,QAAQ,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC;gBACzH,GAAG,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC;gBAClC,SAAS;YACX,CAAC;YACD,OAAO,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACvG,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,wEAAwE;IAC1E,CAAC;IACD,OAAO,QAAQ,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;AACxC,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,113 @@
+import type { Embedder, KeyValueStore, MetadataValue, VectorStore } from '@chatcops/core';
+export type IngestSource = 
+/** A sitemap, a sitemap index, or a gzipped one (`.xml.gz`). */
+{
+    type: 'sitemap';
+    url: string;
+}
+/** Pages listed by hand. */
+ | {
+    type: 'urls';
+    urls: string[];
+}
+/** Local `.md`, `.mdx` and `.html` files, published under `baseUrl`. */
+ | {
+    type: 'directory';
+    path: string;
+    baseUrl: string;
+};
+export interface IngestConfig {
+    /** Names the index, so a new one can be built next to the old. Default `default`. */
+    index?: string;
+    sources: IngestSource[];
+    /** Only URLs matching one of these globs (`*` within a path segment, `**` across them). */
+    include?: string[];
+    /** URLs matching any of these are skipped. */
+    exclude?: string[];
+    /** A CSS selector for the content, by URL glob, for pages where automatic extraction misses it. */
+    selectors?: Record<string, string>;
+    /** Default `ChatCopsBot/<version> (+https://chat.codercops.com/bot)`. */
+    userAgent?: string;
+    /** Obey robots.txt and crawl-delay for network sources. Default true. */
+    respectRobots?: boolean;
+    /** Index at most this many pages (after include/exclude), for a quick look at a big site. */
+    maxPages?: number;
+    /** Pages fetched at once. Default 4. */
+    concurrency?: number;
+    /** Per page. Default 30 seconds. */
+    timeoutMs?: number;
+    chunk?: {
+        targetTokens?: number;
+        overlapTokens?: number;
+    };
+    embedder: Embedder;
+    store: VectorStore;
+    /** Remembers what was indexed, so unchanged pages aren't embedded again. */
+    manifest: KeyValueStore;
+    /** Extra metadata for every chunk of a page, such as `{ collection: 'docs' }`. */
+    metadata?: (doc: ExtractedDoc) => Record<string, MetadataValue>;
+    /** Discover, fetch, extract and chunk, but write nothing and embed nothing. */
+    dryRun?: boolean;
+    /** Re-embed every page, even unchanged ones. */
+    force?: boolean;
+    /**
+     * Remove pages that are in the index but no longer discovered. Skipped when
+     * discovery or robots.txt errors prevent verifying the site's pages.
+     */
+    prune?: boolean;
+    /** Progress, one event per page and step. */
+    onEvent?: (event: IngestEvent) => void;
+    /** For tests and proxies. */
+    fetch?: typeof fetch;
+}
+export interface DiscoveredPage {
+    url: string;
+    /** For directory sources: the file to read instead of fetching. */
+    file?: string;
+}
+export interface ExtractedDoc {
+    url: string;
+    title?: string;
+    /** From `<html lang>`, such as `de` or `pt-BR`. */
+    lang?: string;
+    description?: string;
+    markdown: string;
+}
+export interface Chunk {
+    /** 0-based position in the page. */
+    index: number;
+    /** What's embedded and stored: the heading path, then the text. */
+    text: string;
+    /** The headings above this chunk, such as `Pricing > Enterprise`. */
+    section?: string;
+}
+export type IngestEvent = {
+    type: 'discovered';
+    pages: number;
+    errors: string[];
+} | {
+    type: 'page';
+    url: string;
+    status: 'indexed' | 'unchanged' | 'skipped' | 'failed';
+    chunks?: number;
+    reason?: string;
+} | {
+    type: 'pruned';
+    url: string;
+};
+export interface IngestStats {
+    discovered: number;
+    indexed: number;
+    unchanged: number;
+    skipped: number;
+    failed: number;
+    pruned: number;
+    chunks: number;
+    /** Texts sent to the embedder. 0 on a re-run where nothing changed. */
+    embedded: number;
+    /** Discovery problems and unavailable robots.txt policies, once per origin. */
+    errors: string[];
+    /** Why pruning didn't run, when it was asked for. */
+    pruneSkipped?: string;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE1F,MAAM,MAAM,YAAY;AACtB,gEAAgE;AAC9D;IAAE,IAAI,EAAE,SAAS,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE;AAClC,4BAA4B;GAC1B;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,EAAE,CAAA;CAAE;AAClC,wEAAwE;GACtE;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzD,MAAM,WAAW,YAAY;IAC3B,qFAAqF;IACrF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,2FAA2F;IAC3F,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,8CAA8C;IAC9C,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,mGAAmG;IACnG,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,yEAAyE;IACzE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yEAAyE;IACzE,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,6FAA6F;IAC7F,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wCAAwC;IACxC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oCAAoC;IACpC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,aAAa,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1D,QAAQ,EAAE,QAAQ,CAAC;IACnB,KAAK,EAAE,WAAW,CAAC;IACnB,4EAA4E;IAC5E,QAAQ,EAAE,aAAa,CAAC;IACxB,kFAAkF;IAClF,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,YAAY,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAChE,+EAA+E;IAC/E,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,gDAAgD;IAChD,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;OAGG;IACH,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;IACvC,6BAA6B;IAC7B,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,EAAE,MAAM,CAAC;IACZ,mEAAmE;IACnE,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,mDAAmD;IACnD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,KAAK;IACpB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,MAAM,WAAW,GACnB;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,GACvD;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,SAAS,GAAG,WAAW,GAAG,SAAS,GAAG,QAAQ,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GACvH;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAEpC,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,uEAAuE;IACvE,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB"}

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/dist/user-agent.d.ts

@@ -0,0 +1,2 @@
+export declare const DEFAULT_USER_AGENT = "ChatCopsBot/0.1 (+https://chat.codercops.com/bot)";
+//# sourceMappingURL=user-agent.d.ts.map

package/dist/user-agent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"user-agent.d.ts","sourceRoot":"","sources":["../src/user-agent.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,kBAAkB,sDAAsD,CAAC"}

package/dist/user-agent.js

@@ -0,0 +1,2 @@
+export const DEFAULT_USER_AGENT = 'ChatCopsBot/0.1 (+https://chat.codercops.com/bot)';
+//# sourceMappingURL=user-agent.js.map

package/dist/user-agent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"user-agent.js","sourceRoot":"","sources":["../src/user-agent.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,kBAAkB,GAAG,mDAAmD,CAAC"}