@noodleseed/one 0.126.0 → 0.127.1

package/node_modules/@noodle-borg/knowledge-crawl/dist/html-text.js

@@ -0,0 +1,62 @@
+/**
+ * Deliberately small HTML→text extraction for a lexical index: strip non-content blocks and
+ * markup, decode the common entities, collapse whitespace. Fetched pages are untrusted input —
+ * this reduces them to bounded plain text and nothing here evaluates or resolves anything.
+ */
+const NON_CONTENT_BLOCKS = /<(script|style|noscript|template|svg)\b[\s\S]*?<\/\1\s*>/gi;
+const BLOCK_BREAK = /<\/?(p|div|li|ul|ol|table|tr|td|th|h[1-6]|br|hr|section|article|header|footer|nav|blockquote|pre)\b[^>]*>/gi;
+const NAMED_ENTITIES = {
+    amp: '&',
+    lt: '<',
+    gt: '>',
+    quot: '"',
+    apos: "'",
+    nbsp: ' ',
+    mdash: '—',
+    ndash: '–',
+    hellip: '…',
+    rsquo: '’',
+    lsquo: '‘',
+    rdquo: '”',
+    ldquo: '“',
+    copy: '©',
+};
+function decodeEntities(text) {
+    return text
+        .replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16)))
+        .replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number.parseInt(dec, 10)))
+        .replace(/&([a-z]+);/gi, (match, name) => NAMED_ENTITIES[name.toLowerCase()] ?? match);
+}
+export function extractHtmlText(html) {
+    const withoutBlocks = html.replace(NON_CONTENT_BLOCKS, ' ');
+    const withBreaks = withoutBlocks.replace(BLOCK_BREAK, ' ');
+    const withoutTags = withBreaks.replace(/<[^>]+>/g, '');
+    return decodeEntities(withoutTags).replace(/\s+/g, ' ').trim();
+}
+/**
+ * The content-extraction seam: (content type, bytes) → indexable `{ html, title, text }`, or
+ * `undefined` for media we cannot yet extract. Future formats — PDF is the named next candidate
+ * (ADR 0202 deferred triggers) — become one new case here; everything downstream (sealing,
+ * revisions, indexing, fusion, citations) is format-agnostic.
+ */
+export function extractPageContent(contentType, bytes) {
+    if (!contentType.startsWith('text/html'))
+        return undefined;
+    const html = new TextDecoder().decode(bytes);
+    return { html, title: extractHtmlTitle(html), text: extractHtmlText(html) };
+}
+export function extractHtmlTitle(html) {
+    const title = /<title[^>]*>([\s\S]*?)<\/title\s*>/i.exec(html)?.[1];
+    if (title !== undefined && title.trim() !== '')
+        return decodeEntities(title).replace(/\s+/g, ' ').trim();
+    const heading = /<h1[^>]*>([\s\S]*?)<\/h1\s*>/i.exec(html)?.[1];
+    if (heading !== undefined) {
+        const text = decodeEntities(heading.replace(/<[^>]+>/g, ''))
+            .replace(/\s+/g, ' ')
+            .trim();
+        if (text !== '')
+            return text;
+    }
+    return '';
+}
+//# sourceMappingURL=html-text.js.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/html-text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-text.js","sourceRoot":"","sources":["../src/html-text.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,kBAAkB,GAAG,4DAA4D,CAAC;AACxF,MAAM,WAAW,GACf,6GAA6G,CAAC;AAEhH,MAAM,cAAc,GAAqC;IACvD,GAAG,EAAE,GAAG;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,GAAG;IACP,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,MAAM,EAAE,GAAG;IACX,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;CACV,CAAC;AAEF,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,OAAO,CAAC,mBAAmB,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE,CAC/C,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAC/C;SACA,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,EAAE,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,CAAC;SACxF,OAAO,CAAC,cAAc,EAAE,CAAC,KAAK,EAAE,IAAY,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,KAAK,CAAC,CAAC;AACnG,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;IAC3D,MAAM,WAAW,GAAG,UAAU,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACvD,OAAO,cAAc,CAAC,WAAW,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACjE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAChC,WAAmB,EACnB,KAAkB;IAElB,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC;QAAE,OAAO,SAAS,CAAC;IAC3D,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,gBAAgB,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC3C,MAAM,KAAK,GAAG,qCAAqC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACpE,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;QAC5C,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3D,MAAM,OAAO,GAAG,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAChE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;aACzD,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;aACpB,IAAI,EAAE,CAAC;QACV,IAAI,IAAI,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC;IAC/B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/index.d.ts

@@ -0,0 +1,5 @@
+export { FirecrawlSiteFetcher, type FirecrawlSiteFetcherOptions } from './firecrawl.js';
+export { FirstPartySiteFetcher, type FirstPartySiteFetcherOptions } from './first-party.js';
+export type { CrawlPage, SiteCrawlRequest, SiteFetcher } from './ports.js';
+export { TavilySiteFetcher, type TavilySiteFetcherOptions } from './tavily.js';
+//# sourceMappingURL=index.d.ts.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,KAAK,2BAA2B,EAAE,MAAM,gBAAgB,CAAC;AACxF,OAAO,EAAE,qBAAqB,EAAE,KAAK,4BAA4B,EAAE,MAAM,kBAAkB,CAAC;AAC5F,YAAY,EAAE,SAAS,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,KAAK,wBAAwB,EAAE,MAAM,aAAa,CAAC"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/index.js

@@ -0,0 +1,4 @@
+export { FirecrawlSiteFetcher } from './firecrawl.js';
+export { FirstPartySiteFetcher } from './first-party.js';
+export { TavilySiteFetcher } from './tavily.js';
+//# sourceMappingURL=index.js.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAoC,MAAM,gBAAgB,CAAC;AACxF,OAAO,EAAE,qBAAqB,EAAqC,MAAM,kBAAkB,CAAC;AAE5F,OAAO,EAAE,iBAAiB,EAAiC,MAAM,aAAa,CAAC"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/origin.d.ts

@@ -0,0 +1,3 @@
+export declare function parseOrigin(origin: string): URL;
+export declare function safeUrl(href: string, base?: string): URL | undefined;
+//# sourceMappingURL=origin.d.ts.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/origin.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"origin.d.ts","sourceRoot":"","sources":["../src/origin.ts"],"names":[],"mappings":"AAIA,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,GAAG,CAe/C;AAED,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,GAAG,GAAG,SAAS,CAOpE"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/origin.js

@@ -0,0 +1,30 @@
+/** Shared origin/URL validation for every fetcher (managed and BYO). */
+import { isIP } from 'node:net';
+import { KnowledgeError } from '@noodle-borg/knowledge';
+export function parseOrigin(origin) {
+    let url;
+    try {
+        url = new URL(origin);
+    }
+    catch {
+        throw new KnowledgeError('request', `site origin ${origin} is not a valid URL`);
+    }
+    if (url.protocol !== 'https:') {
+        throw new KnowledgeError('request', 'site origin must use https');
+    }
+    if (isIP(url.hostname.replace(/^\[|\]$/g, '')) !== 0) {
+        // A literal-IP host has no DNS step, which would bypass the socket-level SSRF guard.
+        throw new KnowledgeError('request', 'site origin must be a DNS hostname, not an IP address');
+    }
+    return url;
+}
+export function safeUrl(href, base) {
+    try {
+        const url = base === undefined ? new URL(href) : new URL(href, base);
+        return url.protocol === 'https:' ? url : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+//# sourceMappingURL=origin.js.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/origin.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"origin.js","sourceRoot":"","sources":["../src/origin.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAExD,MAAM,UAAU,WAAW,CAAC,MAAc;IACxC,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,cAAc,CAAC,SAAS,EAAE,eAAe,MAAM,qBAAqB,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,cAAc,CAAC,SAAS,EAAE,4BAA4B,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,qFAAqF;QACrF,MAAM,IAAI,cAAc,CAAC,SAAS,EAAE,uDAAuD,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAAY,EAAE,IAAa;IACjD,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrE,OAAO,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/ports.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Provider-neutral site crawling port (ADR 0202 amendment 2026-08-18): one bounded call in, a
+ * deterministic ordered list of plain-text pages out. Every fetcher — the managed first-party
+ * crawler and the BYO provider adapters — satisfies this contract and its shared tests; provider
+ * concepts (job ids, credits, raw payloads) never cross it.
+ */
+export interface SiteCrawlRequest {
+    /** Exact HTTPS origin, no path. Literal-IP hosts are rejected (they bypass the DNS SSRF guard). */
+    readonly origin: string;
+    /** Path globs a page must match to be indexed; discovery may touch only these plus the root. */
+    readonly include: readonly string[];
+    readonly maxPages: number;
+    readonly maxPageBytes: number;
+}
+export interface CrawlPage {
+    readonly url: string;
+    readonly title: string;
+    readonly text: string;
+}
+export interface SiteFetcher {
+    /**
+     * Crawl the site within the request bounds. Returns pages sorted by URL; individual page
+     * failures are skipped, a total provider failure throws a `KnowledgeError` with an
+     * attributable layer and no content.
+     */
+    fetchSite(request: SiteCrawlRequest): Promise<readonly CrawlPage[]>;
+}
+//# sourceMappingURL=ports.d.ts.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/ports.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ports.d.ts","sourceRoot":"","sources":["../src/ports.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,mGAAmG;IACnG,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,gGAAgG;IAChG,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,WAAW;IAC1B;;;;OAIG;IACH,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,SAAS,EAAE,CAAC,CAAC;CACrE"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/ports.js

@@ -0,0 +1,8 @@
+/**
+ * Provider-neutral site crawling port (ADR 0202 amendment 2026-08-18): one bounded call in, a
+ * deterministic ordered list of plain-text pages out. Every fetcher — the managed first-party
+ * crawler and the BYO provider adapters — satisfies this contract and its shared tests; provider
+ * concepts (job ids, credits, raw payloads) never cross it.
+ */
+export {};
+//# sourceMappingURL=ports.js.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/ports.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ports.js","sourceRoot":"","sources":["../src/ports.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/robots.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Minimal robots.txt evaluation for the managed crawler: `*` user-agent groups only, with the
+ * de-facto longest-match precedence between Allow and Disallow (Allow wins an exact-length tie),
+ * plus global `Sitemap:` collection. Deliberately dependency-free; if richer semantics are ever
+ * needed, verify the dependency license first (root rules).
+ */
+export interface Robots {
+    isAllowed(path: string): boolean;
+    readonly sitemaps: readonly string[];
+}
+export declare function parseRobots(text: string): Robots;
+//# sourceMappingURL=robots.d.ts.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/robots.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.d.ts","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,MAAM,WAAW,MAAM;IACrB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC;IACjC,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAmDhD"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/robots.js

@@ -0,0 +1,64 @@
+/**
+ * Minimal robots.txt evaluation for the managed crawler: `*` user-agent groups only, with the
+ * de-facto longest-match precedence between Allow and Disallow (Allow wins an exact-length tie),
+ * plus global `Sitemap:` collection. Deliberately dependency-free; if richer semantics are ever
+ * needed, verify the dependency license first (root rules).
+ */
+export function parseRobots(text) {
+    const rules = [];
+    const sitemaps = [];
+    let inStarGroup = false;
+    let groupHasAgents = false;
+    for (const rawLine of text.split(/\r?\n/)) {
+        const line = rawLine.replace(/#.*$/, '').trim();
+        if (line === '')
+            continue;
+        const separator = line.indexOf(':');
+        if (separator === -1)
+            continue;
+        const field = line.slice(0, separator).trim().toLowerCase();
+        const value = line.slice(separator + 1).trim();
+        if (field === 'sitemap') {
+            if (value !== '')
+                sitemaps.push(value);
+            continue;
+        }
+        if (field === 'user-agent') {
+            // Consecutive user-agent lines share one group; a rule line closes the agent list, so a
+            // user-agent line after rules starts a fresh group.
+            if (groupHasAgents) {
+                inStarGroup = false;
+                groupHasAgents = false;
+            }
+            if (value === '*')
+                inStarGroup = true;
+            continue;
+        }
+        if (field === 'allow' || field === 'disallow') {
+            groupHasAgents = true;
+            if (!inStarGroup)
+                continue;
+            // A bare "Disallow:" means allow-all per the de-facto standard.
+            if (value === '')
+                continue;
+            rules.push({ allow: field === 'allow', path: value });
+        }
+    }
+    return {
+        sitemaps,
+        isAllowed(path) {
+            let best;
+            for (const rule of rules) {
+                if (!path.startsWith(rule.path))
+                    continue;
+                if (best === undefined ||
+                    rule.path.length > best.path.length ||
+                    (rule.path.length === best.path.length && rule.allow)) {
+                    best = rule;
+                }
+            }
+            return best?.allow ?? true;
+        },
+    };
+}
+//# sourceMappingURL=robots.js.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/robots.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"robots.js","sourceRoot":"","sources":["../src/robots.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAYH,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,IAAI,IAAI,KAAK,EAAE;YAAE,SAAS;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QACpC,IAAI,SAAS,KAAK,CAAC,CAAC;YAAE,SAAS;QAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC5D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC/C,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,IAAI,KAAK,KAAK,EAAE;gBAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACvC,SAAS;QACX,CAAC;QACD,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YAC3B,wFAAwF;YACxF,oDAAoD;YACpD,IAAI,cAAc,EAAE,CAAC;gBACnB,WAAW,GAAG,KAAK,CAAC;gBACpB,cAAc,GAAG,KAAK,CAAC;YACzB,CAAC;YACD,IAAI,KAAK,KAAK,GAAG;gBAAE,WAAW,GAAG,IAAI,CAAC;YACtC,SAAS;QACX,CAAC;QACD,IAAI,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YAC9C,cAAc,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC,WAAW;gBAAE,SAAS;YAC3B,gEAAgE;YAChE,IAAI,KAAK,KAAK,EAAE;gBAAE,SAAS;YAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,KAAK,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IACD,OAAO;QACL,QAAQ;QACR,SAAS,CAAC,IAAY;YACpB,IAAI,IAA4B,CAAC;YACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS;gBAC1C,IACE,IAAI,KAAK,SAAS;oBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM;oBACnC,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,EACrD,CAAC;oBACD,IAAI,GAAG,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;YACD,OAAO,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC;QAC7B,CAAC;KACF,CAAC;AACJ,CAAC"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/sitemap.d.ts

@@ -0,0 +1,3 @@
+/** Bounded `<loc>` extraction from sitemap XML — locations only, nothing else is read. */
+export declare function parseSitemapLocations(xml: string): readonly string[];
+//# sourceMappingURL=sitemap.d.ts.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/sitemap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.d.ts","sourceRoot":"","sources":["../src/sitemap.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAC1F,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAOpE"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/sitemap.js

@@ -0,0 +1,11 @@
+/** Bounded `<loc>` extraction from sitemap XML — locations only, nothing else is read. */
+export function parseSitemapLocations(xml) {
+    const locations = [];
+    for (const match of xml.matchAll(/<loc>\s*([^<]+?)\s*<\/loc>/gi)) {
+        const location = match[1];
+        if (location !== undefined && location !== '')
+            locations.push(location);
+    }
+    return locations;
+}
+//# sourceMappingURL=sitemap.js.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/sitemap.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sitemap.js","sourceRoot":"","sources":["../src/sitemap.ts"],"names":[],"mappings":"AAAA,0FAA0F;AAC1F,MAAM,UAAU,qBAAqB,CAAC,GAAW;IAC/C,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,8BAA8B,CAAC,EAAE,CAAC;QACjE,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,EAAE;YAAE,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/tavily.d.ts

@@ -0,0 +1,11 @@
+import type { CrawlPage, SiteCrawlRequest, SiteFetcher } from './ports.js';
+export interface TavilySiteFetcherOptions {
+    readonly apiKey: string;
+    readonly fetchImpl?: typeof fetch;
+}
+export declare class TavilySiteFetcher implements SiteFetcher {
+    #private;
+    constructor(options: TavilySiteFetcherOptions);
+    fetchSite(request: SiteCrawlRequest): Promise<readonly CrawlPage[]>;
+}
+//# sourceMappingURL=tavily.d.ts.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/tavily.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tavily.d.ts","sourceRoot":"","sources":["../src/tavily.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAI3E,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;CACnC;AAED,qBAAa,iBAAkB,YAAW,WAAW;;gBAIvC,OAAO,EAAE,wBAAwB;IAKvC,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,SAAS,EAAE,CAAC;CAqC1E"}

package/node_modules/@noodle-borg/knowledge-crawl/dist/tavily.js

@@ -0,0 +1,53 @@
+/**
+ * BYO Tavily fetcher (ADR 0202 amendment 2026-08-18): the customer's own Tavily account crawls
+ * the site through the synchronous hosted REST API. Wire shape transcribed from the official API
+ * reference 2026-08-18: `POST https://api.tavily.com/crawl` with Bearer auth and
+ * `{ url, limit, allow_external, format, extract_depth }` → `{ results: [{ url, raw_content }] }`.
+ * Crawled pages carry no separate title field, so the URL stands in. Errors are attributable and
+ * content-free — provider bodies and the API key never cross the port.
+ */
+import { KnowledgeError } from '@noodle-borg/knowledge';
+import { boundPages } from './bounds.js';
+import { parseOrigin } from './origin.js';
+const API = 'https://api.tavily.com/crawl';
+export class TavilySiteFetcher {
+    #apiKey;
+    #fetch;
+    constructor(options) {
+        this.#apiKey = options.apiKey;
+        this.#fetch = options.fetchImpl ?? fetch;
+    }
+    async fetchSite(request) {
+        const origin = parseOrigin(request.origin);
+        let response;
+        try {
+            response = await this.#fetch(API, {
+                method: 'POST',
+                headers: {
+                    authorization: `Bearer ${this.#apiKey}`,
+                    'content-type': 'application/json',
+                },
+                body: JSON.stringify({
+                    url: origin.origin,
+                    limit: request.maxPages,
+                    allow_external: false,
+                    format: 'text',
+                    extract_depth: 'basic',
+                }),
+            });
+        }
+        catch {
+            throw new KnowledgeError('provider', 'tavily crawl request failed');
+        }
+        if (!response.ok) {
+            throw new KnowledgeError('provider', `tavily crawl returned HTTP ${response.status}`);
+        }
+        const body = (await response.json().catch(() => ({})));
+        return boundPages((body.results ?? []).map((result) => ({
+            url: result.url ?? '',
+            title: '',
+            text: result.raw_content ?? '',
+        })), request, origin);
+    }
+}
+//# sourceMappingURL=tavily.js.map

package/node_modules/@noodle-borg/knowledge-crawl/dist/tavily.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tavily.js","sourceRoot":"","sources":["../src/tavily.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAG1C,MAAM,GAAG,GAAG,8BAA8B,CAAC;AAO3C,MAAM,OAAO,iBAAiB;IACnB,OAAO,CAAS;IAChB,MAAM,CAAe;IAE9B,YAAY,OAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,IAAI,KAAK,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAAyB;QACvC,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE;gBAChC,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,aAAa,EAAE,UAAU,IAAI,CAAC,OAAO,EAAE;oBACvC,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACnB,GAAG,EAAE,MAAM,CAAC,MAAM;oBAClB,KAAK,EAAE,OAAO,CAAC,QAAQ;oBACvB,cAAc,EAAE,KAAK;oBACrB,MAAM,EAAE,MAAM;oBACd,aAAa,EAAE,OAAO;iBACvB,CAAC;aACH,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,6BAA6B,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,cAAc,CAAC,UAAU,EAAE,8BAA8B,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAEpD,CAAC;QACF,OAAO,UAAU,CACf,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE;YACrB,KAAK,EAAE,EAAE;YACT,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,EAAE;SAC/B,CAAC,CAAC,EACH,OAAO,EACP,MAAM,CACP,CAAC;IACJ,CAAC;CACF"}

package/node_modules/@noodle-borg/knowledge-crawl/package.json

@@ -0,0 +1,32 @@
+{
+  "name": "@noodle-borg/knowledge-crawl",
+  "version": "0.0.0",
+  "private": true,
+  "license": "Apache-2.0",
+  "type": "module",
+  "engines": {
+    "node": ">=24"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit"
+  },
+  "dependencies": {
+    "@noodle-borg/connector-http": "0.0.0",
+    "@noodle-borg/knowledge": "0.0.0"
+  },
+  "devDependencies": {
+    "@types/node": "^24.13.1"
+  }
+}

package/node_modules/@noodle-borg/knowledge-operations/dist/crawl-lifecycle.d.ts

@@ -0,0 +1,71 @@
+import { type SearchBudgetCeilings, type SearchBudgetStore, type SearchHit, type SearchRequest } from '@noodle-borg/knowledge';
+import type { SiteFetcher } from '@noodle-borg/knowledge-crawl';
+import type { KnowledgeDeployHooks } from './publication.js';
+import type { KnowledgeTenantRef } from './routes.js';
+/** The reserved revision namespace for a component's crawled site corpus. */
+export declare function siteCorpusName(componentName: string): string;
+export type CrawlStatus = 'not_started' | 'in_progress' | 'completed' | 'failed';
+export interface SiteCrawlState {
+    readonly status: CrawlStatus;
+    readonly lastCompletedAt?: number;
+    readonly lastError?: string;
+    readonly pagesIndexed: number;
+    readonly nextRefreshAt?: number;
+}
+export interface CrawlStateStore {
+    get(tenant: KnowledgeTenantRef, componentName: string): Promise<SiteCrawlState | undefined>;
+    put(tenant: KnowledgeTenantRef, componentName: string, state: SiteCrawlState): Promise<void>;
+}
+export declare class InMemoryCrawlStateStore implements CrawlStateStore {
+    private readonly states;
+    get(tenant: KnowledgeTenantRef, componentName: string): Promise<SiteCrawlState | undefined>;
+    put(tenant: KnowledgeTenantRef, componentName: string, state: SiteCrawlState): Promise<void>;
+}
+/** A provider config reference by name (variable()/secret() doctrine) — never a value. */
+export interface ProviderConfigRef {
+    readonly kind: 'variable' | 'secret';
+    readonly name: string;
+}
+/** The component facts a crawl needs; a structural subset of the compiled component. */
+export interface CrawlableComponent {
+    readonly name: string;
+    readonly sites: readonly {
+        readonly origin: string;
+        readonly include: readonly string[];
+        readonly refreshMinutes?: number | undefined;
+    }[];
+    readonly crawler?: {
+        readonly provider: 'firecrawl' | 'tavily';
+        readonly config: {
+            readonly apiKey: ProviderConfigRef;
+        };
+    } | undefined;
+    readonly index?: {
+        readonly provider: 'algolia' | 'meilisearch';
+        readonly config: Readonly<Record<string, ProviderConfigRef>>;
+    } | undefined;
+}
+export interface CrawlDeps {
+    readonly hooks: KnowledgeDeployHooks;
+    readonly crawlState: CrawlStateStore;
+    readonly fetcherFor: (tenant: KnowledgeTenantRef, component: CrawlableComponent) => SiteFetcher;
+    readonly budget?: {
+        readonly store: SearchBudgetStore;
+        readonly ceilings: (tenant: KnowledgeTenantRef) => Promise<SearchBudgetCeilings>;
+    };
+    readonly now?: () => number;
+}
+/**
+ * Crawl one component's sites and publish the site revision. Never throws: the outcome —
+ * including budget refusal and provider failure — lands in the crawl state, and the previous
+ * corpus keeps serving on any failure.
+ */
+export declare function runComponentCrawl(deps: CrawlDeps, tenant: KnowledgeTenantRef, component: CrawlableComponent): Promise<SiteCrawlState>;
+/** Crawl every served component whose refresh is due (or that has never crawled). */
+export declare function runDueCrawls(deps: CrawlDeps, servedComponents: () => Promise<readonly {
+    tenant: KnowledgeTenantRef;
+    component: CrawlableComponent;
+}[]>, at: number): Promise<void>;
+/** Serve the crawled corpus: the bundled index over the site revision, hits remapped to `site`. */
+export declare function searchSiteCorpus(hooks: KnowledgeDeployHooks, tenant: KnowledgeTenantRef, componentName: string, request: SearchRequest): Promise<readonly SearchHit[]>;
+//# sourceMappingURL=crawl-lifecycle.d.ts.map

package/node_modules/@noodle-borg/knowledge-operations/dist/crawl-lifecycle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl-lifecycle.d.ts","sourceRoot":"","sources":["../src/crawl-lifecycle.ts"],"names":[],"mappings":"AAUA,OAAO,EAEL,KAAK,oBAAoB,EACzB,KAAK,iBAAiB,EACtB,KAAK,SAAS,EACd,KAAK,aAAa,EAEnB,MAAM,wBAAwB,CAAC;AAMhC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAC7D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAEtD,6EAA6E;AAC7E,wBAAgB,cAAc,CAAC,aAAa,EAAE,MAAM,GAAG,MAAM,CAE5D;AAED,MAAM,MAAM,WAAW,GAAG,aAAa,GAAG,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC;AAEjF,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,CAAC;IAC5F,GAAG,CAAC,MAAM,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9F;AAED,qBAAa,uBAAwB,YAAW,eAAe;IAC7D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqC;IAEtD,GAAG,CACP,MAAM,EAAE,kBAAkB,EAC1B,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAIhC,GAAG,CACP,MAAM,EAAE,kBAAkB,EAC1B,aAAa,EAAE,MAAM,EACrB,KAAK,EAAE,cAAc,GACpB,OAAO,CAAC,IAAI,CAAC;CAGjB;AAMD,0FAA0F;AAC1F,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,QAAQ,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,wFAAwF;AACxF,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,SAAS;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;QACpC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;KAC9C,EAAE,CAAC;IACJ,QAAQ,CAAC,OAAO,CAAC,EACb;QACE,QAAQ,CAAC,QAAQ,EAAE,WAAW,GAAG,QAAQ,CAAC;QAC1C,QAAQ,CAAC,MAAM,EAAE;YAAE,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAA;SAAE,CAAC;KACzD,GACD,SAAS,CAAC;IACd,QAAQ,CAAC,KAAK,CAAC,EACX;QACE,QAAQ,CAAC,QAAQ,EAAE,SAAS,GAAG,aAAa,CAAC;QAC7C,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC,CAAC;KAC9D,GACD,SAAS,CAAC;CACf;AAED,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,KAAK,EAAE,oBAAoB,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,kBAAkB,EAAE,SAAS,EAAE,kBAAkB,KAAK,WAAW,CAAC;IAChG,QAAQ,CAAC,MAAM,CAAC,EAAE;QAChB,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC;QAClC,QAAQ,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,kBAAkB,KAAK,OAAO,CAAC,oBAAoB,CAAC,CAAC;KAClF,CAAC;IACF,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CAC7B;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,SAAS,EACf,MAAM,EAAE,kBAAkB,EAC1B,SAAS,EAAE,kBAAkB,GAC5B,OAAO,CAAC,cAAc,CAAC,CA6FzB;AAED,qFAAqF;AACrF,wBAAsB,YAAY,CAChC,IAAI,EAAE,SAAS,EACf,gBAAgB,EAAE,MAAM,OAAO,CAC7B,SAAS;IAAE,MAAM,EAAE,kBAAkB,CAAC;IAAC,SAAS,EAAE,kBAAkB,CAAA;CAAE,EAAE,CACzE,EACD,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,IAAI,CAAC,CASf;AAED,mGAAmG;AACnG,wBAAsB,gBAAgB,CACpC,KAAK,EAAE,oBAAoB,EAC3B,MAAM,EAAE,kBAAkB,EAC1B,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,SAAS,SAAS,EAAE,CAAC,CAO/B"}

package/node_modules/@noodle-borg/knowledge-operations/dist/crawl-lifecycle.js

@@ -0,0 +1,138 @@
+/**
+ * Site crawl lifecycle (ADR 0202 amendment 2026-08-18): a crawl fetches the component's declared
+ * sites through the selected `SiteFetcher`, meters the page count against the org/app crawl
+ * budget, and publishes the pages as a **site revision** — the reserved `<name>#site` component
+ * namespace in the same revision store and bundled index that serve documents. Site revisions
+ * refresh latest-wins and are never deployment-pinned ("live site never rolls back" survives as
+ * "rollback keeps serving the latest crawl"); a failed crawl keeps the last good corpus serving
+ * and records an attributable, content-free failure in the crawl state.
+ */
+import { createHash } from 'node:crypto';
+import { KnowledgeError, } from '@noodle-borg/knowledge';
+import { DEFAULT_SITE_REFRESH_MINUTES, MAX_CRAWL_PAGE_BYTES, MAX_CRAWL_PAGES_PER_SITE, } from '@noodle-borg/knowledge/limits';
+/** The reserved revision namespace for a component's crawled site corpus. */
+export function siteCorpusName(componentName) {
+    return `${componentName}#site`;
+}
+export class InMemoryCrawlStateStore {
+    states = new Map();
+    async get(tenant, componentName) {
+        return this.states.get(stateKey(tenant, componentName));
+    }
+    async put(tenant, componentName, state) {
+        this.states.set(stateKey(tenant, componentName), state);
+    }
+}
+function stateKey(tenant, componentName) {
+    return `${tenant.org}/${tenant.app}/${tenant.env}#${componentName}`;
+}
+/**
+ * Crawl one component's sites and publish the site revision. Never throws: the outcome —
+ * including budget refusal and provider failure — lands in the crawl state, and the previous
+ * corpus keeps serving on any failure.
+ */
+export async function runComponentCrawl(deps, tenant, component) {
+    const now = deps.now ?? Date.now;
+    if (component.sites.length === 0) {
+        const state = { status: 'not_started', pagesIndexed: 0 };
+        await deps.crawlState.put(tenant, component.name, state);
+        return state;
+    }
+    const previous = await deps.crawlState.get(tenant, component.name);
+    const refreshMinutes = Math.min(...component.sites.map((site) => site.refreshMinutes ?? DEFAULT_SITE_REFRESH_MINUTES));
+    await deps.crawlState.put(tenant, component.name, {
+        status: 'in_progress',
+        pagesIndexed: previous?.pagesIndexed ?? 0,
+        ...(previous?.lastCompletedAt === undefined
+            ? {}
+            : { lastCompletedAt: previous.lastCompletedAt }),
+    });
+    const fail = async (lastError) => {
+        const state = {
+            status: 'failed',
+            lastError,
+            pagesIndexed: previous?.pagesIndexed ?? 0,
+            ...(previous?.lastCompletedAt === undefined
+                ? {}
+                : { lastCompletedAt: previous.lastCompletedAt }),
+            nextRefreshAt: now() + refreshMinutes * 60 * 1000,
+        };
+        await deps.crawlState.put(tenant, component.name, state);
+        return state;
+    };
+    const documents = [];
+    try {
+        const fetcher = deps.fetcherFor(tenant, component);
+        for (const site of component.sites) {
+            const pages = await fetcher.fetchSite({
+                origin: site.origin,
+                include: site.include,
+                maxPages: MAX_CRAWL_PAGES_PER_SITE,
+                maxPageBytes: MAX_CRAWL_PAGE_BYTES,
+            });
+            for (const page of pages) {
+                documents.push({
+                    descriptor: {
+                        path: page.url,
+                        title: page.title,
+                        sha256: createHash('sha256').update(page.text).digest('hex'),
+                        bytes: Buffer.byteLength(page.text, 'utf8'),
+                        sourceUrl: page.url,
+                    },
+                    text: page.text,
+                });
+            }
+        }
+    }
+    catch (error) {
+        // Request-layer messages are our own (content-free by construction — e.g. the exact
+        // `noodle secrets set` fix for an unset BYO key) and stay visible; provider messages may
+        // carry URLs, bodies, or credentials and reduce to the attributable layer alone.
+        if (error instanceof KnowledgeError && error.layer === 'request') {
+            return fail(error.message);
+        }
+        const layer = error instanceof KnowledgeError ? error.layer : 'provider';
+        return fail(`crawl failed at the ${layer} layer`);
+    }
+    if (deps.budget !== undefined && documents.length > 0) {
+        const ceilings = await deps.budget.ceilings(tenant);
+        const decision = await deps.budget.store.consume({ org: tenant.org, app: tenant.app }, documents.length, ceilings);
+        if (!decision.granted)
+            return fail('crawl page budget exhausted for this month');
+    }
+    try {
+        await deps.hooks.publishSiteCorpus({ org: tenant.org, app: tenant.app, env: tenant.env }, siteCorpusName(component.name), documents);
+    }
+    catch {
+        return fail('site corpus publication failed at the store layer');
+    }
+    const state = {
+        status: 'completed',
+        lastCompletedAt: now(),
+        pagesIndexed: documents.length,
+        nextRefreshAt: now() + refreshMinutes * 60 * 1000,
+    };
+    await deps.crawlState.put(tenant, component.name, state);
+    return state;
+}
+/** Crawl every served component whose refresh is due (or that has never crawled). */
+export async function runDueCrawls(deps, servedComponents, at) {
+    for (const { tenant, component } of await servedComponents()) {
+        if (component.sites.length === 0)
+            continue;
+        const state = await deps.crawlState.get(tenant, component.name);
+        if (state?.status === 'in_progress')
+            continue;
+        if (state?.nextRefreshAt !== undefined && state.nextRefreshAt > at)
+            continue;
+        if (state?.status === 'completed' && state.nextRefreshAt === undefined)
+            continue;
+        await runComponentCrawl({ ...deps, now: () => at }, tenant, component);
+    }
+}
+/** Serve the crawled corpus: the bundled index over the site revision, hits remapped to `site`. */
+export async function searchSiteCorpus(hooks, tenant, componentName, request) {
+    const hits = await hooks.searchDocuments({ org: tenant.org, app: tenant.app, env: tenant.env }, siteCorpusName(componentName), request);
+    return hits.map((hit) => ({ ...hit, sourceKind: 'site' }));
+}
+//# sourceMappingURL=crawl-lifecycle.js.map

package/node_modules/@noodle-borg/knowledge-operations/dist/crawl-lifecycle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawl-lifecycle.js","sourceRoot":"","sources":["../src/crawl-lifecycle.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,cAAc,GAMf,MAAM,wBAAwB,CAAC;AAChC,OAAO,EACL,4BAA4B,EAC5B,oBAAoB,EACpB,wBAAwB,GACzB,MAAM,+BAA+B,CAAC;AAKvC,6EAA6E;AAC7E,MAAM,UAAU,cAAc,CAAC,aAAqB;IAClD,OAAO,GAAG,aAAa,OAAO,CAAC;AACjC,CAAC;AAiBD,MAAM,OAAO,uBAAuB;IACjB,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;IAE5D,KAAK,CAAC,GAAG,CACP,MAA0B,EAC1B,aAAqB;QAErB,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,GAAG,CACP,MAA0B,EAC1B,aAAqB,EACrB,KAAqB;QAErB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,aAAa,CAAC,EAAE,KAAK,CAAC,CAAC;IAC1D,CAAC;CACF;AAED,SAAS,QAAQ,CAAC,MAA0B,EAAE,aAAqB;IACjE,OAAO,GAAG,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,GAAG,IAAI,aAAa,EAAE,CAAC;AACtE,CAAC;AAyCD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAe,EACf,MAA0B,EAC1B,SAA6B;IAE7B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACjC,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,MAAM,KAAK,GAAmB,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACzD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;IACnE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAC7B,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,IAAI,4BAA4B,CAAC,CACtF,CAAC;IACF,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE;QAChD,MAAM,EAAE,aAAa;QACrB,YAAY,EAAE,QAAQ,EAAE,YAAY,IAAI,CAAC;QACzC,GAAG,CAAC,QAAQ,EAAE,eAAe,KAAK,SAAS;YACzC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC;KACnD,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,KAAK,EAAE,SAAiB,EAA2B,EAAE;QAChE,MAAM,KAAK,GAAmB;YAC5B,MAAM,EAAE,QAAQ;YAChB,SAAS;YACT,YAAY,EAAE,QAAQ,EAAE,YAAY,IAAI,CAAC;YACzC,GAAG,CAAC,QAAQ,EAAE,eAAe,KAAK,SAAS;gBACzC,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,EAAE,eAAe,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC;YAClD,aAAa,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,EAAE,GAAG,IAAI;SAClD,CAAC;QACF,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QACzD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,SAAS,GAAqB,EAAE,CAAC;IACvC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACnD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC;gBACpC,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,wBAAwB;gBAClC,YAAY,EAAE,oBAAoB;aACnC,CAAC,CAAC;YACH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,SAAS,CAAC,IAAI,CAAC;oBACb,UAAU,EAAE;wBACV,IAAI,EAAE,IAAI,CAAC,GAAG;wBACd,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,MAAM,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;wBAC5D,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;wBAC3C,SAAS,EAAE,IAAI,CAAC,GAAG;qBACpB;oBACD,IAAI,EAAE,IAAI,CAAC,IAAI;iBAChB,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,oFAAoF;QACpF,yFAAyF;QACzF,iFAAiF;QACjF,IAAI,KAAK,YAAY,cAAc,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YACjE,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;QACD,MAAM,KAAK,GAAG,KAAK,YAAY,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;QACzE,OAAO,IAAI,CAAC,uBAAuB,KAAK,QAAQ,CAAC,CAAC;IACpD,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAC9C,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACpC,SAAS,CAAC,MAAM,EAChB,QAAQ,CACT,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,4CAA4C,CAAC,CAAC;IACnF,CAAC;IAED,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAChC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACrD,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,EAC9B,SAAS,CACV,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC,mDAAmD,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,KAAK,GAAmB;QAC5B,MAAM,EAAE,WAAW;QACnB,eAAe,EAAE,GAAG,EAAE;QACtB,YAAY,EAAE,SAAS,CAAC,MAAM;QAC9B,aAAa,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,EAAE,GAAG,IAAI;KAClD,CAAC;IACF,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACzD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,qFAAqF;AACrF,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAe,EACf,gBAEC,EACD,EAAU;IAEV,KAAK,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,MAAM,gBAAgB,EAAE,EAAE,CAAC;QAC7D,IAAI,SAAS,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC3C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QAChE,IAAI,KAAK,EAAE,MAAM,KAAK,aAAa;YAAE,SAAS;QAC9C,IAAI,KAAK,EAAE,aAAa,KAAK,SAAS,IAAI,KAAK,CAAC,aAAa,GAAG,EAAE;YAAE,SAAS;QAC7E,IAAI,KAAK,EAAE,MAAM,KAAK,WAAW,IAAI,KAAK,CAAC,aAAa,KAAK,SAAS;YAAE,SAAS;QACjF,MAAM,iBAAiB,CAAC,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,mGAAmG;AACnG,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,KAA2B,EAC3B,MAA0B,EAC1B,aAAqB,EACrB,OAAsB;IAEtB,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,eAAe,CACtC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,EACrD,cAAc,CAAC,aAAa,CAAC,EAC7B,OAAO,CACR,CAAC;IACF,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,GAAG,EAAE,UAAU,EAAE,MAAe,EAAE,CAAC,CAAC,CAAC;AACtE,CAAC"}