@cavelang/cli 0.27.3 → 0.29.0

package/dist/internal/ingest/run.js

@@ -0,0 +1,350 @@
+/**
+ * The ingestion orchestrator.
+ *
+ * `cave ingest` walks a source list (file globs and http(s) URLs — see
+ * `web.ts`), skips sources whose content is already ingested (digest
+ * provenance claims), batches the rest, composes one prompt per batch,
+ * and drives an *agent* over it:
+ *
+ * - a shell command template (`--agent 'claude -p --mcp-config {mcp-config} …'`)
+ *   for headless Claude Code / Copilot CLI runs — the prompt is piped to
+ *   stdin and written to `{prompt-file}`;
+ * - a JavaScript function (`agent: async (prompt, files) => text`) for
+ *   Claude/Copilot SDK scripts using the library API;
+ * - or no agent at all — `--plan` emits the batches as NDJSON for a
+ *   fully external driver.
+ *
+ * API access for the agent is the MCP server (`{mcp-config}` points at a
+ * generated client configuration for `cave mcp --db …`); `--stdout` mode
+ * instead treats the agent's stdout as CAVE text and stores it here.
+ * Prompts are built lazily per batch, so the injected knowledge context
+ * reflects what earlier batches recorded.
+ */
+import { spawn } from 'node:child_process';
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import * as Canonical from '@cavelang/canonical';
+import { Registry } from '@cavelang/canonical';
+import { open } from '@cavelang/store';
+import { syncDb, syncText } from '@cavelang/cli/sync';
+import * as Files from "./files.js";
+import * as Web from "./web.js";
+import * as Context from "./context.js";
+import * as Prompt from "./prompt.js";
+/** Writes an MCP client configuration pointing at `cave mcp --db <db>`. */
+export const writeMcpConfig = (db, options = {}) => {
+    const dir = options.dir ?? mkdtempSync(join(tmpdir(), 'cave-ingest-'));
+    const server = fileURLToPath(import.meta.resolve('@cavelang/cli/mcp/bin'));
+    const path = join(dir, 'cave-mcp.json');
+    writeFileSync(path, `${JSON.stringify({
+        mcpServers: {
+            cave: {
+                command: process.execPath,
+                args: [server, '--db', resolve(db), ...options.noPrelude === true ? ['--no-prelude'] : []]
+            }
+        }
+    }, undefined, 2)}\n`);
+    return path;
+};
+const promptFiles = (files, embed, cwd) => files.map(file => ({
+    path: file.path,
+    // URL sources carry their extracted text and are always embedded —
+    // the agent has no other way to read them readability-cleaned.
+    ...file.content !== undefined ? { content: file.content } :
+        embed ? { content: readFileSync(resolve(cwd, file.path), 'utf8') } : {}
+}));
+/** Builds the prompt for one batch against the store's *current* state. */
+export const promptFor = (store, files, options) => {
+    const cwd = options.cwd ?? process.cwd();
+    const context = Context.contextFor(store, files.map(file => file.path));
+    return Prompt.buildPrompt({
+        files: promptFiles(files, options.embed === true, cwd),
+        ...options.instructions === undefined ? {} : { instructions: Prompt.readInstructions(options.instructions) },
+        ...context === undefined ? {} : { context },
+        mode: options.mode ?? 'mcp'
+    });
+};
+/** Selects and batches the sources to process — file globs, literal paths and URLs. */
+export const selectBatches = async (store, options) => {
+    const urls = options.patterns.filter(Web.isUrl);
+    const expanded = Files.expand(options.patterns.filter(pattern => !Web.isUrl(pattern)), options.cwd);
+    const paths = [...new Set([...expanded, ...options.files ?? []])].sort();
+    const local = Files.select(store, paths, {
+        force: options.force === true,
+        ...options.cwd === undefined ? {} : { cwd: options.cwd }
+    });
+    const remote = await Web.select(store, urls, {
+        force: options.force === true,
+        ...options.fetchImpl === undefined ? {} : { fetchImpl: options.fetchImpl }
+    });
+    const selection = {
+        files: [...local.files, ...remote.files],
+        skipped: [...local.skipped, ...remote.skipped],
+        failures: remote.failures
+    };
+    return { selection, batches: Files.batch(selection.files, options.batchSize ?? 8) };
+};
+const claimCount = (store) => store.db.prepare('SELECT COUNT(*) AS n FROM cave_claim').get().n;
+/**
+ * POSIX single-quoting for substituted template values — the act/automate
+ * hook convention (spec §25.4): a value always lands as one argument and
+ * is never shell-evaluated, so placeholders are written bare.
+ */
+const shellQuote = (value) => `'${value.replaceAll("'", `'\\''`)}'`;
+/**
+ * Runs one shell agent invocation: `{name}` placeholders substituted into
+ * the template — each value shell-quoted, so paths with spaces or shell
+ * metacharacters stay single arguments — the prompt piped to stdin,
+ * stdout captured. Shared with `@cavelang/cli/eval`, whose agents and judges
+ * follow the same contract.
+ */
+export const runShellAgent = (template, prompt, substitutions, timeoutSeconds, cwd) => {
+    const command = Object.entries(substitutions)
+        .reduce((acc, [name, value]) => acc.replaceAll(`{${name}}`, shellQuote(value)), template);
+    return new Promise(resolvePromise => {
+        let settled = false;
+        const settle = (result) => {
+            if (!settled) {
+                settled = true;
+                resolvePromise(result);
+            }
+        };
+        const child = spawn(command, {
+            shell: true,
+            cwd,
+            timeout: timeoutSeconds * 1000,
+            stdio: ['pipe', 'pipe', 'inherit']
+        });
+        let stdout = '';
+        child.stdout.on('data', chunk => {
+            stdout += String(chunk);
+        });
+        child.on('error', error => settle({ code: null, stdout, error: error.message }));
+        child.on('close', code => settle({ code, stdout }));
+        child.stdin.on('error', () => { });
+        child.stdin.write(prompt);
+        child.stdin.end();
+    });
+};
+/** Extracts CAVE text from stdout-mode agent output (```cave fences win). */
+export const caveTextOf = (output) => {
+    const fences = [...output.matchAll(/```(?:cave)?\n([\s\S]*?)```/g)].map(match => match[1]);
+    return fences.length > 0 ? fences.join('\n') : output;
+};
+/** Runs batches against one mutable store. Strict callers provide a stage. */
+const runMutable = async (options) => {
+    const store = options.store;
+    const cwd = options.cwd ?? process.cwd();
+    const mode = options.mode ?? 'mcp';
+    const policy = options.policy;
+    const timeoutSeconds = options.timeoutSeconds ?? 600;
+    const { selection, batches } = await selectBatches(store, options);
+    const reports = [];
+    const mcpConfig = typeof options.agent === 'string' && options.agent.includes('{mcp-config}') ?
+        writeMcpConfig(options.db, { noPrelude: options.noPrelude === true }) :
+        undefined;
+    const promptDir = mkdtempSync(join(tmpdir(), 'cave-prompt-'));
+    try {
+        // Strict input validation is fail-fast before the first paid call. The
+        // complete source manifest below still marks healthy selected inputs as
+        // not-run and each failed URL as rejected.
+        if (policy === 'strict' && selection.failures.length > 0) {
+            return {
+                policy,
+                applied: false,
+                matched: selection.files.length + selection.skipped.length + selection.failures.length,
+                skipped: selection.skipped,
+                batches: [],
+                sources: [
+                    ...selection.skipped.map(path => ({ path, status: 'skipped', problems: [] })),
+                    ...selection.files.map(file => ({ path: file.path, status: 'not-run', problems: [] })),
+                    ...selection.failures.map(failure => ({
+                        path: failure.path,
+                        status: 'rejected',
+                        problems: [failure.message],
+                        failure: failure.kind,
+                        retryable: failure.retryable,
+                        ...failure.status === undefined ? {} : { httpStatus: failure.status }
+                    }))
+                ],
+                added: 0,
+                failed: selection.failures.length
+            };
+        }
+        for (const [index, files] of batches.entries()) {
+            const prompt = promptFor(store, files, { ...options, mode });
+            const paths = files.map(file => file.path);
+            if (options.agent === undefined) {
+                reports.push({ files: paths, ok: false, added: 0, problems: [], note: 'no agent configured' });
+                if (policy === 'strict')
+                    break;
+                continue;
+            }
+            const before = claimCount(store);
+            let ok;
+            let output;
+            if (typeof options.agent === 'function') {
+                try {
+                    output = await options.agent(prompt, paths, {
+                        db: options.db,
+                        ...mcpConfig === undefined ? {} : { mcpConfig }
+                    });
+                    ok = true;
+                }
+                catch (error) {
+                    output = '';
+                    ok = false;
+                    reports.push({
+                        files: paths, ok, added: claimCount(store) - before, problems: [],
+                        note: error instanceof Error ? error.message : String(error)
+                    });
+                    if (policy === 'strict')
+                        break;
+                    continue;
+                }
+            }
+            else {
+                const promptFile = join(promptDir, `batch-${index + 1}.md`);
+                writeFileSync(promptFile, prompt);
+                const result = await runShellAgent(options.agent, prompt, {
+                    'prompt-file': promptFile,
+                    ...mcpConfig === undefined ? {} : { 'mcp-config': mcpConfig },
+                    db: resolve(options.db)
+                }, timeoutSeconds, cwd);
+                ok = result.code === 0;
+                output = result.stdout;
+                if (!ok) {
+                    reports.push({
+                        files: paths, ok, added: claimCount(store) - before, problems: [],
+                        note: result.error ?? `agent exited with ${result.code}`
+                    });
+                    if (policy === 'strict')
+                        break;
+                    continue;
+                }
+            }
+            if (mode === 'stdout') {
+                // Actor provenance (spec §9.5): mcp-mode appends are stamped by the
+                // MCP server; here the orchestrator appends, so it stamps — with the
+                // stable ingestion-surface identity, like `src:cli` and
+                // `src:agent/<client>`. A content- or batch-derived identity would
+                // fork the claim key (§9.2) on every source revision, leaving the
+                // old and the re-extracted belief both current.
+                const canonical = Canonical.canonicalizeText(caveTextOf(output), store.registry());
+                const problems = canonical.problems.map(problem => `line ${problem.line}: ${problem.message}`);
+                if (policy === 'strict' && problems.length > 0) {
+                    reports.push({ files: paths, ok: false, added: 0, problems });
+                    break;
+                }
+                const ingested = store.insertResult(canonical, { source: 'ingest' });
+                reports.push({
+                    files: paths,
+                    ok: problems.length === 0,
+                    added: ingested.ids.length,
+                    problems
+                });
+                if (problems.length > 0) {
+                    // A partially invalid extraction may be incomplete — withhold the
+                    // digests so these sources stay eligible for the next run.
+                    continue;
+                }
+            }
+            else {
+                const note = output.trim().split('\n').at(-1) ?? '';
+                reports.push({
+                    files: paths,
+                    ok: true,
+                    added: claimCount(store) - before,
+                    problems: [],
+                    ...note === '' ? {} : { note }
+                });
+            }
+            Files.recordDigests(store, files);
+        }
+    }
+    finally {
+        rmSync(promptDir, { recursive: true, force: true });
+        if (mcpConfig !== undefined) {
+            rmSync(dirname(mcpConfig), { recursive: true, force: true });
+        }
+    }
+    const sources = [
+        ...selection.skipped.map(path => ({ path, status: 'skipped', problems: [] })),
+        ...selection.files.map(file => {
+            const batch = reports.findIndex(report => report.files.includes(file.path));
+            if (batch < 0)
+                return { path: file.path, status: 'not-run', problems: [] };
+            const report = reports[batch];
+            return {
+                path: file.path,
+                status: report.ok ? 'accepted' : 'rejected',
+                batch: batch + 1,
+                problems: report.problems,
+                ...report.note === undefined ? {} : { note: report.note }
+            };
+        }),
+        ...selection.failures.map(failure => ({
+            path: failure.path,
+            status: 'rejected',
+            problems: [failure.message],
+            failure: failure.kind,
+            retryable: failure.retryable,
+            ...failure.status === undefined ? {} : { httpStatus: failure.status }
+        }))
+    ];
+    return {
+        policy,
+        applied: true,
+        matched: selection.files.length + selection.skipped.length + selection.failures.length,
+        skipped: selection.skipped,
+        batches: reports,
+        sources,
+        added: reports.reduce((sum, report) => sum + report.added, 0),
+        failed: reports.filter(report => !report.ok).length + selection.failures.length
+    };
+};
+/**
+ * Runs the full ingestion. Strict is the default: the target is copied into
+ * an isolated staging store, every generated MCP configuration and `{db}`
+ * substitution points there, and one identity-preserving merge applies the
+ * complete run only after every batch succeeds. A fatal batch stops further
+ * agent calls and discards all staged claims and digests.
+ *
+ * Lenient mode writes accepted work batch by batch, continues after failures,
+ * and withholds digests for rejected sources so the next invocation retries
+ * them. The returned source manifest accounts for every matched input.
+ */
+export const run = async (options) => {
+    const policy = options.policy ?? 'strict';
+    if (policy === 'lenient')
+        return runMutable({ ...options, policy });
+    const stageDir = mkdtempSync(join(tmpdir(), 'cave-ingest-stage-'));
+    const stageDb = join(stageDir, 'stage.db');
+    let staged;
+    try {
+        const stage = open(stageDb, options.noPrelude === true ? { registry: Registry.empty } : {});
+        try {
+            const seeded = syncText(stage, options.store.exportText({ tx: true, maxSensitivity: 'restricted' }), {
+                record: false
+            });
+            if (seeded.problems.length > 0) {
+                throw new Error(`could not stage the current store: ${seeded.problems.map(problem => `line ${problem.line}: ${problem.message}`).join('; ')}`);
+            }
+            staged = await runMutable({ ...options, db: stageDb, store: stage, policy });
+        }
+        finally {
+            stage.close();
+        }
+        if (staged.failed > 0 || staged.sources.some(source => source.status === 'not-run')) {
+            return { ...staged, applied: false, added: 0 };
+        }
+        syncDb(options.store, stageDb, { record: false });
+        return staged;
+    }
+    finally {
+        rmSync(stageDir, { recursive: true, force: true });
+    }
+};
+//# sourceMappingURL=run.js.map

package/dist/internal/ingest/run.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/run.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC1E,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAClD,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,KAAK,SAAS,MAAM,qBAAqB,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,OAAO,EAAE,IAAI,EAAc,MAAM,iBAAiB,CAAA;AAClD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AACjD,OAAO,KAAK,KAAK,MAAM,YAAY,CAAA;AACnC,OAAO,KAAK,GAAG,MAAM,UAAU,CAAA;AAC/B,OAAO,KAAK,OAAO,MAAM,cAAc,CAAA;AACvC,OAAO,KAAK,MAAM,MAAM,aAAa,CAAA;AAwFrC,2EAA2E;AAC3E,MAAM,CAAC,MAAM,cAAc,GAAG,CAC5B,EAAU,EACV,UAAiD,EAAE,EAC3C,EAAE;IACV,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAA;IACtE,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC,CAAA;IACtE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,eAAe,CAAC,CAAA;IACvC,aAAa,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QACpC,UAAU,EAAE;YACV,IAAI,EAAE;gBACJ,OAAO,EAAE,OAAO,CAAC,QAAQ;gBACzB,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,EAAE,GAAG,OAAO,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;aAC3F;SACF;KACF,EAAE,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;IACrB,OAAO,IAAI,CAAA;AACb,CAAC,CAAA;AAED,MAAM,WAAW,GAAG,CAAC,KAAgC,EAAE,KAAc,EAAE,GAAW,EAAE,EAAE,CACpF,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,IAAI,EAAE,IAAI,CAAC,IAAI;IACf,mEAAmE;IACnE,+DAA+D;IAC/D,GAAG,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QACzD,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;CAC1E,CAAC,CAAC,CAAA;AAEL,2EAA2E;AAC3E,MAAM,CAAC,MAAM,SAAS,GAAG,CACvB,KAAY,EACZ,KAAgC,EAChC,OAAiE,EACzD,EAAE;IACV,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACxC,MAAM,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IACvE,OAAO,MAAM,CAAC,WAAW,CAAC;QACxB,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,KAAK,IAAI,EAAE,GAAG,CAAC;QACtD,GAAG,OAAO,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,gBAAgB,CAAC,OAAO,CAAC,YAAY,CAAE,EAAE;QAC7G,GAAG,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE;QAC3C,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,KAAK;KAC5B,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,uFAAuF;AACvF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,KAAY,EACZ,OAAgB,EAC6F,EAAE;IAC/G,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IAC/C,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAA;IACnG,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;IACxE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,EAAE;QACvC,KAAK,EAAE,OAAO,CAAC,KAAK,KAAK,IAAI;QAC7B,GAAG,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;KACzD,CAAC,CAAA;IACF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE;QAC3C,KAAK,EAAE,OAAO,CAAC,KAAK,KAAK,IAAI;QAC7B,GAAG,OAAO,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE;KAC3E,CAAC,CAAA;IACF,MAAM,SAAS,GAAG;QAChB,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,KAAK,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC;QACxC,OAAO,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC;QAC9C,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAA;IACD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,IAAI,CAAC,CAAC,EAAE,CAAA;AACrF,CAAC,CAAA;AAED,MAAM,UAAU,GAAG,CAAC,KAAY,EAAU,EAAE,CACzC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,sCAAsC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAA;AAErF;;;;GAIG;AACH,MAAM,UAAU,GAAG,CAAC,KAAa,EAAU,EAAE,CAC3C,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,OAAO,CAAC,GAAG,CAAA;AAEvC;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAC3B,QAAgB,EAChB,MAAc,EACd,aAA+C,EAC/C,cAAsB,EACtB,GAAW,EACuD,EAAE;IACpE,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC;SAC1C,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAA;IAC3F,OAAO,IAAI,OAAO,CAAC,cAAc,CAAC,EAAE;QAClC,IAAI,OAAO,GAAG,KAAK,CAAA;QACnB,MAAM,MAAM,GAAG,CAAC,MAA+D,EAAQ,EAAE;YACvF,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAA;gBACd,cAAc,CAAC,MAAM,CAAC,CAAA;YACxB,CAAC;QACH,CAAC,CAAA;QACD,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE;YAC3B,KAAK,EAAE,IAAI;YACX,GAAG;YACH,OAAO,EAAE,cAAc,GAAG,IAAI;YAC9B,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC;SACnC,CAAC,CAAA;QACF,IAAI,MAAM,GAAG,EAAE,CAAA;QACf,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE;YAC9B,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,CAAA;QACzB,CAAC,CAAC,CAAA;QACF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;QAChF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;QACnD,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QACjC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACzB,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAA;IACnB,CAAC,CAAC,CAAA;AACJ,CAAC,CAAA;AAED,6EAA6E;AAC7E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,MAAc,EAAU,EAAE;IACnD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,8BAA8B,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;IAC3F,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;AACvD,CAAC,CAAA;AAED,8EAA8E;AAC9E,MAAM,UAAU,GAAG,KAAK,EAAE,OAAqC,EAAmB,EAAE;IAClF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAA;IAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACxC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,KAAK,CAAA;IAClC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;IAC7B,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,GAAG,CAAA;IACpD,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAClE,MAAM,OAAO,GAAkB,EAAE,CAAA;IACjC,MAAM,SAAS,GAAG,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC;QAC7F,cAAc,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;QACvE,SAAS,CAAA;IACX,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,cAAc,CAAC,CAAC,CAAA;IAC7D,IAAI,CAAC;QACH,uEAAuE;QACvE,wEAAwE;QACxE,2CAA2C;QAC3C,IAAI,MAAM,KAAK,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzD,OAAO;gBACL,MAAM;gBACN,OAAO,EAAE,KAAK;gBACd,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM;gBACtF,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,OAAO,EAAE,EAAE;gBACX,OAAO,EAAE;oBACP,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,SAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;oBACtF,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,SAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;oBAC/F,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;wBACpC,IAAI,EAAE,OAAO,CAAC,IAAI;wBAClB,MAAM,EAAE,UAAmB;wBAC3B,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;wBAC3B,OAAO,EAAE,OAAO,CAAC,IAAI;wBACrB,SAAS,EAAE,OAAO,CAAC,SAAS;wBAC5B,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE;qBACtE,CAAC,CAAC;iBACJ;gBACD,KAAK,EAAE,CAAC;gBACR,MAAM,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM;aAClC,CAAA;QACH,CAAC;QACD,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/C,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,GAAG,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;YAC5D,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC1C,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAChC,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAA;gBAC9F,IAAI,MAAM,KAAK,QAAQ;oBAAE,MAAK;gBAC9B,SAAQ;YACV,CAAC;YACD,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;YAChC,IAAI,EAAW,CAAA;YACf,IAAI,MAAc,CAAA;YAClB,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;gBACxC,IAAI,CAAC;oBACH,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE;wBAC1C,EAAE,EAAE,OAAO,CAAC,EAAE;wBACd,GAAG,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE;qBAChD,CAAC,CAAA;oBACF,EAAE,GAAG,IAAI,CAAA;gBACX,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,GAAG,EAAE,CAAA;oBACX,EAAE,GAAG,KAAK,CAAA;oBACV,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,EAAE;wBACjE,IAAI,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;qBAC7D,CAAC,CAAA;oBACF,IAAI,MAAM,KAAK,QAAQ;wBAAE,MAAK;oBAC9B,SAAQ;gBACV,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,KAAK,GAAG,CAAC,KAAK,CAAC,CAAA;gBAC3D,aAAa,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;gBACjC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE;oBACxD,aAAa,EAAE,UAAU;oBACzB,GAAG,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,SAAS,EAAE;oBAC7D,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;iBACxB,EAAE,cAAc,EAAE,GAAG,CAAC,CAAA;gBACvB,EAAE,GAAG,MAAM,CAAC,IAAI,KAAK,CAAC,CAAA;gBACtB,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;gBACtB,IAAI,CAAC,EAAE,EAAE,CAAC;oBACR,OAAO,CAAC,IAAI,CAAC;wBACX,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,EAAE;wBACjE,IAAI,EAAE,MAAM,CAAC,KAAK,IAAI,qBAAqB,MAAM,CAAC,IAAI,EAAE;qBACzD,CAAC,CAAA;oBACF,IAAI,MAAM,KAAK,QAAQ;wBAAE,MAAK;oBAC9B,SAAQ;gBACV,CAAC;YACH,CAAC;YACD,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,oEAAoE;gBACpE,qEAAqE;gBACrE,wDAAwD;gBACxD,mEAAmE;gBACnE,kEAAkE;gBAClE,gDAAgD;gBAChD,MAAM,SAAS,GAAG,SAAS,CAAC,gBAAgB,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAA;gBAClF,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAA;gBAC9F,IAAI,MAAM,KAAK,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC/C,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAA;oBAC7D,MAAK;gBACP,CAAC;gBACD,MAAM,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;gBACpE,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK,EAAE,KAAK;oBACZ,EAAE,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;oBACzB,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,MAAM;oBAC1B,QAAQ;iBACT,CAAC,CAAA;gBACF,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACxB,kEAAkE;oBAClE,2DAA2D;oBAC3D,SAAQ;gBACV,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;gBACnD,OAAO,CAAC,IAAI,CAAC;oBACX,KAAK,EAAE,KAAK;oBACZ,EAAE,EAAE,IAAI;oBACR,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,GAAG,MAAM;oBACjC,QAAQ,EAAE,EAAE;oBACZ,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE;iBAC/B,CAAC,CAAA;YACJ,CAAC;YACD,KAAK,CAAC,aAAa,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QACnC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACnD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;YAC5B,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9D,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAmB;QAC9B,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,SAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;QACtF,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE;YAC5B,MAAM,KAAK,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;YAC3E,IAAI,KAAK,GAAG,CAAC;gBAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,SAAkB,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAA;YACnF,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAE,CAAA;YAC9B,OAAO;gBACL,IAAI,EAAE,IAAI,CAAC,IAAI;gBACf,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,UAAmB,CAAC,CAAC,CAAC,UAAmB;gBAC7D,KAAK,EAAE,KAAK,GAAG,CAAC;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,GAAG,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE;aAC1D,CAAA;QACH,CAAC,CAAC;QACF,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YACpC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,UAAmB;YAC3B,QAAQ,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC;YAC3B,OAAO,EAAE,OAAO,CAAC,IAAI;YACrB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,GAAG,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE;SACtE,CAAC,CAAC;KACJ,CAAA;IACD,OAAO;QACL,MAAM;QACN,OAAO,EAAE,IAAI;QACb,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM;QACtF,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,OAAO,EAAE,OAAO;QAChB,OAAO;QACP,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QAC7D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,MAAM;KAChF,CAAA;AACH,CAAC,CAAA;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,MAAM,GAAG,GAAG,KAAK,EAAE,OAAgB,EAAmB,EAAE;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAA;IACzC,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,UAAU,CAAC,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC,CAAA;IAEnE,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAAA;IAClE,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAA;IAC1C,IAAI,MAA0B,CAAA;IAC9B,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;QAC3F,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,CAAC,EAAE;gBACnG,MAAM,EAAE,KAAK;aACd,CAAC,CAAA;YACF,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,IAAI,KAAK,CAAC,sCAAsC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAClF,QAAQ,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAC7D,CAAC;YACD,MAAM,GAAG,MAAM,UAAU,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;QAC9E,CAAC;gBAAS,CAAC;YACT,KAAK,CAAC,KAAK,EAAE,CAAA;QACf,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,EAAE,CAAC;YACpF,OAAO,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;QAChD,CAAC;QACD,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;QACjD,OAAO,MAAM,CAAA;IACf,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;IACpD,CAAC;AACH,CAAC,CAAA"}

package/dist/internal/ingest/web.d.ts

@@ -0,0 +1,48 @@
+/**
+ * URL sources for ingestion — `cave ingest https://…` fetches the page
+ * with the built-in fetch and reduces HTML to its readable article text
+ * (Readability over linkedom), so the agent sees prose instead of markup.
+ * Non-HTML responses (markdown, plain text, JSON) pass through verbatim.
+ *
+ * Provenance mirrors files: the URL is the claim subject and the digest
+ * is taken over the *extracted* text, so a page is re-ingested only when
+ * its readable content changes — not on every markup or chrome tweak.
+ */
+import type { Store } from '@cavelang/store';
+import type { Selected, Selection } from './files.ts';
+/** @returns whether an ingest pattern is a fetchable URL. */
+export declare const isUrl: (pattern: string) => boolean;
+/**
+ * @returns the readable text of an HTML page — Readability's article when
+ * it finds one, otherwise the whole body with scripts/styles dropped —
+ * prefixed with the page title as a heading.
+ */
+export declare const readableTextOf: (html: string) => string;
+/** Injection point for tests; the built-in fetch otherwise. */
+export type FetchLike = (url: string, init: RequestInit) => Promise<Response>;
+export type FailureKind = 'network' | 'http';
+export type Failure = {
+    readonly path: string;
+    readonly kind: FailureKind;
+    readonly retryable: boolean;
+    readonly message: string;
+    readonly status?: number;
+};
+/**
+ * Fetches one URL and returns it as a selectable source — readable text
+ * for HTML responses, the verbatim body for anything else.
+ */
+export declare const fetchDocument: (url: string, fetchImpl?: FetchLike, timeoutSeconds?: number) => Promise<Selected>;
+/**
+ * Fetches and digests the given URLs, skipping the ones whose current
+ * `ingest-digest` belief already matches (pass `force` to re-ingest all).
+ * The fetch always happens — the digest is over the extracted content —
+ * but unchanged pages cost no agent run.
+ */
+export declare const select: (store: Store, urls: readonly string[], options?: {
+    force?: boolean;
+    fetchImpl?: FetchLike;
+}) => Promise<Selection & {
+    failures: readonly Failure[];
+}>;
+//# sourceMappingURL=web.d.ts.map

package/dist/internal/ingest/web.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"web.d.ts","sourceRoot":"","sources":["../../src/web.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,iBAAiB,CAAA;AAE5C,OAAO,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAErD,6DAA6D;AAC7D,eAAO,MAAM,KAAK,GAAI,SAAS,MAAM,KAAG,OACT,CAAA;AA0C/B;;;;GAIG;AACH,eAAO,MAAM,cAAc,GAAI,MAAM,MAAM,KAAG,MAe7C,CAAA;AAED,+DAA+D;AAC/D,MAAM,MAAM,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAA;AAE7E,MAAM,MAAM,WAAW,GAAG,SAAS,GAAG,MAAM,CAAA;AAE5C,MAAM,MAAM,OAAO,GAAG;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;IAC1B,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB,CAAA;AAoBD;;;GAGG;AACH,eAAO,MAAM,aAAa,GACxB,KAAK,MAAM,EACX,YAAW,SAAiB,EAC5B,uBAAmB,KAClB,OAAO,CAAC,QAAQ,CA4BlB,CAAA;AAED;;;;;GAKG;AACH,eAAO,MAAM,MAAM,GACjB,OAAO,KAAK,EACZ,MAAM,SAAS,MAAM,EAAE,EACvB,UAAS;IAAE,KAAK,CAAC,EAAE,OAAO,CAAC;IAAC,SAAS,CAAC,EAAE,SAAS,CAAA;CAAO,KACvD,OAAO,CAAC,SAAS,GAAG;IAAE,QAAQ,EAAE,SAAS,OAAO,EAAE,CAAA;CAAE,CAwBtD,CAAA"}

package/dist/internal/ingest/web.js

@@ -0,0 +1,140 @@
+/**
+ * URL sources for ingestion — `cave ingest https://…` fetches the page
+ * with the built-in fetch and reduces HTML to its readable article text
+ * (Readability over linkedom), so the agent sees prose instead of markup.
+ * Non-HTML responses (markdown, plain text, JSON) pass through verbatim.
+ *
+ * Provenance mirrors files: the URL is the claim subject and the digest
+ * is taken over the *extracted* text, so a page is re-ingested only when
+ * its readable content changes — not on every markup or chrome tweak.
+ */
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import { digestOf, isIngested } from "./files.js";
+/** @returns whether an ingest pattern is a fetchable URL. */
+export const isUrl = (pattern) => /^https?:\/\//i.test(pattern);
+/** Block-level elements worth a line of their own in the extracted text. */
+const blockSelector = 'p, h1, h2, h3, h4, h5, h6, li, pre, blockquote, td, figcaption';
+const headingPrefix = {
+    H1: '# ', H2: '## ', H3: '### ', H4: '#### ', H5: '##### ', H6: '###### ', LI: '- '
+};
+/** Readability's types are written against the browser DOM lib, which a
+ * node-only tsconfig does not load — retype the constructor structurally. */
+const Parser = Readability;
+/** Flattens a DOM subtree into markdown-ish text, one block per line. */
+const textOf = (root) => {
+    const blocks = [...root.querySelectorAll(blockSelector)]
+        // Keep outermost blocks only — a <p> inside a kept <li> is its text.
+        .filter(element => element.parentElement === null || element.parentElement.closest(blockSelector) === null)
+        .map(element => {
+        const text = element.textContent ?? '';
+        return element.tagName === 'PRE' ?
+            text.trim() :
+            `${headingPrefix[element.tagName] ?? ''}${text.replace(/\s+/g, ' ').trim()}`;
+    })
+        .filter(line => line !== '');
+    return blocks.length > 0 ? blocks.join('\n\n') : (root.textContent ?? '').trim();
+};
+/**
+ * @returns the readable text of an HTML page — Readability's article when
+ * it finds one, otherwise the whole body with scripts/styles dropped —
+ * prefixed with the page title as a heading.
+ */
+export const readableTextOf = (html) => {
+    const parse = () => parseHTML(html).document;
+    const article = new Parser(parse()).parse();
+    const body = () => {
+        const document = parse();
+        for (const element of document.querySelectorAll('script, style, noscript, template')) {
+            element.remove();
+        }
+        return textOf((document.body ?? document));
+    };
+    const text = article === null || article.content === null || article.content === undefined ?
+        body() :
+        textOf(parseHTML(`<html><body>${article.content}</body></html>`).document);
+    const title = (article?.title ?? parse().title ?? '').trim();
+    return title === '' || text.startsWith(`# ${title}`) ? text : `# ${title}\n\n${text}`;
+};
+class FetchFailure extends Error {
+    failure;
+    constructor(failure) {
+        super(failure.message);
+        this.failure = failure;
+    }
+}
+const retryableStatus = (status) => status === 408 || status === 425 || status === 429 || status >= 500;
+const failureOf = (url, error) => {
+    if (error instanceof FetchFailure)
+        return error.failure;
+    const detail = error instanceof Error ? error.message : String(error);
+    return { path: url, kind: 'network', retryable: true, message: `fetch ${url} failed: ${detail}` };
+};
+/**
+ * Fetches one URL and returns it as a selectable source — readable text
+ * for HTML responses, the verbatim body for anything else.
+ */
+export const fetchDocument = async (url, fetchImpl = fetch, timeoutSeconds = 60) => {
+    let response;
+    try {
+        response = await fetchImpl(url, {
+            headers: {
+                'user-agent': 'cave-ingest',
+                accept: 'text/html, text/markdown, text/plain, application/json;q=0.9, */*;q=0.8'
+            },
+            redirect: 'follow',
+            signal: AbortSignal.timeout(timeoutSeconds * 1000)
+        });
+    }
+    catch (error) {
+        throw new FetchFailure(failureOf(url, error));
+    }
+    if (!response.ok) {
+        throw new FetchFailure({
+            path: url,
+            kind: 'http',
+            retryable: retryableStatus(response.status),
+            status: response.status,
+            message: `fetch ${url} failed: ${response.status} ${response.statusText}`
+        });
+    }
+    const type = response.headers.get('content-type') ?? '';
+    const body = await response.text();
+    const html = /html|xhtml/i.test(type) || (type === '' && /^\s*</.test(body));
+    const content = html ? readableTextOf(body) : body;
+    return { path: url, digest: digestOf(content), content };
+};
+/**
+ * Fetches and digests the given URLs, skipping the ones whose current
+ * `ingest-digest` belief already matches (pass `force` to re-ingest all).
+ * The fetch always happens — the digest is over the extracted content —
+ * but unchanged pages cost no agent run.
+ */
+export const select = async (store, urls, options = {}) => {
+    const files = [];
+    const skipped = [];
+    const failures = [];
+    const outcomes = await Promise.all([...new Set(urls)].map(async (url) => {
+        try {
+            return { document: await fetchDocument(url, options.fetchImpl) };
+        }
+        catch (error) {
+            return { failure: failureOf(url, error) };
+        }
+    }));
+    for (const outcome of outcomes) {
+        if ('failure' in outcome) {
+            failures.push(outcome.failure);
+            continue;
+        }
+        const document = outcome.document;
+        if (options.force !== true && isIngested(store, document.path, document.digest)) {
+            skipped.push(document.path);
+        }
+        else {
+            files.push(document);
+        }
+    }
+    return { files, skipped, failures };
+};
+//# sourceMappingURL=web.js.map

package/dist/internal/ingest/web.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"web.js","sourceRoot":"","sources":["../../src/web.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AAEpC,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAGjD,6DAA6D;AAC7D,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,OAAe,EAAW,EAAE,CAChD,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;AAE/B,4EAA4E;AAC5E,MAAM,aAAa,GAAG,gEAAgE,CAAA;AAEtF,MAAM,aAAa,GAAqC;IACtD,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI;CACpF,CAAA;AAeD;6EAC6E;AAC7E,MAAM,MAAM,GAAG,WAC+E,CAAA;AAE9F,yEAAyE;AACzE,MAAM,MAAM,GAAG,CAAC,IAAc,EAAU,EAAE;IACxC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;QACtD,qEAAqE;SACpE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,IAAI,CAAC;SAC1G,GAAG,CAAC,OAAO,CAAC,EAAE;QACb,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,IAAI,EAAE,CAAA;QACtC,OAAO,OAAO,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;YACb,GAAG,aAAa,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAA;IAChF,CAAC,CAAC;SACD,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,CAAA;IAC9B,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;AAClF,CAAC,CAAA;AAED;;;;GAIG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,IAAY,EAAU,EAAE;IACrD,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAA;IAC5C,MAAM,OAAO,GAAG,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAA;IAC3C,MAAM,IAAI,GAAG,GAAW,EAAE;QACxB,MAAM,QAAQ,GAAG,KAAK,EAAE,CAAA;QACxB,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,gBAAgB,CAAC,mCAAmC,CAA0B,EAAE,CAAC;YAC9G,OAAO,CAAC,MAAM,EAAE,CAAA;QAClB,CAAC;QACD,OAAO,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAa,CAAC,CAAA;IACxD,CAAC,CAAA;IACD,MAAM,IAAI,GAAG,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,OAAO,KAAK,IAAI,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC;QAC1F,IAAI,EAAE,CAAC,CAAC;QACR,MAAM,CAAC,SAAS,CAAC,eAAe,OAAO,CAAC,OAAO,gBAAgB,CAAC,CAAC,QAA+B,CAAC,CAAA;IACnG,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,KAAK,IAAI,KAAK,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAC5D,OAAO,KAAK,KAAK,EAAE,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK,OAAO,IAAI,EAAE,CAAA;AACvF,CAAC,CAAA;AAeD,MAAM,YAAa,SAAQ,KAAK;IACrB,OAAO,CAAS;IAEzB,YAAY,OAAgB;QAC1B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;QACtB,IAAI,CAAC,OAAO,GAAG,OAAO,CAAA;IACxB,CAAC;CACF;AAED,MAAM,eAAe,GAAG,CAAC,MAAc,EAAW,EAAE,CAClD,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,IAAI,GAAG,CAAA;AAErE,MAAM,SAAS,GAAG,CAAC,GAAW,EAAE,KAAc,EAAW,EAAE;IACzD,IAAI,KAAK,YAAY,YAAY;QAAE,OAAO,KAAK,CAAC,OAAO,CAAA;IACvD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACrE,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,GAAG,YAAY,MAAM,EAAE,EAAE,CAAA;AACnG,CAAC,CAAA;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,GAAW,EACX,YAAuB,KAAK,EAC5B,cAAc,GAAG,EAAE,EACA,EAAE;IACrB,IAAI,QAAkB,CAAA;IACtB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE;YAC9B,OAAO,EAAE;gBACP,YAAY,EAAE,aAAa;gBAC3B,MAAM,EAAE,yEAAyE;aAClF;YACD,QAAQ,EAAE,QAAQ;YAClB,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;SACnD,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAA;IAC/C,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,YAAY,CAAC;YACrB,IAAI,EAAE,GAAG;YACT,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3C,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,OAAO,EAAE,SAAS,GAAG,YAAY,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE;SAC1E,CAAC,CAAA;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAA;IACvD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAA;IAClC,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAClD,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,CAAA;AAC1D,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,MAAM,GAAG,KAAK,EACzB,KAAY,EACZ,IAAuB,EACvB,UAAsD,EAAE,EACD,EAAE;IACzD,MAAM,KAAK,GAAe,EAAE,CAAA;IAC5B,MAAM,OAAO,GAAa,EAAE,CAAA;IAC5B,MAAM,QAAQ,GAAc,EAAE,CAAA;IAC9B,MAAM,QAAQ,GAAsD,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,EAAC,GAAG,EAAC,EAAE;QACvH,IAAI,CAAC;YACH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE,CAAA;QAClE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAA;QAC3C,CAAC;IACH,CAAC,CAAC,CAAC,CAAA;IACH,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,IAAI,SAAS,IAAI,OAAO,EAAE,CAAC;YACzB,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAC9B,SAAQ;QACV,CAAC;QACD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAA;QACjC,IAAI,OAAO,CAAC,KAAK,KAAK,IAAI,IAAI,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YAChF,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QAC7B,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACtB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAA;AACrC,CAAC,CAAA"}

package/dist/internal/loop/demo.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Runnable demo (spec §18): the multi-hop recovery pattern central to the
+ * active-reconstruction thesis (spec §11.3).
+ *
+ * Starting from the single cue `reject-valid-tokens` — the *symptom* — the
+ * loop must travel edges that only exist as inverse reads:
+ *
+ * 1. `reject-valid-tokens` ← CAUSED-BY — `token-expiry` (inverse CAUSE)
+ * 2. `token-expiry` ← PART-OF — `topic/auth-hardening` (inverse CONTAINS)
+ * 3. topic expansion → `auth/middleware`, and with it the bug, the broken
+ *    check and the fix.
+ *
+ * ```
+ * pnpm --filter @cavelang/cli/loop demo
+ * ```
+ */
+export declare const knowledge = "\nauth/middleware HAS bug: token-expiry #security #topic:auth-hardening\ntoken-expiry CAUSE reject-valid-tokens\nexpiry-check USES `<`\nexpiry-check NEEDS `<=`\n`<=` FIX token-expiry @auth.ts:42\nauth/middleware NEEDS test: boundary-cases @ 70%\ntopic/auth-hardening CONTAINS token-expiry\ntopic/auth-hardening CONTAINS auth/middleware\ntopic/auth-hardening CONTAINS expiry-check\ndeploy VIA github-actions\nunrelated/service USES postgres\n";
+export declare const run: () => {
+    lines: string[];
+};
+export declare const isDirectInvocation: (moduleUrl: string, entryPath: string | undefined, platform?: NodeJS.Platform) => boolean;
+//# sourceMappingURL=demo.d.ts.map

package/dist/internal/loop/demo.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"demo.d.ts","sourceRoot":"","sources":["../../src/demo.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAQH,eAAO,MAAM,SAAS,8bAYrB,CAAA;AAED,eAAO,MAAM,GAAG,QAAO;IAAE,KAAK,EAAE,MAAM,EAAE,CAAA;CAoBvC,CAAA;AAED,eAAO,MAAM,kBAAkB,GAC7B,WAAW,MAAM,EACjB,WAAW,MAAM,GAAG,SAAS,EAC7B,WAAU,MAAM,CAAC,QAA2B,KAC3C,OAKF,CAAA"}