agentfootprint 8.9.0 → 8.10.0

package/dist/esm/rag/indexCorpus.js

@@ -0,0 +1,339 @@
+/**
+ * indexCorpus — building the index is itself a flowchart, so the run explains
+ * itself.
+ *
+ * Pattern: a footprintjs chart. Not "a function that logs" — the stages, the
+ *          decision, the fan-out and the report are the real thing, and the
+ *          commit log IS the indexing report.
+ * Role:    rag/ layer. Step three, and the one that ties the other two to a
+ *          store.
+ * Emits:   `agentfootprint.embedding.generated` (`inputKind: 'document'`) per
+ *          embedded batch, through the emit channel.
+ *
+ * ── Why a chart and not a loop ──────────────────────────────────────────────
+ * Indexing is where a corpus acquires the properties retrieval later depends
+ * on, and every one of those is a question someone eventually asks at an
+ * awkward moment: *why is this chunk in the index? why was that one skipped?
+ * what did this run cost? which document went missing?* A `for` loop answers
+ * none of them after it has finished. A chart answers all of them from its own
+ * commit log, months later, without the caller having saved anything.
+ *
+ *   discover → load → split → plan (DECIDER)
+ *            → take-window → embed (FAN-OUT + retry) → tally → more? ⟲
+ *            → remove → report
+ *
+ * The **plan** stage is a real decider rather than an `if`, because "skip this
+ * chunk" is a decision with evidence: the stored content hash equals the new
+ * one AND the stored embedder fingerprint equals the current one. Both
+ * predicates read from scope, so `decide()` captures the values that drove the
+ * choice and `FlowRecorder.onDecision` carries them — which is what makes
+ * "why was that one skipped?" answerable rather than merely plausible.
+ *
+ * The **embed** stage fans out with `addParallelForEach`, one branch per batch,
+ * each with a declarative `retry`. A hand-rolled retry inside the stage would
+ * be invisible: the narrative would show one stage and the two rate-limited
+ * attempts before the successful one would leave no mark. Declared, every
+ * attempt is in the record.
+ *
+ * It fans out over a WINDOW rather than over every batch, and loops. That is
+ * not an optimisation: `maxBranches` truncates the extra items rather than
+ * queueing them, so a single fan-out over 200 batches with a ceiling of 4
+ * would embed 4 and report success. The window can never exceed the ceiling,
+ * and the loop comes back for the rest.
+ *
+ * ── Pausing, and the honest limit on it ─────────────────────────────────────
+ * A large corpus can exceed a budget mid-run, so the window loop is where a
+ * long run can stop. **It can only stop at a WINDOW BOUNDARY**, and that is a
+ * real constraint rather than a preference: `interrupt()` re-runs its stage
+ * from the top on resume, so a pause inside an embedder call would re-embed
+ * everything that call had already done. Every completed batch is committed
+ * before the loop comes round, so a resume re-embeds at most the window that
+ * was in flight.
+ */
+import { flowChart } from 'footprintjs';
+import { decide } from 'footprintjs';
+import { emitEmbedding } from '../memory/embedding/emitEmbedding.js';
+import { DEFAULT_CORPUS_IDENTITY } from '../lib/rag/defineRAG.js';
+import { loadDocuments } from './loadDocuments.js';
+import { splitDocuments } from './splitDocuments.js';
+import { byHeading } from './splitters/byHeading.js';
+const DEFAULT_MAX_CHUNK_CHARS = 2000;
+/**
+ * Build and run the indexing chart.
+ *
+ * @returns the report — which is also committed to the run's own log, so it is
+ *   evidence rather than only a return value.
+ *
+ * @example
+ * ```ts
+ * const report = await indexCorpus({
+ *   source: { dir: './docs' },
+ *   store: sqliteVectorStore({ file: './corpus.db' }),
+ *   embedder: staticEmbedder(),
+ * });
+ * // { discovered: 3, loaded: 3, chunks: 14, embedded: 14, skipped: 0, removed: 0, … }
+ * ```
+ */
+export async function indexCorpus(config) {
+    const chart = buildIndexChart(config);
+    // `run()` hands back `{ state, output, executionTree, commitLog }`. The
+    // report is read off the final state — and it is ALSO in `commitLog`, which
+    // is the copy that survives without the caller keeping this return value.
+    const result = (await chart.run({}));
+    const report = result?.state?.report;
+    if (!report) {
+        throw new Error('indexCorpus: the chart finished without producing a report.');
+    }
+    return report;
+}
+/**
+ * The chart itself, exposed so a caller can mount it, attach recorders to it,
+ * or run it under their own executor — the same freedom every other chart in
+ * this library gives.
+ */
+export function buildIndexChart(config) {
+    return compile(config);
+}
+function compile(config) {
+    const corpus = config.corpus ?? DEFAULT_CORPUS_IDENTITY;
+    const splitter = config.splitter ?? byHeading();
+    const embedderId = config.embedderId ?? config.embedder.id ?? 'default-embedder';
+    const batchSize = Math.max(1, Math.floor(config.batchSize ?? 64));
+    const maxBatches = Math.max(1, Math.floor(config.maxConcurrentBatches ?? 4));
+    const attempts = Math.max(1, Math.floor(config.attempts ?? 3));
+    const maxChunkChars = config.maxChunkChars ?? DEFAULT_MAX_CHUNK_CHARS;
+    const removeMissing = config.removeMissing ?? true;
+    const startedAt = Date.now();
+    const fingerprint = `${embedderId}@${config.embedder.dimensions}`;
+    return (flowChart('Discover + load', async (scope) => {
+        const { documents, failed, discovered } = await loadDocuments(config.source, {
+            ...(config.loaders && { loaders: config.loaders }),
+        });
+        scope.discoveredCount = discovered;
+        scope.documents = documents;
+        scope.failed = failed;
+    }, 'discover-load', { description: 'Walk the source and read every document a loader claims' })
+        .addFunction('Split', (scope) => {
+        const chunks = splitDocuments(scope.documents ?? [], splitter);
+        scope.chunks = chunks;
+        scope.truncated = chunks
+            .filter((c) => c.text.length > maxChunkChars)
+            .map((c) => ({ id: c.id, chars: c.text.length }));
+    }, 'split', `Cut every document with '${splitter.name}'`)
+        // ── The plan. A decider, because "skip" is a decision with evidence. ──
+        .addDeciderFunction('Plan', (async (scope) => {
+        const chunks = scope.chunks ?? [];
+        const wanted = new Map(chunks.map((c) => [c.id, c]));
+        // What is already there, and is it still current? Two predicates,
+        // both recorded: same content, same embedding space.
+        const toEmbed = [];
+        const toSkip = [];
+        for (const chunk of chunks) {
+            const existing = await config.store.get(corpus, chunk.id);
+            const storedHash = existing?.value?.contentHash;
+            const storedFingerprint = existing?.embeddingModel === undefined
+                ? undefined
+                : `${existing.embeddingModel}@${(existing.embedding ?? []).length}`;
+            const sameContent = storedHash === chunk.contentHash;
+            const sameEmbedder = storedFingerprint === fingerprint;
+            if (existing && sameContent && sameEmbedder)
+                toSkip.push(chunk.id);
+            else
+                toEmbed.push(chunk);
+        }
+        // Chunks in the index whose document is no longer in the source, or
+        // which this document no longer produces (it got shorter).
+        const toRemove = [];
+        if (removeMissing) {
+            const liveDocs = new Set((scope.documents ?? []).map((d) => d.uri));
+            let cursor;
+            do {
+                const page = await config.store.list(corpus, {
+                    limit: 500,
+                    ...(cursor !== undefined && { cursor }),
+                });
+                for (const entry of page.entries) {
+                    if (wanted.has(entry.id))
+                        continue;
+                    const docUri = entry.value?.docUri;
+                    // Only ever remove chunks of documents this run actually saw
+                    // the source for. A namespace shared with something else is
+                    // not ours to prune.
+                    if (docUri !== undefined && liveDocs.has(docUri))
+                        toRemove.push(entry.id);
+                    else if (docUri !== undefined && isFromThisCorpus(docUri, scope)) {
+                        toRemove.push(entry.id);
+                    }
+                }
+                cursor = page.cursor;
+            } while (cursor !== undefined);
+        }
+        scope.toEmbed = toEmbed;
+        scope.toSkip = toSkip;
+        scope.toRemove = toRemove;
+        scope.pendingBatches = chunk(toEmbed, batchSize);
+        scope.embeddedCount = 0;
+        return decide(scope, [
+            {
+                when: (s) => (s.toEmbed ?? []).length === 0 && (s.toRemove ?? []).length === 0,
+                then: 'nothing-to-do',
+                label: 'every chunk is already indexed with this embedder, and nothing was removed',
+            },
+            {
+                when: (s) => (s.toSkip ?? []).length > 0,
+                then: 'incremental',
+                label: 'some chunks are unchanged — same content hash, same embedder fingerprint',
+            },
+        ], 'full-index');
+    }), 'plan', 'Decide what to embed, what to skip, and what to remove')
+        .addFunctionBranch('nothing-to-do', 'NothingToDo', noop, 'No work — index is current')
+        .addFunctionBranch('incremental', 'Incremental', noop, 'Embed only what changed')
+        .addFunctionBranch('full-index', 'FullIndex', noop, 'Embed everything discovered')
+        // Back to the trunk: the branch above is the RECORD of which plan was
+        // taken, and the work after it is the same whichever one fired — an
+        // empty `toEmbed` fans out over nothing, which costs nothing and still
+        // files a report.
+        .end()
+        // ── Embed, one WINDOW of batches at a time. ──────────────────────────
+        // `maxBranches` is a hard ceiling, and footprintjs does not queue the
+        // extra items — it TRUNCATES them. Fanning out over every batch at once
+        // would therefore embed only the first `maxConcurrentBatches` of them
+        // and report success: an index silently containing a fraction of the
+        // corpus, which is the worst thing this release could ship.
+        //
+        // So the fan-out runs over a WINDOW that can never exceed the ceiling,
+        // and the loop after it comes back for the next one. Bounded
+        // parallelism, unbounded total work, nothing dropped.
+        .addFunction('Take window', (scope) => {
+        const pending = scope.pendingBatches ?? [];
+        scope.window = pending.slice(0, maxBatches);
+        scope.pendingBatches = pending.slice(maxBatches);
+    }, 'take-window', `Take up to ${maxBatches} batches for this pass`)
+        .addParallelForEach('Embed batches', 'embed', {
+        items: (scope) => scope.window ?? [],
+        branch: (batch, index) => flowChart(`Embed batch ${index + 1}`, async (scope) => {
+            const batchChunks = (scope.item ?? batch);
+            if (batchChunks.length === 0) {
+                scope.written = 0;
+                return;
+            }
+            const texts = batchChunks.map((c) => c.text);
+            const startedEmbedAt = Date.now();
+            const vectors = config.embedder.embedBatch
+                ? await config.embedder.embedBatch({ texts })
+                : await Promise.all(texts.map((text) => config.embedder.embed({ text })));
+            emitEmbedding(scope, {
+                model: embedderId,
+                provider: 'custom',
+                inputKind: 'document',
+                dimension: config.embedder.dimensions,
+                count: vectors.length,
+                durationMs: Date.now() - startedEmbedAt,
+            });
+            const now = Date.now();
+            const entries = batchChunks.map((c, i) => ({
+                id: c.id,
+                value: c,
+                version: 1,
+                createdAt: now,
+                updatedAt: now,
+                lastAccessedAt: now,
+                accessCount: 0,
+                ...(vectors[i] && { embedding: [...vectors[i]] }),
+                embeddingModel: embedderId,
+            }));
+            // One transactional write per batch — so a batch is all-in or
+            // all-out, and a resumed run re-embeds at most this batch.
+            await config.store.putMany(corpus, entries);
+            scope.written = entries.length;
+        }, `embed-batch-${index}`, { description: `Embed + store ${batch.length} chunks` })
+            // Declared, not hand-rolled: every attempt is in the record.
+            .retry({
+            attempts,
+            backoffMs: (n) => Math.min(8000, 250 * 2 ** (n - 1)),
+        })
+            .build(),
+        maxBranches: maxBatches,
+        into: 'batchResults',
+        // A failed batch must fail the RUN. Best-effort (the default) leaves a
+        // half-indexed corpus that keeps answering and quietly cannot see the
+        // documents that did not land — which reads as "the model does not know
+        // that" rather than as a failure, and is the single most expensive way
+        // for this to go wrong.
+        failFast: true,
+    })
+        .addFunction('Tally window', (scope) => {
+        // Counted from what the batches actually WROTE, never from what the
+        // plan intended: a report that says "embedded 3" because three were
+        // queued cannot be trusted about the one number anybody checks.
+        //
+        // A fan-out branch's slot holds that branch's final STATE, so the
+        // count comes off `written` — the field the branch sets after the
+        // store call returned.
+        const written = (scope.batchResults ?? []).reduce((total, result) => {
+            const n = result?.written;
+            return total + (typeof n === 'number' ? n : 0);
+        }, 0);
+        scope.embeddedCount = (scope.embeddedCount ?? 0) + written;
+    }, 'tally-window', 'Add this window to the running total')
+        // More batches? Go round again. This loop is also the ONLY point at
+        // which a long indexing run can be interrupted safely: every completed
+        // batch is already committed, so a resume re-embeds at most the window
+        // that was in flight.
+        .addDeciderFunction('More batches?', ((scope) => decide(scope, [
+        {
+            when: (s) => (s.pendingBatches ?? []).length > 0,
+            then: 'more-batches',
+            label: 'batches remain — take another window',
+        },
+    ], 'batches-done')), 'more-batches-decider', 'Is there another window of batches to embed?')
+        .addFunctionBranch('more-batches', 'MoreBatches', noop, 'Another window remains')
+        .loopTo('take-window')
+        .addFunctionBranch('batches-done', 'BatchesDone', noop, 'Every batch is embedded')
+        .end()
+        .addFunction('Store removals', async (scope) => {
+        let removed = 0;
+        for (const id of scope.toRemove ?? []) {
+            await config.store.delete(corpus, id);
+            removed += 1;
+        }
+        scope.removedCount = removed;
+    }, 'store-removals', 'Delete chunks whose document changed or disappeared')
+        .addFunction('Report', (scope) => {
+        // The report is a COMMIT, not just a return value. A later reader
+        // asks the trace what this run did; nothing had to be saved.
+        scope.report = {
+            discovered: scope.discoveredCount ?? 0,
+            loaded: (scope.documents ?? []).length,
+            chunks: (scope.chunks ?? []).length,
+            embedded: scope.embeddedCount ?? 0,
+            skipped: (scope.toSkip ?? []).length,
+            removed: scope.removedCount ?? 0,
+            failed: scope.failed ?? [],
+            truncated: scope.truncated ?? [],
+            embedderFingerprint: fingerprint,
+            splitter: splitter.name,
+            elapsedMs: Date.now() - startedAt,
+        };
+    }, 'report', 'File what this run did — the commit log IS the report')
+        .build());
+}
+const noop = () => {
+    /* The branch itself is the record; the work is downstream. */
+};
+/** Was this document uri produced by the source this run walked? */
+function isFromThisCorpus(docUri, scope) {
+    // A chunk whose document was in the index and is NOT in this run's document
+    // list came from a document that disappeared — provided this run walked the
+    // same source. `documents` empty means the source turned up nothing, and
+    // pruning the whole namespace on an empty walk is how a typo in a path
+    // deletes a corpus, so it does not.
+    return (scope.documents ?? []).length > 0 && !docUri.startsWith('');
+}
+function chunk(items, size) {
+    const out = [];
+    for (let i = 0; i < items.length; i += size)
+        out.push([...items.slice(i, i + size)]);
+    return out;
+}
+//# sourceMappingURL=indexCorpus.js.map

package/dist/esm/rag/indexCorpus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexCorpus.js","sourceRoot":"","sources":["../../../src/rag/indexCorpus.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmDG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAExC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAMrC,OAAO,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AACrE,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAoFrD,MAAM,uBAAuB,GAAG,IAAI,CAAC;AAErC;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,MAAyB;IACzD,MAAM,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IACtC,wEAAwE;IACxE,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,MAAM,GAAG,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAyC,CAAC;IAC7E,MAAM,MAAM,GAAG,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC;IACrC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,MAAyB;IACvD,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,OAAO,CAAC,MAAyB;IACxC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,uBAAuB,CAAC;IACxD,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,SAAS,EAAE,CAAC;IAChD,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,EAAE,IAAI,kBAAkB,CAAC;IACjF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;IAClE,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,oBAAoB,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,uBAAuB,CAAC;IACtE,MAAM,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,WAAW,GAAG,GAAG,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;IAElE,OAAO,CACL,SAAS,CACP,iBAAiB,EACjB,KAAK,EAAE,KAAK,EAAE,EAAE;QACd,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,MAAM,EAAE;YAC3E,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;SACnD,CAAC,CAAC;QACH,KAAK,CAAC,eAAe,GAAG,UAAU,CAAC;QACnC,KAAK,CAAC,SAAS,GAAG,SAAS,CAAC;QAC5B,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;IACxB,CAAC,EACD,eAAe,EACf,EAAE,WAAW,EAAE,yDAAyD,EAAE,CAC3E;SACE,WAAW,CACV,OAAO,EACP,CAAC,KAAK,EAAE,EAAE;QACR,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC;QAC/D,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;QACtB,KAAK,CAAC,SAAS,GAAG,MAAM;aACrB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC;aAC5C,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACtD,CAAC,EACD,OAAO,EACP,4BAA4B,QAAQ,CAAC,IAAI,GAAG,CAC7C;QAED,yEAAyE;SACxE,kBAAkB,CACjB,MAAM,EACN,CAAC,KAAK,EAAE,KAA6B,EAAE,EAAE;QACvC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC;QAClC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAErD,kEAAkE;QAClE,qDAAqD;QACrD,MAAM,OAAO,GAAY,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,GAAG,CAA2B,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;YACpF,MAAM,UAAU,GAAG,QAAQ,EAAE,KAAK,EAAE,WAAW,CAAC;YAChD,MAAM,iBAAiB,GACrB,QAAQ,EAAE,cAAc,KAAK,SAAS;gBACpC,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,GAAG,QAAQ,CAAC,cAAc,IAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC;YACxE,MAAM,WAAW,GAAG,UAAU,KAAK,KAAK,CAAC,WAAW,CAAC;YACrD,MAAM,YAAY,GAAG,iBAAiB,KAAK,WAAW,CAAC;YACvD,IAAI,QAAQ,IAAI,WAAW,IAAI,YAAY;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;;gBAC9D,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC;QAED,oEAAoE;QACpE,2DAA2D;QAC3D,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,IAAI,aAAa,EAAE,CAAC;YAClB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACpE,IAAI,MAA0B,CAAC;YAC/B,GAAG,CAAC;gBACF,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,IAAI,CAAsB,MAAM,EAAE;oBAChE,KAAK,EAAE,GAAG;oBACV,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,CAAC;iBACxC,CAAC,CAAC;gBACH,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;oBACjC,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;wBAAE,SAAS;oBACnC,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC;oBACnC,6DAA6D;oBAC7D,4DAA4D;oBAC5D,qBAAqB;oBACrB,IAAI,MAAM,KAAK,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC;wBAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;yBACrE,IAAI,MAAM,KAAK,SAAS,IAAI,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;wBACjE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC1B,CAAC;gBACH,CAAC;gBACD,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;YACvB,CAAC,QAAQ,MAAM,KAAK,SAAS,EAAE;QACjC,CAAC;QAED,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;QACxB,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC;QACtB,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC1B,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;QACjD,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC;QAExB,OAAO,MAAM,CACX,KAAK,EACL;YACE;gBACE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC;gBAC9E,IAAI,EAAE,eAAe;gBACrB,KAAK,EAAE,4EAA4E;aACpF;YACD;gBACE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;gBACxC,IAAI,EAAE,aAAa;gBACnB,KAAK,EAAE,0EAA0E;aAClF;SACF,EACD,YAAY,CACb,CAAC;IACJ,CAAC,CAAU,EACX,MAAM,EACN,wDAAwD,CACzD;SACA,iBAAiB,CAChB,eAAe,EACf,aAAa,EACb,IAAa,EACb,4BAA4B,CAC7B;SACA,iBAAiB,CAAC,aAAa,EAAE,aAAa,EAAE,IAAa,EAAE,yBAAyB,CAAC;SACzF,iBAAiB,CAAC,YAAY,EAAE,WAAW,EAAE,IAAa,EAAE,6BAA6B,CAAC;QAC3F,sEAAsE;QACtE,oEAAoE;QACpE,uEAAuE;QACvE,kBAAkB;SACjB,GAAG,EAAE;QAEN,wEAAwE;QACxE,sEAAsE;QACtE,wEAAwE;QACxE,sEAAsE;QACtE,qEAAqE;QACrE,4DAA4D;QAC5D,EAAE;QACF,uEAAuE;QACvE,6DAA6D;QAC7D,sDAAsD;SACrD,WAAW,CACV,aAAa,EACb,CAAC,KAAK,EAAE,EAAE;QACR,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc,IAAI,EAAE,CAAC;QAC3C,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;QAC5C,KAAK,CAAC,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACnD,CAAC,EACD,aAAa,EACb,cAAc,UAAU,wBAAwB,CACjD;SACA,kBAAkB,CAAmB,eAAe,EAAE,OAAO,EAAE;QAC9D,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE;QACpC,MAAM,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CACvB,SAAS,CACP,eAAe,KAAK,GAAG,CAAC,EAAE,EAC1B,KAAK,EAAE,KAAK,EAAE,EAAE;YACd,MAAM,WAAW,GAAG,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAqB,CAAC;YAC9D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC;gBAClB,OAAO;YACT,CAAC;YACD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU;gBACxC,CAAC,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,KAAK,EAAE,CAAC;gBAC7C,CAAC,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;YAE5E,aAAa,CAAC,KAAc,EAAE;gBAC5B,KAAK,EAAE,UAAU;gBACjB,QAAQ,EAAE,QAAQ;gBAClB,SAAS,EAAE,UAAU;gBACrB,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU;gBACrC,KAAK,EAAE,OAAO,CAAC,MAAM;gBACrB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc;aACxC,CAAC,CAAC;YAEH,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,OAAO,GAAyB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/D,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,KAAK,EAAE,CAAC;gBACR,OAAO,EAAE,CAAC;gBACV,SAAS,EAAE,GAAG;gBACd,SAAS,EAAE,GAAG;gBACd,cAAc,EAAE,GAAG;gBACnB,WAAW,EAAE,CAAC;gBACd,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC,GAAI,OAAO,CAAC,CAAC,CAAc,CAAC,EAAE,CAAC;gBAC/D,cAAc,EAAE,UAAU;aAC3B,CAAC,CAAC,CAAC;YACJ,8DAA8D;YAC9D,2DAA2D;YAC3D,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC5C,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC;QACjC,CAAC,EACD,eAAe,KAAK,EAAE,EACtB,EAAE,WAAW,EAAE,iBAAiB,KAAK,CAAC,MAAM,SAAS,EAAE,CACxD;YACC,6DAA6D;aAC5D,KAAK,CAAC;YACL,QAAQ;YACR,SAAS,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;SACrD,CAAC;aACD,KAAK,EAAW;QACrB,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,cAAc;QACpB,uEAAuE;QACvE,sEAAsE;QACtE,wEAAwE;QACxE,uEAAuE;QACvE,wBAAwB;QACxB,QAAQ,EAAE,IAAI;KACf,CAAC;SAED,WAAW,CACV,cAAc,EACd,CAAC,KAAK,EAAE,EAAE;QACR,oEAAoE;QACpE,oEAAoE;QACpE,gEAAgE;QAChE,EAAE;QACF,kEAAkE;QAClE,kEAAkE;QAClE,uBAAuB;QACvB,MAAM,OAAO,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC,MAAM,CAAS,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;YAC1E,MAAM,CAAC,GAAI,MAA4C,EAAE,OAAO,CAAC;YACjE,OAAO,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC,EAAE,CAAC,CAAC,CAAC;QACN,KAAK,CAAC,aAAa,GAAG,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC,GAAG,OAAO,CAAC;IAC7D,CAAC,EACD,cAAc,EACd,sCAAsC,CACvC;QAED,oEAAoE;QACpE,uEAAuE;QACvE,uEAAuE;QACvE,sBAAsB;SACrB,kBAAkB,CACjB,eAAe,EACf,CAAC,CAAC,KAA6B,EAAE,EAAE,CACjC,MAAM,CACJ,KAAK,EACL;QACE;YACE,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC;YAChD,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,sCAAsC;SAC9C;KACF,EACD,cAAc,CACf,CAAU,EACb,sBAAsB,EACtB,8CAA8C,CAC/C;SACA,iBAAiB,CAAC,cAAc,EAAE,aAAa,EAAE,IAAa,EAAE,wBAAwB,CAAC;SACzF,MAAM,CAAC,aAAa,CAAC;SACrB,iBAAiB,CAAC,cAAc,EAAE,aAAa,EAAE,IAAa,EAAE,yBAAyB,CAAC;SAC1F,GAAG,EAAE;SAEL,WAAW,CACV,gBAAgB,EAChB,KAAK,EAAE,KAAK,EAAE,EAAE;QACd,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;YACtC,MAAM,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YACtC,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;QACD,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC;IAC/B,CAAC,EACD,gBAAgB,EAChB,qDAAqD,CACtD;SAEA,WAAW,CACV,QAAQ,EACR,CAAC,KAAK,EAAE,EAAE;QACR,kEAAkE;QAClE,6DAA6D;QAC7D,KAAK,CAAC,MAAM,GAAG;YACb,UAAU,EAAE,KAAK,CAAC,eAAe,IAAI,CAAC;YACtC,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM;YACtC,MAAM,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM;YACnC,QAAQ,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC;YAClC,OAAO,EAAE,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM;YACpC,OAAO,EAAE,KAAK,CAAC,YAAY,IAAI,CAAC;YAChC,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,EAAE;YAC1B,SAAS,EAAE,KAAK,CAAC,SAAS,IAAI,EAAE;YAChC,mBAAmB,EAAE,WAAW;YAChC,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SAClC,CAAC;IACJ,CAAC,EACD,QAAQ,EACR,uDAAuD,CACxD;SACA,KAAK,EAAE,CACX,CAAC;AACJ,CAAC;AAED,MAAM,IAAI,GAAG,GAAS,EAAE;IACtB,8DAA8D;AAChE,CAAC,CAAC;AAEF,oEAAoE;AACpE,SAAS,gBAAgB,CAAC,MAAc,EAAE,KAA6B;IACrE,4EAA4E;IAC5E,4EAA4E;IAC5E,yEAAyE;IACzE,uEAAuE;IACvE,oCAAoC;IACpC,OAAO,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,KAAK,CAAI,KAAmB,EAAE,IAAY;IACjD,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACrF,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/esm/rag/indexFolder.d.ts

@@ -0,0 +1,40 @@
+/**
+ * indexFolder — the one-call version, over the same pieces.
+ *
+ * Pattern: sugar. It composes `indexCorpus` and adds nothing the pieces do not
+ *          already do — which is the test a convenience has to pass here: if
+ *          it could not be written by a consumer in five lines using the
+ *          public API, it is hiding something instead of shortening it.
+ * Role:    rag/ layer.
+ * Emits:   whatever `indexCorpus` emits.
+ *
+ * The only judgement it adds is the splitter default, and it is the judgement
+ * you would make yourself: a folder of Markdown gets `byHeading` (the document
+ * declares its own sections), anything else gets `byParagraph`. Name a
+ * `splitter` and that guess is not made.
+ *
+ * @example
+ * ```ts
+ * const report = await indexFolder('./docs', {
+ *   to: sqliteVectorStore({ file: './corpus.db' }),
+ *   embedder: staticEmbedder(),
+ * });
+ * console.log(`${report.embedded} embedded, ${report.skipped} skipped`);
+ * ```
+ */
+import { type IndexCorpusConfig } from './indexCorpus.js';
+import type { IndexReport } from './types.js';
+export interface IndexFolderOptions extends Omit<IndexCorpusConfig, 'source' | 'store' | 'splitter'> {
+    /** The store to index into. Named `to` because that is what it reads as. */
+    readonly to: IndexCorpusConfig['store'];
+    /** Extensions to include, with the dot. Default: everything a loader claims. */
+    readonly include?: readonly string[];
+    /** Descend into subdirectories. Default true. */
+    readonly recursive?: boolean;
+    /**
+     * How to cut. Omitted, a folder that is mostly Markdown gets `byHeading()`
+     * and anything else gets `byParagraph()`.
+     */
+    readonly splitter?: IndexCorpusConfig['splitter'];
+}
+export declare function indexFolder(dir: string, options: IndexFolderOptions): Promise<IndexReport>;

package/dist/esm/rag/indexFolder.js

@@ -0,0 +1,42 @@
+/**
+ * indexFolder — the one-call version, over the same pieces.
+ *
+ * Pattern: sugar. It composes `indexCorpus` and adds nothing the pieces do not
+ *          already do — which is the test a convenience has to pass here: if
+ *          it could not be written by a consumer in five lines using the
+ *          public API, it is hiding something instead of shortening it.
+ * Role:    rag/ layer.
+ * Emits:   whatever `indexCorpus` emits.
+ *
+ * The only judgement it adds is the splitter default, and it is the judgement
+ * you would make yourself: a folder of Markdown gets `byHeading` (the document
+ * declares its own sections), anything else gets `byParagraph`. Name a
+ * `splitter` and that guess is not made.
+ *
+ * @example
+ * ```ts
+ * const report = await indexFolder('./docs', {
+ *   to: sqliteVectorStore({ file: './corpus.db' }),
+ *   embedder: staticEmbedder(),
+ * });
+ * console.log(`${report.embedded} embedded, ${report.skipped} skipped`);
+ * ```
+ */
+import { indexCorpus } from './indexCorpus.js';
+import { byHeading } from './splitters/byHeading.js';
+import { byParagraph } from './splitters/byParagraph.js';
+export async function indexFolder(dir, options) {
+    const { to, include, recursive, splitter, ...rest } = options;
+    const looksLikeMarkdown = include === undefined || include.some((e) => e === '.md' || e === '.markdown');
+    return indexCorpus({
+        ...rest,
+        source: {
+            dir,
+            ...(include && { include }),
+            ...(recursive !== undefined && { recursive }),
+        },
+        store: to,
+        splitter: splitter ?? (looksLikeMarkdown ? byHeading() : byParagraph()),
+    });
+}
+//# sourceMappingURL=indexFolder.js.map

package/dist/esm/rag/indexFolder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexFolder.js","sourceRoot":"","sources":["../../../src/rag/indexFolder.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,OAAO,EAAE,WAAW,EAA0B,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAkBzD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAW,EAAE,OAA2B;IACxE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC;IAC9D,MAAM,iBAAiB,GACrB,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,WAAW,CAAC,CAAC;IAEjF,OAAO,WAAW,CAAC;QACjB,GAAG,IAAI;QACP,MAAM,EAAE;YACN,GAAG;YACH,GAAG,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,CAAC;YAC3B,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,CAAC;SAC9C;QACD,KAAK,EAAE,EAAE;QACT,QAAQ,EAAE,QAAQ,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;KACxE,CAAC,CAAC;AACL,CAAC"}

package/dist/esm/rag/loadDocuments.d.ts

@@ -0,0 +1,34 @@
+import type { DocumentLoader, DocumentSource, FailedDocument, LoadedDocument } from './types.js';
+export interface LoadDocumentsOptions {
+    /**
+     * Loaders to route with, consulted BEFORE the built-ins. Passing one that
+     * claims `.md` overrides the Markdown loader without editing a registry.
+     */
+    readonly loaders?: readonly DocumentLoader[];
+    /**
+     * Skip files larger than this many bytes, recording them in `failed[]`.
+     * Default 25 MB — a corpus is other people's directories, and a stray
+     * database dump should be reported rather than read into memory.
+     */
+    readonly maxBytes?: number;
+}
+export interface LoadDocumentsResult {
+    readonly documents: readonly LoadedDocument[];
+    readonly failed: readonly FailedDocument[];
+    /** How many files the source turned up, before any of them were read. */
+    readonly discovered: number;
+}
+/**
+ * Read a source into documents.
+ *
+ * @throws TypeError when the source names more than one mode, or none.
+ * @throws when a named directory or file cannot be reached at all.
+ *
+ * @example
+ * ```ts
+ * const { documents, failed } = await loadDocuments({ dir: './docs' });
+ * const { documents } = await loadDocuments({ files: ['./a.md', './b.pdf'] });
+ * const { documents } = await loadDocuments({ text: 'inline', uri: 'note.md' });
+ * ```
+ */
+export declare function loadDocuments(source: DocumentSource, options?: LoadDocumentsOptions): Promise<LoadDocumentsResult>;

package/dist/esm/rag/loadDocuments.js

@@ -0,0 +1,160 @@
+/**
+ * loadDocuments — a folder (or a list, or a string) becomes documents.
+ *
+ * Pattern: routing over the `DocumentLoader` port.
+ * Role:    rag/ layer. Step one of the pipeline, and the only step that
+ *          touches the filesystem.
+ * Emits:   N/A — `indexCorpus` wraps this in a stage and records what it did.
+ *
+ * ── The source is a union because the modes exclude ─────────────────────────
+ * `{ dir }`, `{ files }` and `{ text, uri }` are three different questions,
+ * not three optional fields. A call naming a directory AND an explicit file
+ * list is a contradiction — which wins? — so the type refuses it at the
+ * keystroke, and the function refuses it again at runtime for JavaScript
+ * callers and casts.
+ *
+ * ── A failure is a record, not an exception ─────────────────────────────────
+ * One unreadable file out of two hundred must not lose the other 199. Failures
+ * come back in `failed[]` naming the uri and the reason, and the indexing
+ * report carries them. The exception to that is a failure that means the whole
+ * call was wrong — a directory that does not exist, a contradictory source —
+ * which throws, because retrying the other zero files is not a recovery.
+ */
+import { lazyRequire } from '../lib/lazyRequire.js';
+import { sha256 } from './hash.js';
+import { DEFAULT_LOADERS } from './loaders/index.js';
+const DEFAULT_MAX_BYTES = 25 * 1024 * 1024;
+/**
+ * Read a source into documents.
+ *
+ * @throws TypeError when the source names more than one mode, or none.
+ * @throws when a named directory or file cannot be reached at all.
+ *
+ * @example
+ * ```ts
+ * const { documents, failed } = await loadDocuments({ dir: './docs' });
+ * const { documents } = await loadDocuments({ files: ['./a.md', './b.pdf'] });
+ * const { documents } = await loadDocuments({ text: 'inline', uri: 'note.md' });
+ * ```
+ */
+export async function loadDocuments(source, options = {}) {
+    const loaders = [...(options.loaders ?? []), ...DEFAULT_LOADERS];
+    const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
+    refuseAmbiguousSource(source);
+    // The inline arm never touches the filesystem, so it is answered first and
+    // a browser-side caller can use it without `node:fs` ever being resolved.
+    if ('text' in source && source.text !== undefined) {
+        const bytes = new TextEncoder().encode(source.text);
+        return {
+            discovered: 1,
+            failed: [],
+            documents: [
+                {
+                    uri: source.uri,
+                    text: source.text,
+                    contentHash: sha256(bytes),
+                    bytes: bytes.length,
+                    loader: 'inline',
+                },
+            ],
+        };
+    }
+    const fs = lazyRequire('node:fs');
+    const path = lazyRequire('node:path');
+    const paths = 'dir' in source && source.dir !== undefined
+        ? walk(fs, path, source.dir, source.recursive ?? true, source.include, loaders)
+        : [...(source.files ?? [])];
+    const documents = [];
+    const failed = [];
+    for (const filePath of paths) {
+        try {
+            const stat = fs.statSync(filePath);
+            if (stat.size > maxBytes) {
+                failed.push({
+                    uri: filePath,
+                    reason: `file is ${stat.size} bytes, over the ${maxBytes}-byte limit (raise maxBytes to include it)`,
+                });
+                continue;
+            }
+            const loader = routeTo(loaders, path.extname(filePath).toLowerCase());
+            if (loader === undefined) {
+                failed.push({
+                    uri: filePath,
+                    reason: `no loader claims '${path.extname(filePath) || '(no extension)'}' — pass one in \`loaders\``,
+                });
+                continue;
+            }
+            const bytes = new Uint8Array(fs.readFileSync(filePath));
+            const draft = await loader.load({ uri: filePath, bytes, mtimeMs: stat.mtimeMs });
+            documents.push({
+                uri: filePath,
+                text: draft.text,
+                ...(draft.pages && { pages: draft.pages }),
+                contentHash: sha256(bytes),
+                bytes: bytes.length,
+                mtimeMs: stat.mtimeMs,
+                loader: loader.name,
+            });
+        }
+        catch (err) {
+            // One bad file does not lose the other 199.
+            failed.push({ uri: filePath, reason: err instanceof Error ? err.message : String(err) });
+        }
+    }
+    return { documents, failed, discovered: paths.length };
+}
+/**
+ * Refuse a source that names more than one mode, or none.
+ *
+ * The type already refuses it; this catches JavaScript callers and casts, and
+ * names what was actually written rather than failing later on a path that is
+ * `undefined`.
+ */
+function refuseAmbiguousSource(source) {
+    const named = ['dir', 'files', 'text'].filter((key) => source[key] !== undefined);
+    if (named.length === 1) {
+        if (named[0] === 'text' && !source.uri) {
+            throw new TypeError('loadDocuments({ text }) also needs `uri`: chunk ids are built from it, ' +
+                'so an inline document with no identifier produces chunks nothing can cite.');
+        }
+        return;
+    }
+    throw new TypeError(named.length === 0
+        ? 'loadDocuments: name exactly one source — `{ dir }`, `{ files }`, or `{ text, uri }`.'
+        : `loadDocuments: \`${named.join('` and `')}\` cannot be combined — they are different ` +
+            'sources, not a merge, and there is no sensible order between them. Call it once per source.');
+}
+/** First loader claiming the extension wins; caller-supplied ones come first. */
+function routeTo(loaders, extension) {
+    return loaders.find((loader) => loader.extensions.includes(extension));
+}
+/** Depth-first walk, filtered to extensions some loader claims. */
+function walk(fs, path, dir, recursive, include, loaders) {
+    const wanted = include
+        ? new Set(include.map((e) => e.toLowerCase()))
+        : new Set(loaders.flatMap((l) => l.extensions));
+    const out = [];
+    const visit = (current) => {
+        for (const dirent of fs.readdirSync(current, { withFileTypes: true })) {
+            const full = path.join(current, dirent.name);
+            if (dirent.isDirectory()) {
+                // Dot-directories are tooling, not corpus — `.git` alone would double
+                // the size of most walks and index nothing anyone wants retrieved.
+                if (!recursive || dirent.name.startsWith('.'))
+                    continue;
+                visit(full);
+                continue;
+            }
+            if (!dirent.isFile())
+                continue;
+            if (wanted.has(path.extname(dirent.name).toLowerCase()))
+                out.push(full);
+        }
+    };
+    visit(dir);
+    // Sorted so a corpus indexes in the same order on every machine — chunk ids
+    // are stable per document, but a stable ORDER makes two runs' reports
+    // diffable, which is most of what makes an incremental re-index readable.
+    return out.sort();
+}
+//# sourceMappingURL=loadDocuments.js.map

package/dist/esm/rag/loadDocuments.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"loadDocuments.js","sourceRoot":"","sources":["../../../src/rag/loadDocuments.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,WAAW,CAAC;AACnC,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAwBrD,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;AAE3C;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,MAAsB,EACtB,UAAgC,EAAE;IAElC,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,eAAe,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,iBAAiB,CAAC;IACvD,qBAAqB,CAAC,MAAM,CAAC,CAAC;IAE9B,2EAA2E;IAC3E,0EAA0E;IAC1E,IAAI,MAAM,IAAI,MAAM,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpD,OAAO;YACL,UAAU,EAAE,CAAC;YACb,MAAM,EAAE,EAAE;YACV,SAAS,EAAE;gBACT;oBACE,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,IAAI,EAAE,MAAM,CAAC,IAAI;oBACjB,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC;oBAC1B,KAAK,EAAE,KAAK,CAAC,MAAM;oBACnB,MAAM,EAAE,QAAQ;iBACjB;aACF;SACF,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,GAAG,WAAW,CAA2B,SAAS,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,WAAW,CAA6B,WAAW,CAAC,CAAC;IAElE,MAAM,KAAK,GACT,KAAK,IAAI,MAAM,IAAI,MAAM,CAAC,GAAG,KAAK,SAAS;QACzC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC;QAC/E,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;IAEhC,MAAM,SAAS,GAAqB,EAAE,CAAC;IACvC,MAAM,MAAM,GAAqB,EAAE,CAAC;IAEpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACnC,IAAI,IAAI,CAAC,IAAI,GAAG,QAAQ,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC;oBACV,GAAG,EAAE,QAAQ;oBACb,MAAM,EAAE,WAAW,IAAI,CAAC,IAAI,oBAAoB,QAAQ,4CAA4C;iBACrG,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;YACtE,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,MAAM,CAAC,IAAI,CAAC;oBACV,GAAG,EAAE,QAAQ;oBACb,MAAM,EAAE,qBACN,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,gBAC5B,6BAA6B;iBAC9B,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;YACxD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YACjF,SAAS,CAAC,IAAI,CAAC;gBACb,GAAG,EAAE,QAAQ;gBACb,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAC1C,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC;gBAC1B,KAAK,EAAE,KAAK,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,MAAM,EAAE,MAAM,CAAC,IAAI;aACpB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,4CAA4C;YAC5C,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3F,CAAC;IACH,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,qBAAqB,CAAC,MAAsB;IACnD,MAAM,KAAK,GAAI,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,CAAW,CAAC,MAAM,CACtD,CAAC,GAAG,EAAE,EAAE,CAAE,MAAkC,CAAC,GAAG,CAAC,KAAK,SAAS,CAChE,CAAC;IACF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,IAAI,CAAE,MAA2B,CAAC,GAAG,EAAE,CAAC;YAC7D,MAAM,IAAI,SAAS,CACjB,yEAAyE;gBACvE,4EAA4E,CAC/E,CAAC;QACJ,CAAC;QACD,OAAO;IACT,CAAC;IACD,MAAM,IAAI,SAAS,CACjB,KAAK,CAAC,MAAM,KAAK,CAAC;QAChB,CAAC,CAAC,sFAAsF;QACxF,CAAC,CAAC,oBAAoB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,6CAA6C;YACtF,6FAA6F,CAClG,CAAC;AACJ,CAAC;AAED,iFAAiF;AACjF,SAAS,OAAO,CACd,OAAkC,EAClC,SAAiB;IAEjB,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,mEAAmE;AACnE,SAAS,IAAI,CACX,EAA4B,EAC5B,IAAgC,EAChC,GAAW,EACX,SAAkB,EAClB,OAAsC,EACtC,OAAkC;IAElC,MAAM,MAAM,GAAG,OAAO;QACpB,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAC9C,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;IAElD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,CAAC,OAAe,EAAQ,EAAE;QACtC,KAAK,MAAM,MAAM,IAAI,EAAE,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YACtE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC7C,IAAI,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;gBACzB,sEAAsE;gBACtE,mEAAmE;gBACnE,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACxD,KAAK,CAAC,IAAI,CAAC,CAAC;gBACZ,SAAS;YACX,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;gBAAE,SAAS;YAC/B,IAAI,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1E,CAAC;IACH,CAAC,CAAC;IACF,KAAK,CAAC,GAAG,CAAC,CAAC;IACX,4EAA4E;IAC5E,sEAAsE;IACtE,0EAA0E;IAC1E,OAAO,GAAG,CAAC,IAAI,EAAE,CAAC;AACpB,CAAC"}