@seanhogg/builderforce-memory 2026.7.12 → 2026.9.27

package/dist/ingest/IngestionPipeline.js

@@ -0,0 +1,295 @@
+/**
+ * ingest/IngestionPipeline.ts — source → parse → chunk → embed → upsert.
+ *
+ * The one-off script that loads a corpus into a vector database is easy. What
+ * makes a pipeline PRODUCTION-grade is everything that happens on the second run:
+ *
+ *   • **Incremental sync.** Re-ingesting a 200k-document corpus must not re-embed
+ *     200k documents because three of them changed. Chunks carry a content hash;
+ *     with a manifest, unchanged hashes are skipped and vanished chunks deleted.
+ *     Embedding is the cost line, so this is the difference between a pipeline that
+ *     runs nightly and one that runs once.
+ *   • **Deterministic ids.** A chunk's id is derived from its source id and its
+ *     ordinal, so a re-run overwrites rather than duplicating. Random ids are why
+ *     corpora silently triple in size.
+ *   • **Batching and bounded concurrency.** Embedders and vector databases both
+ *     rate-limit; an unbounded `Promise.all` over a corpus is a self-inflicted 429.
+ *   • **Per-document failure isolation.** One malformed file fails that document
+ *     and is reported, rather than aborting the run.
+ *   • **Governance by construction.** Tenant, ACL and sensitivity are written onto
+ *     every chunk during ingest, because a retrieval-time ACL that depends on
+ *     ingest-time diligence is not an ACL.
+ */
+import { chunkText } from '../retrieval/chunk.js';
+import { estimateTokens } from '../telemetry/pricing.js';
+import { ACL_FIELD, ACL_PUBLIC, SENSITIVITY_FIELD, TENANT_FIELD, } from '../vectorstore/types.js';
+import { ParserRegistry } from './parsers.js';
+import { CHUNK_INDEX_FIELD, CONTENT_HASH_FIELD, PARSER_FIELD, SOURCE_ID_FIELD, TITLE_FIELD, URI_FIELD, } from './types.js';
+export class IngestionPipeline {
+    _store;
+    _embed;
+    _chunkOpts;
+    _parsers;
+    _manifest;
+    _strategy;
+    _embedBatch;
+    _upsertBatch;
+    _concurrency;
+    _embeddingModel;
+    _tracer;
+    constructor(opts) {
+        this._store = opts.store;
+        this._embed = opts.embed;
+        this._chunkOpts = opts.chunk ?? {};
+        this._parsers = opts.parsers instanceof ParserRegistry
+            ? opts.parsers
+            : new ParserRegistry(opts.parsers);
+        this._manifest = opts.manifest;
+        this._strategy = opts.syncStrategy ?? (opts.manifest ? 'diff' : 'replace');
+        this._embedBatch = Math.max(1, opts.embedBatchSize ?? 64);
+        this._upsertBatch = Math.max(1, opts.upsertBatchSize ?? 100);
+        this._concurrency = Math.max(1, opts.concurrency ?? 4);
+        this._embeddingModel = opts.embeddingModel ?? 'embedding';
+        this._tracer = opts.tracer;
+    }
+    async ingest(sources) {
+        const started = Date.now();
+        const report = {
+            documents: 0, sections: 0, chunks: 0, embedded: 0, upserted: 0,
+            skippedUnchanged: 0, deletedStale: 0, failures: [], durationMs: 0,
+        };
+        const root = this._tracer?.startSpan('ingest.run', {
+            kind: 'ingest',
+            attributes: { 'ingest.sources': sources.length, 'ingest.strategy': this._strategy },
+        });
+        if (root)
+            report.traceId = root.traceId;
+        await mapWithConcurrency(sources, this._concurrency, async (source) => {
+            try {
+                const result = await this._ingestOne(source, root);
+                report.documents += 1;
+                report.sections += result.sections;
+                report.chunks += result.chunks;
+                report.embedded += result.embedded;
+                report.upserted += result.upserted;
+                report.skippedUnchanged += result.skippedUnchanged;
+                report.deletedStale += result.deletedStale;
+            }
+            catch (err) {
+                // One bad document must not abort a corpus-wide run; it is reported
+                // with its id so the operator can requeue exactly that source.
+                report.failures.push({
+                    sourceId: source.id,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        });
+        report.durationMs = Date.now() - started;
+        root?.setAttributes({
+            'ingest.chunks': report.chunks,
+            'ingest.embedded': report.embedded,
+            'ingest.skipped_unchanged': report.skippedUnchanged,
+            'ingest.failures': report.failures.length,
+        });
+        root?.end(report.failures.length > 0 && report.documents === 0 ? 'error' : 'ok');
+        await this._tracer?.flush();
+        return report;
+    }
+    /** Removes every chunk a source produced — a GDPR/right-to-erasure primitive. */
+    async forget(sourceId) {
+        const deleted = await this._store.deleteByFilter({
+            op: 'eq', field: SOURCE_ID_FIELD, value: sourceId,
+        });
+        await this._manifest?.delete(sourceId);
+        return deleted;
+    }
+    async _ingestOne(source, parent) {
+        const span = this._tracer?.startSpan('ingest.document', {
+            kind: 'ingest',
+            ...(parent ? { parent } : {}),
+            attributes: { 'ingest.source_id': source.id, 'ingest.tenant': source.tenantId },
+        });
+        try {
+            const parser = this._parsers.resolve(source);
+            if (!parser)
+                throw new Error(`No parser accepts source "${source.id}".`);
+            const sections = parser.parse(source);
+            const base = this._baseMetadata(source, parser.id);
+            // Chunk index is global across sections so ids stay stable and unique
+            // even when a section is added in the middle of a document.
+            const pending = [];
+            let index = 0;
+            for (const section of sections) {
+                for (const chunk of chunkText(section.text, this._chunkOpts)) {
+                    const hash = hashText(chunk.text);
+                    const metadata = {
+                        ...base,
+                        ...(section.metadata ?? {}),
+                        [CHUNK_INDEX_FIELD]: index,
+                        [CONTENT_HASH_FIELD]: hash,
+                    };
+                    pending.push({
+                        record: { id: `${source.id}#${index}`, text: chunk.text, metadata },
+                        hash,
+                    });
+                    index += 1;
+                }
+            }
+            const { toWrite, toDelete, skipped } = await this._planSync(source.id, pending);
+            let embedded = 0;
+            for (let i = 0; i < toWrite.length; i += this._embedBatch) {
+                const batch = toWrite.slice(i, i + this._embedBatch);
+                const vectors = await this._embedBatchTraced(batch.map((r) => r.text), span);
+                batch.forEach((record, j) => {
+                    const vector = vectors[j];
+                    if (vector)
+                        record.vector = vector;
+                });
+                embedded += batch.length;
+            }
+            let upserted = 0;
+            for (let i = 0; i < toWrite.length; i += this._upsertBatch) {
+                const result = await this._store.upsert(toWrite.slice(i, i + this._upsertBatch));
+                upserted += result.upserted;
+            }
+            let deletedStale = 0;
+            if (toDelete.length > 0) {
+                await this._store.delete(toDelete);
+                deletedStale = toDelete.length;
+            }
+            await this._manifest?.set(source.id, pending.map(({ record, hash }) => ({ id: record.id, hash })));
+            span?.setAttributes({
+                'ingest.sections': sections.length,
+                'ingest.chunks': pending.length,
+                'ingest.embedded': embedded,
+                'ingest.skipped_unchanged': skipped,
+                'ingest.parser': parser.id,
+            });
+            span?.end('ok');
+            return {
+                sections: sections.length,
+                chunks: pending.length,
+                embedded,
+                upserted,
+                skippedUnchanged: skipped,
+                deletedStale,
+            };
+        }
+        catch (err) {
+            span?.fail(err);
+            throw err;
+        }
+    }
+    /**
+     * Decides what actually has to be written and deleted.
+     *
+     * `diff` compares content hashes against the manifest; `replace` clears the
+     * source's chunks first (correct without a manifest, at the cost of re-embedding);
+     * `append` writes everything and deletes nothing.
+     */
+    async _planSync(sourceId, pending) {
+        if (this._strategy === 'diff' && this._manifest) {
+            const previous = (await this._manifest.get(sourceId)) ?? [];
+            const previousByHash = new Map(previous.map((f) => [f.id, f.hash]));
+            const currentIds = new Set(pending.map((p) => p.record.id));
+            const toWrite = [];
+            let skipped = 0;
+            for (const { record, hash } of pending) {
+                if (previousByHash.get(record.id) === hash)
+                    skipped += 1;
+                else
+                    toWrite.push(record);
+            }
+            const toDelete = previous.map((f) => f.id).filter((id) => !currentIds.has(id));
+            return { toWrite, toDelete, skipped };
+        }
+        if (this._strategy === 'replace') {
+            await this._store.deleteByFilter({ op: 'eq', field: SOURCE_ID_FIELD, value: sourceId });
+        }
+        return { toWrite: pending.map((p) => p.record), toDelete: [], skipped: 0 };
+    }
+    async _embedBatchTraced(texts, parent) {
+        if (!this._tracer)
+            return this._embed(texts);
+        const span = this._tracer.startSpan('embed.batch', {
+            kind: 'embedding',
+            ...(parent ? { parent } : {}),
+            attributes: { 'embedding.batch_size': texts.length },
+        });
+        try {
+            const vectors = await this._embed(texts);
+            span.recordUsage({
+                model: this._embeddingModel,
+                inputTokens: texts.reduce((sum, t) => sum + estimateTokens(t), 0),
+                outputTokens: 0,
+                estimated: true,
+            });
+            span.end('ok');
+            return vectors;
+        }
+        catch (err) {
+            span.fail(err);
+            throw err;
+        }
+    }
+    _baseMetadata(source, parserId) {
+        const metadata = {
+            ...(source.metadata ?? {}),
+            [TENANT_FIELD]: source.tenantId,
+            // Defaulting to tenant-public keeps the ACL array NON-EMPTY, which is
+            // what makes `accessFilter` fail closed: a record with no acl matches
+            // nothing rather than everything.
+            [ACL_FIELD]: source.acl?.length ? source.acl : [ACL_PUBLIC],
+            [SOURCE_ID_FIELD]: source.id,
+            [PARSER_FIELD]: parserId,
+        };
+        if (source.sensitivity !== undefined)
+            metadata[SENSITIVITY_FIELD] = source.sensitivity;
+        if (source.title)
+            metadata[TITLE_FIELD] = source.title;
+        if (source.uri)
+            metadata[URI_FIELD] = source.uri;
+        return metadata;
+    }
+}
+/** In-process manifest — the default for a single-process or test deployment. */
+export class InMemoryIngestManifest {
+    _entries = new Map();
+    async get(sourceId) {
+        return this._entries.get(sourceId);
+    }
+    async set(sourceId, fingerprints) {
+        this._entries.set(sourceId, fingerprints);
+    }
+    async delete(sourceId) {
+        this._entries.delete(sourceId);
+    }
+    get size() { return this._entries.size; }
+}
+/**
+ * FNV-1a, 32-bit, rendered as hex.
+ *
+ * A content hash here answers "did this chunk change", not "was this chunk
+ * tampered with", so a fast non-cryptographic hash is the right tool — and it is
+ * synchronous, unlike WebCrypto, which keeps chunking a pure function.
+ */
+export function hashText(text) {
+    let hash = 0x811c9dc5;
+    for (let i = 0; i < text.length; i++) {
+        hash ^= text.charCodeAt(i);
+        hash = Math.imul(hash, 0x01000193) >>> 0;
+    }
+    return hash.toString(16).padStart(8, '0');
+}
+/** Bounded-concurrency map — the reason a large corpus does not become a 429. */
+export async function mapWithConcurrency(items, limit, fn) {
+    let cursor = 0;
+    const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (cursor < items.length) {
+            const index = cursor++;
+            await fn(items[index], index);
+        }
+    });
+    await Promise.all(workers);
+}
+//# sourceMappingURL=IngestionPipeline.js.map

package/dist/ingest/IngestionPipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"IngestionPipeline.js","sourceRoot":"","sources":["../../src/ingest/IngestionPipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,SAAS,EAAqB,MAAM,uBAAuB,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EACH,SAAS,EACT,UAAU,EACV,iBAAiB,EACjB,YAAY,GAIf,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,EACH,iBAAiB,EACjB,kBAAkB,EAClB,YAAY,EACZ,eAAe,EACf,WAAW,EACX,SAAS,GAQZ,MAAM,YAAY,CAAC;AAwBpB,MAAM,OAAO,iBAAiB;IACT,MAAM,CAAc;IACpB,MAAM,CAAgB;IACtB,UAAU,CAAe;IACzB,QAAQ,CAAiB;IACzB,SAAS,CAA6B;IACtC,SAAS,CAAe;IACxB,WAAW,CAAS;IACpB,YAAY,CAAS;IACrB,YAAY,CAAS;IACrB,eAAe,CAAS;IACxB,OAAO,CAAqB;IAE7C,YAAY,IAA8B;QACtC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,YAAY,cAAc;YAClD,CAAC,CAAC,IAAI,CAAC,OAAO;YACd,CAAC,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC3E,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;QAC1D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,eAAe,IAAI,GAAG,CAAC,CAAC;QAC7D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,cAAc,IAAI,WAAW,CAAC;QAC1D,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAkC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAoB;YAC5B,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC;YAC9D,gBAAgB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC;SACpE,CAAC;QAEF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,YAAY,EAAE;YAC/C,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE,EAAE,gBAAgB,EAAE,OAAO,CAAC,MAAM,EAAE,iBAAiB,EAAE,IAAI,CAAC,SAAS,EAAE;SACtF,CAAC,CAAC;QACH,IAAI,IAAI;YAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAExC,MAAM,kBAAkB,CAAC,OAAO,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAClE,IAAI,CAAC;gBACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBACnD,MAAM,CAAC,SAAS,IAAI,CAAC,CAAC;gBACtB,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC/B,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;gBACnC,MAAM,CAAC,gBAAgB,IAAI,MAAM,CAAC,gBAAgB,CAAC;gBACnD,MAAM,CAAC,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;YAC/C,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACX,oEAAoE;gBACpE,+DAA+D;gBAC/D,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;oBACjB,QAAQ,EAAE,MAAM,CAAC,EAAE;oBACnB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC1D,CAAC,CAAC;YACP,CAAC;QACL,CAAC,CAAC,CAAC;QAEH,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QACzC,IAAI,EAAE,aAAa,CAAC;YAChB,eAAe,EAAE,MAAM,CAAC,MAAM;YAC9B,iBAAiB,EAAE,MAAM,CAAC,QAAQ;YAClC,0BAA0B,EAAE,MAAM,CAAC,gBAAgB;YACnD,iBAAiB,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM;SAC5C,CAAC,CAAC;QACH,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACjF,MAAM,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC;QAC5B,OAAO,MAAM,CAAC;IAClB,CAAC;IAED,iFAAiF;IACjF,KAAK,CAAC,MAAM,CAAC,QAAgB;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;YAC7C,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ;SACpD,CAAC,CAAC;QACH,MAAM,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QACvC,OAAO,OAAO,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,UAAU,CACpB,MAAsB,EACtB,MAA8C;QAK9C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,iBAAiB,EAAE;YACpD,IAAI,EAAE,QAAQ;YACd,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,UAAU,EAAE,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,EAAE,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE;SAClF,CAAC,CAAC;QAEH,IAAI,CAAC;YACD,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YAEzE,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;YAEnD,sEAAsE;YACtE,4DAA4D;YAC5D,MAAM,OAAO,GAAkD,EAAE,CAAC;YAClE,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAC7B,KAAK,MAAM,KAAK,IAAI,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC3D,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBAClC,MAAM,QAAQ,GAAmB;wBAC7B,GAAG,IAAI;wBACP,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC;wBAC3B,CAAC,iBAAiB,CAAC,EAAE,KAAK;wBAC1B,CAAC,kBAAkB,CAAC,EAAE,IAAI;qBAC7B,CAAC;oBACF,OAAO,CAAC,IAAI,CAAC;wBACT,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,MAAM,CAAC,EAAE,IAAI,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE;wBACnE,IAAI;qBACP,CAAC,CAAC;oBACH,KAAK,IAAI,CAAC,CAAC;gBACf,CAAC;YACL,CAAC;YAED,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAEhF,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;gBAC7E,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACxB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;oBAC1B,IAAI,MAAM;wBAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;gBACvC,CAAC,CAAC,CAAC;gBACH,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;YAC7B,CAAC;YAED,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACzD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;gBACjF,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC;YAChC,CAAC;YAED,IAAI,YAAY,GAAG,CAAC,CAAC;YACrB,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACnC,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC;YACnC,CAAC;YAED,MAAM,IAAI,CAAC,SAAS,EAAE,GAAG,CACrB,MAAM,CAAC,EAAE,EACT,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAoB,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CACjF,CAAC;YAEF,IAAI,EAAE,aAAa,CAAC;gBAChB,iBAAiB,EAAE,QAAQ,CAAC,MAAM;gBAClC,eAAe,EAAE,OAAO,CAAC,MAAM;gBAC/B,iBAAiB,EAAE,QAAQ;gBAC3B,0BAA0B,EAAE,OAAO;gBACnC,eAAe,EAAE,MAAM,CAAC,EAAE;aAC7B,CAAC,CAAC;YACH,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;YAEhB,OAAO;gBACH,QAAQ,EAAE,QAAQ,CAAC,MAAM;gBACzB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,QAAQ;gBACR,QAAQ;gBACR,gBAAgB,EAAE,OAAO;gBACzB,YAAY;aACf,CAAC;QACN,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAChB,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,SAAS,CACnB,QAAgB,EAChB,OAAsD;QAEtD,IAAI,IAAI,CAAC,SAAS,KAAK,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5D,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5D,MAAM,OAAO,GAAmB,EAAE,CAAC;YACnC,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,OAAO,EAAE,CAAC;gBACrC,IAAI,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,IAAI;oBAAE,OAAO,IAAI,CAAC,CAAC;;oBACpD,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;YACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YAC/E,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC1C,CAAC;QAED,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;QAC5F,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAC/E,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAC3B,KAAe,EACf,MAA8C;QAE9C,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAE7C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,aAAa,EAAE;YAC/C,IAAI,EAAE,WAAW;YACjB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC7B,UAAU,EAAE,EAAE,sBAAsB,EAAE,KAAK,CAAC,MAAM,EAAE;SACvD,CAAC,CAAC;QACH,IAAI,CAAC;YACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzC,IAAI,CAAC,WAAW,CAAC;gBACb,KAAK,EAAE,IAAI,CAAC,eAAe;gBAC3B,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACjE,YAAY,EAAE,CAAC;gBACf,SAAS,EAAE,IAAI;aAClB,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,OAAO,OAAO,CAAC;QACnB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACX,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACf,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC;IAEO,aAAa,CAAC,MAAsB,EAAE,QAAgB;QAC1D,MAAM,QAAQ,GAAmB;YAC7B,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC;YAC1B,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC,QAAQ;YAC/B,sEAAsE;YACtE,sEAAsE;YACtE,kCAAkC;YAClC,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC;YAC3D,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC,EAAE;YAC5B,CAAC,YAAY,CAAC,EAAE,QAAQ;SAC3B,CAAC;QACF,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS;YAAE,QAAQ,CAAC,iBAAiB,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC;QACvF,IAAI,MAAM,CAAC,KAAK;YAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;QACvD,IAAI,MAAM,CAAC,GAAG;YAAE,QAAQ,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC;QACjD,OAAO,QAAQ,CAAC;IACpB,CAAC;CACJ;AAED,iFAAiF;AACjF,MAAM,OAAO,sBAAsB;IACd,QAAQ,GAAG,IAAI,GAAG,EAA8B,CAAC;IAElE,KAAK,CAAC,GAAG,CAAC,QAAgB;QACtB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,QAAgB,EAAE,YAAgC;QACxD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAgB;QACzB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAED,IAAI,IAAI,KAAa,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;CACpD;AAED;;;;;;GAMG;AACH,MAAM,UAAU,QAAQ,CAAC,IAAY;IACjC,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC9C,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACpC,KAAmB,EACnB,KAAa,EACb,EAA6C;IAE7C,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC7E,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC;YACvB,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAM,EAAE,KAAK,CAAC,CAAC;QACvC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC/B,CAAC"}

package/dist/ingest/index.d.ts

@@ -0,0 +1,10 @@
+/**
+ * ingest — one pipeline for structured and unstructured sources, with incremental
+ * sync so a nightly re-run re-embeds only what changed.
+ */
+export { IngestionPipeline, InMemoryIngestManifest, hashText, mapWithConcurrency, } from './IngestionPipeline.js';
+export type { IngestionPipelineOptions } from './IngestionPipeline.js';
+export { ParserRegistry, BUILTIN_PARSERS, rowsParser, csvParser, jsonParser, markdownParser, htmlParser, plainTextParser, parseCsv, serializeRow, } from './parsers.js';
+export { SOURCE_ID_FIELD, CHUNK_INDEX_FIELD, CONTENT_HASH_FIELD, TITLE_FIELD, URI_FIELD, PARSER_FIELD, } from './types.js';
+export type { BatchEmbedder, ChunkFingerprint, DocumentParser, IngestManifest, IngestionReport, ParsedSection, SourceContent, SourceDocument, StructuredRow, SyncStrategy, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/ingest/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/ingest/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACH,iBAAiB,EACjB,sBAAsB,EACtB,QAAQ,EACR,kBAAkB,GACrB,MAAM,wBAAwB,CAAC;AAChC,YAAY,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,EACH,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,EACT,UAAU,EACV,cAAc,EACd,UAAU,EACV,eAAe,EACf,QAAQ,EACR,YAAY,GACf,MAAM,cAAc,CAAC;AAEtB,OAAO,EACH,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,YAAY,GACf,MAAM,YAAY,CAAC;AACpB,YAAY,EACR,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,eAAe,EACf,aAAa,EACb,aAAa,EACb,cAAc,EACd,aAAa,EACb,YAAY,GACf,MAAM,YAAY,CAAC"}

package/dist/ingest/index.js

@@ -0,0 +1,8 @@
+/**
+ * ingest — one pipeline for structured and unstructured sources, with incremental
+ * sync so a nightly re-run re-embeds only what changed.
+ */
+export { IngestionPipeline, InMemoryIngestManifest, hashText, mapWithConcurrency, } from './IngestionPipeline.js';
+export { ParserRegistry, BUILTIN_PARSERS, rowsParser, csvParser, jsonParser, markdownParser, htmlParser, plainTextParser, parseCsv, serializeRow, } from './parsers.js';
+export { SOURCE_ID_FIELD, CHUNK_INDEX_FIELD, CONTENT_HASH_FIELD, TITLE_FIELD, URI_FIELD, PARSER_FIELD, } from './types.js';
+//# sourceMappingURL=index.js.map

package/dist/ingest/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/ingest/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACH,iBAAiB,EACjB,sBAAsB,EACtB,QAAQ,EACR,kBAAkB,GACrB,MAAM,wBAAwB,CAAC;AAGhC,OAAO,EACH,cAAc,EACd,eAAe,EACf,UAAU,EACV,SAAS,EACT,UAAU,EACV,cAAc,EACd,UAAU,EACV,eAAe,EACf,QAAQ,EACR,YAAY,GACf,MAAM,cAAc,CAAC;AAEtB,OAAO,EACH,eAAe,EACf,iBAAiB,EACjB,kBAAkB,EAClB,WAAW,EACX,SAAS,EACT,YAAY,GACf,MAAM,YAAY,CAAC"}

package/dist/ingest/parsers.d.ts

@@ -0,0 +1,53 @@
+/**
+ * ingest/parsers.ts — format handlers, as a registry.
+ *
+ * Each parser turns a source into sections and recovers whatever STRUCTURE the
+ * format carries, because structure that survives parsing becomes filterable
+ * metadata, and metadata is what makes retrieval precise:
+ *
+ *   • Markdown keeps its heading path, so a chunk knows it lives under
+ *     "Security › Data retention" and a citation can say so.
+ *   • Rows keep every typed column, so `status = 'open'` is a database predicate
+ *     rather than something the embedding has to imply.
+ *   • JSON keeps its pointer, so a value can be traced back to its exact location.
+ *
+ * A parser that returned only text would throw all of that away and force the
+ * retriever to recover it statistically — the single most common reason a RAG
+ * pilot answers plausibly and cites nothing verifiable.
+ */
+import type { DocumentParser, SourceDocument, StructuredRow } from './types.js';
+/**
+ * Serialises a row as `field: value` lines.
+ *
+ * Field names are kept in the text on purpose: an embedding of `"status: open"`
+ * sits near a question about open items, whereas a bare `"open"` sits near
+ * nothing useful. The typed values are ALSO emitted as metadata, so the same fact
+ * is both semantically searchable and exactly filterable.
+ */
+export declare function serializeRow(row: StructuredRow): string;
+export declare const rowsParser: DocumentParser;
+/**
+ * RFC 4180 subset: quoted fields, doubled quotes, embedded newlines and commas.
+ * Enough for the exports enterprises actually hand over, without a dependency.
+ */
+export declare function parseCsv(text: string, delimiter?: string): StructuredRow[];
+export declare const csvParser: DocumentParser;
+/**
+ * An array of objects is treated as rows (the common export shape); anything else
+ * is flattened to leaf paths so a value keeps its JSON pointer.
+ */
+export declare const jsonParser: DocumentParser;
+export declare const markdownParser: DocumentParser;
+export declare const htmlParser: DocumentParser;
+export declare const plainTextParser: DocumentParser;
+/** Order matters: the first parser that accepts a source handles it. */
+export declare const BUILTIN_PARSERS: readonly DocumentParser[];
+export declare class ParserRegistry {
+    private readonly _parsers;
+    constructor(parsers?: readonly DocumentParser[]);
+    /** Registers a parser at the FRONT, so it can override a builtin. */
+    register(parser: DocumentParser): this;
+    resolve(source: SourceDocument): DocumentParser | undefined;
+    list(): string[];
+}
+//# sourceMappingURL=parsers.d.ts.map

package/dist/ingest/parsers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parsers.d.ts","sourceRoot":"","sources":["../../src/ingest/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAiB,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAU/F;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,aAAa,GAAG,MAAM,CAKvD;AAED,eAAO,MAAM,UAAU,EAAE,cAexB,CAAC;AAIF;;;GAGG;AACH,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,SAAM,GAAG,aAAa,EAAE,CA4CvE;AAmBD,eAAO,MAAM,SAAS,EAAE,cAWvB,CAAC;AAIF;;;GAGG;AACH,eAAO,MAAM,UAAU,EAAE,cA2BxB,CAAC;AAmBF,eAAO,MAAM,cAAc,EAAE,cA4C5B,CAAC;AAIF,eAAO,MAAM,UAAU,EAAE,cA6BxB,CAAC;AAIF,eAAO,MAAM,eAAe,EAAE,cAO7B,CAAC;AAIF,wEAAwE;AACxE,eAAO,MAAM,eAAe,EAAE,SAAS,cAAc,EAOpD,CAAC;AAEF,qBAAa,cAAc;IACvB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmB;gBAEhC,OAAO,GAAE,SAAS,cAAc,EAAoB;IAIhE,qEAAqE;IACrE,QAAQ,CAAC,MAAM,EAAE,cAAc,GAAG,IAAI;IAKtC,OAAO,CAAC,MAAM,EAAE,cAAc,GAAG,cAAc,GAAG,SAAS;IAI3D,IAAI,IAAI,MAAM,EAAE;CAGnB"}