@kuralle-agents/rag 0.8.5 → 0.10.0

package/README.md

@@ -22,7 +22,7 @@ Everything between a raw document and a grounded agent response: chunk documents
 - **Vector stores** — `InMemoryVectorStore`; adapters in sibling packages
 - **Retrievers** — `VectorRetriever`, `HybridRetriever`, `FusionRetriever`, `MultiHopRetriever`, `createLLMRetriever`
 - **Rerankers** — `LLMReranker`, `CohereReranker`
-- **Search** — `BM25Index` (keyword search for hybrid retrieval)
+- **Search** — `KeywordIndex` contract: `BM25Index` (in-memory) and `Fts5KeywordIndex` (persistent SQLite FTS5; survives DO hibernation)
 - **KnowledgeFs** — read-only `FileSystem` over a vector store (`@kuralle-agents/rag/fs`); see [guides/KNOWLEDGEFS.md](./guides/KNOWLEDGEFS.md)
 - **Pipeline** — `RagPipeline`, `RetrievalQualityChecker`
 - **Cache** — `RetrievalCache`, `TurnCache`, `PredictivePreFetcher`
@@ -58,18 +58,51 @@ const results = await retriever.retrieve('What is Kuralle?');
 
 ## Hybrid retrieval
 
-`HybridRetriever` combines vector and BM25 keyword search with configurable weight:
+`FusionRetriever` fuses a BM25 keyword tier with vector similarity (weighted,
+min-max normalized). The keyword tier is any `KeywordIndex`: the in-memory
+`BM25Index`, or the persistent `Fts5KeywordIndex` (SQLite FTS5 — on Cloudflare,
+Durable Object SQLite supports FTS5, so the keyword tier survives hibernation
+with zero rebuild):
 
 ```ts
-import { HybridRetriever, BM25Index } from '@kuralle-agents/rag';
+import { FusionRetriever, BM25Index, Fts5KeywordIndex } from '@kuralle-agents/rag';
+
+const keywordIndex = new BM25Index();           // in-memory
+// const keywordIndex = new Fts5KeywordIndex({  // persistent (DO SQLite / bun:sqlite)
+//   sql: createSqlExecutor(ctx.storage.sql),   // from @kuralle-agents/cf-agent
+// });
+
+const retriever = new FusionRetriever({
+  keywordIndex,
+  vectorStore,
+  embedder,
+  indexName: 'docs',
+  bm25Weight: 0.3, // 70% vector, 30% keyword
+});
+```
+
+`HybridRetriever` is the generic alternative: it fuses any set of `Retriever`s
+with reciprocal rank fusion (`sources: [{ retriever, weight }]`).
 
-const bm25 = new BM25Index();
-// index documents into bm25...
+## Incremental ingest + embedder lock
 
-const retriever = new HybridRetriever({
-  vectorRetriever,
-  keywordRetriever: bm25,
-  alpha: 0.7,   // 0 = keyword-only, 1 = vector-only
+Give `RagPipeline` a persistent `IngestManifest` and it (a) skips unchanged
+documents on re-ingest (SHA-256 content hash — zero embed calls for a stable
+corpus), (b) cleans up stale chunks of changed documents, and (c) **locks the
+index to the embedding model that built it**: ingesting or querying with a
+different model — even one with the same dimension — throws instead of
+silently corrupting relevance.
+
+```ts
+import { RagPipeline, SqlIngestManifest, InMemoryIngestManifest } from '@kuralle-agents/rag';
+
+const pipeline = new RagPipeline({
+  embedder,
+  vectorStore,
+  chunker,
+  indexName: 'docs',
+  manifest: new SqlIngestManifest({ sql }), // DO SQLite / bun:sqlite; InMemoryIngestManifest for dev
+  keywordIndex,                             // optional: kept in sync at ingest
 });
 ```
 

package/dist/embedders/AiSdkEmbedder.d.ts

@@ -34,6 +34,8 @@ export declare class AiSdkEmbedder implements Embedder {
     private cachedDimension;
     constructor(options: AiSdkEmbedderOptions);
     get dimension(): number | undefined;
+    /** Stable model identity (`provider/modelId`) for the ingest-manifest embedder lock. */
+    get id(): string;
     embed(text: string): Promise<readonly number[]>;
     embedMany(texts: string[]): Promise<readonly (readonly number[])[]>;
 }

package/dist/embedders/AiSdkEmbedder.js

@@ -19,6 +19,12 @@ export class AiSdkEmbedder {
     get dimension() {
         return this.cachedDimension;
     }
+    /** Stable model identity (`provider/modelId`) for the ingest-manifest embedder lock. */
+    get id() {
+        if (typeof this.model === 'string')
+            return this.model;
+        return `${this.model.provider}/${this.model.modelId}`;
+    }
     async embed(text) {
         const result = await embed({
             model: this.model,

package/dist/fs/KnowledgeFs.d.ts

@@ -1,11 +1,21 @@
 import type { CpOptions, FileSystem, FileSystemDirent, FsStat, MkdirOptions, RmOptions } from '@kuralle-agents/core';
-import type { BM25Index } from '../search/BM25Index.js';
+import type { KeywordIndex } from '../search/KeywordIndex.js';
 import type { VectorStoreCore } from '../types.js';
 import type { KnowledgeAccessFilter } from './access.js';
 export interface KnowledgeFsOptions {
     store: VectorStoreCore;
     indexName: string;
-    bm25?: BM25Index;
+    /**
+     * Keyword index powering `search()` (the grep tier). Pass a fresh
+     * `BM25Index` (in-memory; seeded from the store on every `open()`) or a
+     * persistent `Fts5KeywordIndex` — a pre-populated persistent index is
+     * detected (`size > 0`) and `open()` skips seeding entirely, which is
+     * what makes a hibernated Durable Object wake with zero rebuild. A
+     * persistent index is kept in sync by the ingest path (e.g.
+     * `RagPipeline`'s `keywordIndex` option); call `clear()` on it to force
+     * a reseed on the next `open()`.
+     */
+    keywordIndex?: KeywordIndex;
     accessFilter?: KnowledgeAccessFilter;
     manifestKey?: string;
 }
@@ -17,7 +27,7 @@ export interface KnowledgeSearchHit {
 export declare class KnowledgeFs implements FileSystem {
     private readonly store;
     private readonly indexName;
-    private readonly bm25?;
+    private readonly keywordIndex?;
     private readonly accessFilter?;
     private readonly manifestKey;
     private tree;

package/dist/fs/KnowledgeFs.js

@@ -21,7 +21,7 @@ const utf8 = new TextEncoder();
 export class KnowledgeFs {
     store;
     indexName;
-    bm25;
+    keywordIndex;
     accessFilter;
     manifestKey;
     tree;
@@ -31,7 +31,7 @@ export class KnowledgeFs {
     constructor(opts) {
         this.store = opts.store;
         this.indexName = opts.indexName;
-        this.bm25 = opts.bm25;
+        this.keywordIndex = opts.keywordIndex;
         this.accessFilter = opts.accessFilter;
         this.manifestKey = opts.manifestKey ?? PATH_TREE_MANIFEST_ID;
     }
@@ -64,8 +64,10 @@ export class KnowledgeFs {
         chunkRecords = chunkRecords.filter((c) => allow(c.slug));
         this.tree = prunePathTree(buildPathTree(slugs), allow);
         this.chunksBySlug = groupChunksBySlug(chunkRecords);
-        if (this.bm25) {
-            this.bm25.add(chunkRecords.map((r) => ({
+        // A pre-populated persistent keyword index (e.g. Fts5KeywordIndex in
+        // DO SQLite surviving hibernation) is reused as-is — zero rebuild.
+        if (this.keywordIndex && this.keywordIndex.size === 0) {
+            this.keywordIndex.add(chunkRecords.map((r) => ({
                 id: `${r.slug}#${r.chunkIndex}`,
                 text: r.text,
             })));
@@ -102,17 +104,25 @@ export class KnowledgeFs {
         const limit = opts?.limit ?? 50;
         const root = opts?.path ? normalizeKnowledgePath(opts.path) : '/';
         let records = allChunkRecords(this.chunksBySlug).filter((r) => pathUnderRoot(r.slug, root));
-        if (this.bm25) {
-            const ranked = this.bm25.search(pattern, limit);
-            const candidateIds = new Set(ranked.map((hit) => hit.id));
-            return records
-                .filter((r) => candidateIds.has(`${r.slug}#${r.chunkIndex}`))
-                .slice(0, limit)
-                .map((r) => ({
-                slug: r.slug,
-                chunkIndex: r.chunkIndex,
-                text: r.text,
-            }));
+        if (this.keywordIndex) {
+            // Over-fetch so root filtering can't starve the limit, and return
+            // hits in BM25 rank order (not corpus order).
+            const ranked = this.keywordIndex.search(pattern, limit * 4);
+            const recordById = new Map(records.map((r) => [`${r.slug}#${r.chunkIndex}`, r]));
+            const hits = [];
+            for (const hit of ranked) {
+                const record = recordById.get(hit.id);
+                if (!record)
+                    continue;
+                hits.push({
+                    slug: record.slug,
+                    chunkIndex: record.chunkIndex,
+                    text: record.text,
+                });
+                if (hits.length >= limit)
+                    break;
+            }
+            return hits;
         }
         let re;
         try {

package/dist/index.d.ts

@@ -24,8 +24,10 @@ export { LLMReranker } from './rerankers/index.js';
 export type { LLMRerankerOptions } from './rerankers/index.js';
 export { CohereReranker } from './rerankers/index.js';
 export type { CohereRerankerOptions } from './rerankers/index.js';
-export { BM25Index } from './search/index.js';
-export type { BM25Document, BM25SearchResult, BM25IndexOptions } from './search/index.js';
+export { BM25Index, Fts5KeywordIndex, tokenizeKeywords } from './search/index.js';
+export type { BM25Document, BM25SearchResult, BM25IndexOptions, KeywordIndex, Fts5KeywordIndexOptions, } from './search/index.js';
+export { execSql, assertSqlIdentifier } from './sql.js';
+export type { SqlExecutor } from './sql.js';
 export { RetrievalCache } from './cache/index.js';
 export type { RetrievalCacheConfig } from './cache/index.js';
 export { TurnCache } from './cache/index.js';
@@ -35,5 +37,7 @@ export { KnowledgeCompiler } from './compiler/index.js';
 export type { KnowledgeCompilerConfig, CompilationResult } from './compiler/index.js';
 export { RagPipeline } from './pipeline/index.js';
 export type { RagPipelineOptions } from './pipeline/index.js';
+export { InMemoryIngestManifest, SqlIngestManifest, sha256Hex, } from './pipeline/index.js';
+export type { IngestManifest, IngestManifestData, IngestManifestDocEntry, SqlIngestManifestOptions, } from './pipeline/index.js';
 export { RetrievalQualityChecker } from './pipeline/index.js';
 export type { RetrievalQualityCheckerOptions, QualityCheckResult, QueryReformulator, } from './pipeline/index.js';

package/dist/index.js

@@ -22,7 +22,9 @@ export { MultiHopRetriever } from './retrievers/index.js';
 export { LLMReranker } from './rerankers/index.js';
 export { CohereReranker } from './rerankers/index.js';
 // Search
-export { BM25Index } from './search/index.js';
+export { BM25Index, Fts5KeywordIndex, tokenizeKeywords } from './search/index.js';
+// SQL executor contract (shared by Fts5KeywordIndex and SqlIngestManifest)
+export { execSql, assertSqlIdentifier } from './sql.js';
 // Cache
 export { RetrievalCache } from './cache/index.js';
 export { TurnCache } from './cache/index.js';
@@ -31,6 +33,7 @@ export { PredictivePreFetcher } from './cache/index.js';
 export { KnowledgeCompiler } from './compiler/index.js';
 // Pipeline
 export { RagPipeline } from './pipeline/index.js';
+export { InMemoryIngestManifest, SqlIngestManifest, sha256Hex, } from './pipeline/index.js';
 export { RetrievalQualityChecker } from './pipeline/index.js';
 // createVectorRetrievalTool now lives in @kuralle-agents/tools (C-8.4).
 // Under alpha's direct-removal posture the rag-side re-export is gone —

package/dist/pipeline/IngestManifest.d.ts

@@ -0,0 +1,57 @@
+/**
+ * IngestManifest — persistent record of what an index was built from.
+ *
+ * Stores two things per index:
+ *   1. The embedder identity (model id + dimension) that built the index,
+ *      so a different model can never silently query or extend it
+ *      ("provider lock").
+ *   2. A content hash + chunk-id list per ingested document, so unchanged
+ *      documents are skipped on re-ingest and stale chunks of changed
+ *      documents are cleaned up ("incremental indexing").
+ */
+import { type SqlExecutor } from '../sql.js';
+export interface IngestManifestDocEntry {
+    /** SHA-256 hex of the document text at last ingest. */
+    hash: string;
+    /** Vector-entry ids written for this document at last ingest. */
+    chunkIds: string[];
+}
+export interface IngestManifestData {
+    /** Identity of the embedder that built this index. */
+    embedder?: {
+        id?: string;
+        dimension?: number;
+    };
+    /** Per-document ingest record, keyed by document id. */
+    docs: Record<string, IngestManifestDocEntry>;
+}
+export interface IngestManifest {
+    load(indexName: string): Promise<IngestManifestData | undefined>;
+    save(indexName: string, data: IngestManifestData): Promise<void>;
+}
+/** SHA-256 hex via WebCrypto — identical on Node 18+, Bun, and Workers. */
+export declare function sha256Hex(text: string): Promise<string>;
+/** Process-lifetime manifest for dev/tests and long-lived Node servers. */
+export declare class InMemoryIngestManifest implements IngestManifest {
+    private readonly data;
+    load(indexName: string): Promise<IngestManifestData | undefined>;
+    save(indexName: string, data: IngestManifestData): Promise<void>;
+}
+export interface SqlIngestManifestOptions {
+    /** Tagged-template SQL executor (DO SQLite, bun:sqlite, better-sqlite3). */
+    sql: SqlExecutor;
+    /** Table name. Default: 'kuralle_rag_manifest'. */
+    tableName?: string;
+}
+/**
+ * SQLite-backed manifest. On Cloudflare, pass the Durable Object's
+ * `createSqlExecutor(ctx.storage.sql)` from `@kuralle-agents/cf-agent`;
+ * the manifest then survives hibernation alongside the session state.
+ */
+export declare class SqlIngestManifest implements IngestManifest {
+    private readonly sql;
+    private readonly table;
+    constructor(options: SqlIngestManifestOptions);
+    load(indexName: string): Promise<IngestManifestData | undefined>;
+    save(indexName: string, data: IngestManifestData): Promise<void>;
+}

package/dist/pipeline/IngestManifest.js

@@ -0,0 +1,52 @@
+/**
+ * IngestManifest — persistent record of what an index was built from.
+ *
+ * Stores two things per index:
+ *   1. The embedder identity (model id + dimension) that built the index,
+ *      so a different model can never silently query or extend it
+ *      ("provider lock").
+ *   2. A content hash + chunk-id list per ingested document, so unchanged
+ *      documents are skipped on re-ingest and stale chunks of changed
+ *      documents are cleaned up ("incremental indexing").
+ */
+import { assertSqlIdentifier, execSql } from '../sql.js';
+/** SHA-256 hex via WebCrypto — identical on Node 18+, Bun, and Workers. */
+export async function sha256Hex(text) {
+    const digest = await crypto.subtle.digest('SHA-256', new TextEncoder().encode(text));
+    return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, '0')).join('');
+}
+/** Process-lifetime manifest for dev/tests and long-lived Node servers. */
+export class InMemoryIngestManifest {
+    data = new Map();
+    async load(indexName) {
+        const entry = this.data.get(indexName);
+        return entry ? structuredClone(entry) : undefined;
+    }
+    async save(indexName, data) {
+        this.data.set(indexName, structuredClone(data));
+    }
+}
+/**
+ * SQLite-backed manifest. On Cloudflare, pass the Durable Object's
+ * `createSqlExecutor(ctx.storage.sql)` from `@kuralle-agents/cf-agent`;
+ * the manifest then survives hibernation alongside the session state.
+ */
+export class SqlIngestManifest {
+    sql;
+    table;
+    constructor(options) {
+        this.sql = options.sql;
+        this.table = assertSqlIdentifier(options.tableName ?? 'kuralle_rag_manifest');
+        execSql(this.sql, `CREATE TABLE IF NOT EXISTS ${this.table} (index_name TEXT PRIMARY KEY, data TEXT NOT NULL)`);
+    }
+    async load(indexName) {
+        const rows = execSql(this.sql, `SELECT data FROM ${this.table} WHERE index_name = ?`, [indexName]);
+        if (rows.length === 0)
+            return undefined;
+        return JSON.parse(rows[0].data);
+    }
+    async save(indexName, data) {
+        execSql(this.sql, `INSERT INTO ${this.table} (index_name, data) VALUES (?, ?) ` +
+            `ON CONFLICT(index_name) DO UPDATE SET data = excluded.data`, [indexName, JSON.stringify(data)]);
+    }
+}

package/dist/pipeline/RagPipeline.d.ts

@@ -1,4 +1,6 @@
 import type { Embedder, VectorStoreCore, Chunker, Reranker, Document, RetrievalResult, RetrievalOptions, Retriever } from '../types.js';
+import type { KeywordIndex } from '../search/KeywordIndex.js';
+import { type IngestManifest } from './IngestManifest.js';
 export interface RagPipelineOptions {
     /** Embedder for converting text to vectors. */
     embedder: Embedder;
@@ -24,6 +26,24 @@ export interface RagPipelineOptions {
     metric?: 'cosine' | 'euclidean' | 'dotproduct';
     /** Batch size for embedMany calls. Default: 100. */
     batchSize?: number;
+    /**
+     * Persistent ingest manifest. When provided, the pipeline:
+     *   - locks the index to the embedder that built it ({@link Embedder.id}
+     *     + dimension) and throws on mismatch at ingest AND retrieve time —
+     *     a different same-dimension model silently corrupts relevance;
+     *   - skips unchanged documents on re-ingest (SHA-256 content hash);
+     *   - deletes stale chunks of changed documents (when the store
+     *     supports admin deletes).
+     * Without a manifest, every ingest re-embeds everything and no
+     * embedder lock is enforced (previous behavior).
+     */
+    manifest?: IngestManifest;
+    /**
+     * Keyword index kept in sync with ingestion (the keyword tier for
+     * `FusionRetriever`/`HybridRetriever`). Chunk ids match the vector
+     * entry ids (`${docId}:${chunkId}`).
+     */
+    keywordIndex?: KeywordIndex;
 }
 /**
  * Convenience class that wires embedder, vector store, chunker,
@@ -41,6 +61,10 @@ export declare class RagPipeline implements Retriever {
     private readonly defaultTopK;
     private readonly metric;
     private readonly batchSize;
+    private readonly manifest?;
+    private readonly keywordIndex?;
+    private lockChecked;
+    private lockedDimension?;
     constructor(options: RagPipelineOptions);
     /**
      * Ensure the vector index exists, creating it if necessary.
@@ -52,8 +76,21 @@ export declare class RagPipeline implements Retriever {
      * surfaces later at query time via the backend's own error.
      */
     ensureIndex(): Promise<void>;
+    /**
+     * Throw if the manifest records a different embedder than this
+     * pipeline's. Two models with the SAME dimension still produce
+     * incompatible vector spaces — without this check the mismatch is
+     * silent and every query degrades to near-random.
+     */
+    private assertEmbedderLock;
+    private loadManifestData;
     /**
      * Ingest documents: chunk, embed, and store in the vector store.
+     *
+     * With a manifest configured, documents whose content hash is unchanged
+     * since the last ingest are skipped entirely (zero embed calls), and
+     * stale chunks of changed documents are removed from the vector store
+     * (admin stores) and the keyword index.
      */
     ingest(documents: Document[]): Promise<void>;
     /**

package/dist/pipeline/RagPipeline.js

@@ -1,4 +1,5 @@
 import { hasIndexAdmin } from '../types.js';
+import { sha256Hex, } from './IngestManifest.js';
 /**
  * Convenience class that wires embedder, vector store, chunker,
  * and optional reranker into a single ingestion + retrieval pipeline.
@@ -15,6 +16,10 @@ export class RagPipeline {
     defaultTopK;
     metric;
     batchSize;
+    manifest;
+    keywordIndex;
+    lockChecked = false;
+    lockedDimension;
     constructor(options) {
         this.embedder = options.embedder;
         this.vectorStore = options.vectorStore;
@@ -24,6 +29,8 @@ export class RagPipeline {
         this.defaultTopK = options.topK ?? 10;
         this.metric = options.metric ?? 'cosine';
         this.batchSize = options.batchSize ?? 100;
+        this.manifest = options.manifest;
+        this.keywordIndex = options.keywordIndex;
     }
     /**
      * Ensure the vector index exists, creating it if necessary.
@@ -53,12 +60,61 @@ export class RagPipeline {
             metric: this.metric,
         });
     }
+    /**
+     * Throw if the manifest records a different embedder than this
+     * pipeline's. Two models with the SAME dimension still produce
+     * incompatible vector spaces — without this check the mismatch is
+     * silent and every query degrades to near-random.
+     */
+    assertEmbedderLock(data) {
+        const locked = data.embedder;
+        if (!locked)
+            return;
+        const mismatchedId = locked.id !== undefined &&
+            this.embedder.id !== undefined &&
+            locked.id !== this.embedder.id;
+        const mismatchedDim = locked.dimension !== undefined &&
+            this.embedder.dimension !== undefined &&
+            locked.dimension !== this.embedder.dimension;
+        if (mismatchedId || mismatchedDim) {
+            throw new Error(`RagPipeline: index '${this.indexName}' was built with embedder ` +
+                `'${locked.id ?? 'unknown'}' (dimension ${locked.dimension ?? '?'}) but this ` +
+                `pipeline uses '${this.embedder.id ?? 'unknown'}' (dimension ` +
+                `${this.embedder.dimension ?? '?'}). Mixing embedding models silently corrupts ` +
+                `relevance. Either restore the original embedder, or re-index: clear the ` +
+                `index and its manifest entry, then ingest with the new model.`);
+        }
+    }
+    async loadManifestData() {
+        if (!this.manifest)
+            return undefined;
+        return this.manifest.load(this.indexName);
+    }
     /**
      * Ingest documents: chunk, embed, and store in the vector store.
+     *
+     * With a manifest configured, documents whose content hash is unchanged
+     * since the last ingest are skipped entirely (zero embed calls), and
+     * stale chunks of changed documents are removed from the vector store
+     * (admin stores) and the keyword index.
      */
     async ingest(documents) {
         await this.ensureIndex();
+        const data = await this.loadManifestData();
+        if (data) {
+            this.assertEmbedderLock(data);
+            this.lockedDimension = data.embedder?.dimension;
+            this.lockChecked = true;
+        }
+        const docRecords = { ...(data?.docs ?? {}) };
+        const skipped = [];
         for (const doc of documents) {
+            const hash = this.manifest ? await sha256Hex(doc.text) : '';
+            const previous = this.manifest ? docRecords[doc.id] : undefined;
+            if (previous && previous.hash === hash) {
+                skipped.push(doc);
+                continue;
+            }
             const chunks = this.chunker.chunk(doc.text);
             if (chunks.length === 0)
                 continue;
@@ -82,6 +138,45 @@ export class RagPipeline {
                 document: chunk.text,
             }));
             await this.vectorStore.upsert(this.indexName, entries);
+            const newIds = new Set(entries.map(e => e.id));
+            if (previous) {
+                const stale = previous.chunkIds.filter(id => !newIds.has(id));
+                if (stale.length > 0) {
+                    if (hasIndexAdmin(this.vectorStore)) {
+                        await this.vectorStore.deleteVectors(this.indexName, { ids: stale });
+                    }
+                    for (const id of stale)
+                        this.keywordIndex?.remove(id);
+                }
+            }
+            this.keywordIndex?.add(entries.map(e => ({ id: e.id, text: e.document })));
+            if (this.manifest) {
+                docRecords[doc.id] = { hash, chunkIds: [...newIds] };
+            }
+        }
+        // Restart recovery for a non-persistent keyword index: manifest-skip
+        // means skipped docs are never re-added, so after a process restart an
+        // in-memory BM25Index would come back empty and hybrid retrieval would
+        // silently degrade to vector-only. Re-seed by chunking alone — zero
+        // embed calls. A persistent index (Fts5KeywordIndex) is non-empty here
+        // and skips this entirely.
+        if (this.keywordIndex && this.keywordIndex.size === 0 && skipped.length > 0) {
+            for (const doc of skipped) {
+                const chunks = this.chunker.chunk(doc.text);
+                this.keywordIndex.add(chunks.map(c => ({ id: `${doc.id}:${c.id}`, text: c.text })));
+            }
+        }
+        if (this.manifest) {
+            // A skip-only ingest may not have embedded anything, leaving
+            // embedder.dimension undefined — preserve the recorded identity
+            // rather than erasing it.
+            await this.manifest.save(this.indexName, {
+                embedder: {
+                    id: this.embedder.id ?? data?.embedder?.id,
+                    dimension: this.embedder.dimension ?? data?.embedder?.dimension,
+                },
+                docs: docRecords,
+            });
         }
     }
     /**
@@ -89,9 +184,23 @@ export class RagPipeline {
      * Implements the Retriever interface.
      */
     async retrieve(query, options) {
+        if (this.manifest && !this.lockChecked) {
+            const data = await this.loadManifestData();
+            if (data) {
+                this.assertEmbedderLock(data);
+                this.lockedDimension = data.embedder?.dimension;
+            }
+            this.lockChecked = true;
+        }
         const topK = options?.topK ?? this.defaultTopK;
         // Use pre-computed query embedding when available to avoid double-embed cost
         const queryVector = options?.queryEmbedding ?? await this.embedder.embed(query);
+        if (this.lockedDimension !== undefined && queryVector.length !== this.lockedDimension) {
+            throw new Error(`RagPipeline: index '${this.indexName}' stores ${this.lockedDimension}-dimensional ` +
+                `vectors but the query embedding has ${queryVector.length} dimensions — the ` +
+                `embedding model differs from the one that built the index. Restore the ` +
+                `original embedder or re-index with the new model.`);
+        }
         const includeVectors = options?.includeEmbeddings ?? false;
         // Fetch more than topK if reranking, to give the reranker a wider pool
         const fetchK = this.reranker ? topK * 3 : topK;

package/dist/pipeline/RetrievalQualityChecker.d.ts

@@ -19,6 +19,12 @@ export interface QualityCheckResult {
     avgScore: number;
     /** 0-1 estimate of how well results cover the query. */
     coverageEstimate: number;
+    /**
+     * Estimated prompt-token cost of the result set (~chars/4). Retrieval
+     * pays for itself in tokens — track this to catch a retriever that is
+     * "accurate" only by flooding the context window.
+     */
+    estimatedTokens: number;
     /** Whether the query was reformulated and re-retrieved. */
     reformulated: boolean;
     /** The reformulated query string, if reformulation occurred. */

package/dist/pipeline/RetrievalQualityChecker.js

@@ -24,11 +24,12 @@ export class RetrievalQualityChecker {
      */
     assess(results) {
         if (results.length === 0) {
-            return { quality: 'low', topScore: 0, avgScore: 0, coverageEstimate: 0 };
+            return { quality: 'low', topScore: 0, avgScore: 0, coverageEstimate: 0, estimatedTokens: 0 };
         }
         const scores = results.map((r) => r.score ?? 0);
         const topScore = Math.max(...scores);
         const avgScore = scores.reduce((a, b) => a + b, 0) / scores.length;
+        const estimatedTokens = results.reduce((s, r) => s + Math.ceil(r.text.length / 4), 0);
         // Coverage: fraction of results above the medium threshold
         const aboveThreshold = scores.filter((s) => s >= this.mediumThreshold).length;
         const coverageEstimate = aboveThreshold / results.length;
@@ -42,7 +43,7 @@ export class RetrievalQualityChecker {
         else {
             quality = 'low';
         }
-        return { quality, topScore, avgScore, coverageEstimate };
+        return { quality, topScore, avgScore, coverageEstimate, estimatedTokens };
     }
     /**
      * Full quality check with optional reformulation for text agents.

package/dist/pipeline/index.d.ts

@@ -1,4 +1,6 @@
 export { RagPipeline } from './RagPipeline.js';
 export type { RagPipelineOptions } from './RagPipeline.js';
+export { InMemoryIngestManifest, SqlIngestManifest, sha256Hex, } from './IngestManifest.js';
+export type { IngestManifest, IngestManifestData, IngestManifestDocEntry, SqlIngestManifestOptions, } from './IngestManifest.js';
 export { RetrievalQualityChecker } from './RetrievalQualityChecker.js';
 export type { RetrievalQualityCheckerOptions, QualityCheckResult, QueryReformulator, } from './RetrievalQualityChecker.js';

package/dist/pipeline/index.js

@@ -1,2 +1,3 @@
 export { RagPipeline } from './RagPipeline.js';
+export { InMemoryIngestManifest, SqlIngestManifest, sha256Hex, } from './IngestManifest.js';
 export { RetrievalQualityChecker } from './RetrievalQualityChecker.js';

package/dist/retrievers/FusionRetriever.d.ts

@@ -1,8 +1,11 @@
 import type { Retriever, Reranker, RetrievalResult, RetrievalOptions, VectorStoreCore, Embedder } from '../types.js';
-import type { BM25Index } from '../search/BM25Index.js';
+import type { KeywordIndex } from '../search/KeywordIndex.js';
 export interface FusionRetrieverOptions {
-    /** The BM25 index for keyword search. */
-    bm25: BM25Index;
+    /**
+     * The keyword index for the BM25 tier — in-memory `BM25Index` or
+     * persistent `Fts5KeywordIndex`.
+     */
+    keywordIndex: KeywordIndex;
     /** The vector store for semantic search. */
     vectorStore: VectorStoreCore;
     /** The embedder to convert query text to vectors. */
@@ -36,7 +39,7 @@ export interface FusionRetrieverOptions {
  * An optional reranker can refine results after fusion.
  */
 export declare class FusionRetriever implements Retriever {
-    private readonly bm25;
+    private readonly keywordIndex;
     private readonly vectorStore;
     private readonly embedder;
     private readonly indexName;

package/dist/retrievers/FusionRetriever.js

@@ -13,7 +13,7 @@
  * An optional reranker can refine results after fusion.
  */
 export class FusionRetriever {
-    bm25;
+    keywordIndex;
     vectorStore;
     embedder;
     indexName;
@@ -22,7 +22,7 @@ export class FusionRetriever {
     defaultTopK;
     fetchK;
     constructor(options) {
-        this.bm25 = options.bm25;
+        this.keywordIndex = options.keywordIndex;
         this.vectorStore = options.vectorStore;
         this.embedder = options.embedder;
         this.indexName = options.indexName;
@@ -39,7 +39,7 @@ export class FusionRetriever {
         const queryVector = options?.queryEmbedding ?? await this.embedder.embed(query);
         // Run BM25 and vector search in parallel
         const [bm25Results, vectorResults] = await Promise.all([
-            Promise.resolve(this.bm25.search(query, fetchK)),
+            Promise.resolve(this.keywordIndex.search(query, fetchK)),
             this.vectorStore.query(this.indexName, {
                 queryVector,
                 topK: fetchK,

package/dist/search/BM25Index.d.ts

@@ -8,6 +8,7 @@
  * BM25 parameters default to the standard values from Robertson & Zaragoza
  * (2009): k1 = 1.2, b = 0.75.
  */
+import type { KeywordIndex } from './KeywordIndex.js';
 export interface BM25Document {
     /** Unique identifier for this document. */
     id: string;
@@ -34,7 +35,16 @@ export interface BM25IndexOptions {
      */
     b?: number;
 }
-export declare class BM25Index {
+/**
+ * Shared keyword tokenizer — exported so `Fts5KeywordIndex` queries are
+ * tokenized identically to `BM25Index` for ranking parity.
+ *
+ * Keeps combining marks (`\p{M}`) as token characters — Indic scripts
+ * (Tamil, Sinhala, Hindi, ...) encode vowel signs as combining marks, and
+ * treating them as separators splits every word mid-syllable.
+ */
+export declare function tokenizeKeywords(text: string): string[];
+export declare class BM25Index implements KeywordIndex {
     private readonly k1;
     private readonly b;
     /** Ordered list of indexed documents. Index = internal doc ordinal. */
@@ -48,7 +58,7 @@ export declare class BM25Index {
     /** Running sum of document lengths for avgdl computation. */
     private totalLength;
     constructor(options?: BM25IndexOptions);
-    /** Number of documents in the index. */
+    /** Number of active (non-removed) documents in the index. */
     get size(): number;
     /**
      * Add documents to the index. If a document with the same ID already

package/dist/search/BM25Index.js

@@ -22,13 +22,22 @@ const STOP_WORDS = new Set([
     'such', 'that', 'the', 'their', 'then', 'there', 'these', 'they',
     'this', 'to', 'was', 'will', 'with',
 ]);
-function tokenize(text) {
+/**
+ * Shared keyword tokenizer — exported so `Fts5KeywordIndex` queries are
+ * tokenized identically to `BM25Index` for ranking parity.
+ *
+ * Keeps combining marks (`\p{M}`) as token characters — Indic scripts
+ * (Tamil, Sinhala, Hindi, ...) encode vowel signs as combining marks, and
+ * treating them as separators splits every word mid-syllable.
+ */
+export function tokenizeKeywords(text) {
     return text
         .toLowerCase()
-        .replace(/[^\p{L}\p{N}\s]/gu, ' ')
+        .replace(/[^\p{L}\p{N}\p{M}\s]/gu, ' ')
         .split(/\s+/)
         .filter(t => t.length >= 2 && !STOP_WORDS.has(t));
 }
+const tokenize = tokenizeKeywords;
 export class BM25Index {
     k1;
     b;
@@ -46,9 +55,9 @@ export class BM25Index {
         this.k1 = options?.k1 ?? 1.2;
         this.b = options?.b ?? 0.75;
     }
-    /** Number of documents in the index. */
+    /** Number of active (non-removed) documents in the index. */
     get size() {
-        return this.docs.length;
+        return this.idToOrdinal.size;
     }
     /**
      * Add documents to the index. If a document with the same ID already

package/dist/search/Fts5KeywordIndex.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Fts5KeywordIndex — persistent BM25 keyword index over SQLite FTS5.
+ *
+ * The durable counterpart to the in-memory `BM25Index`: rows live in
+ * SQLite, so on Cloudflare (Durable Object SQLite supports the FTS5
+ * module) the keyword tier survives hibernation — a waking DO opens the
+ * index with ZERO rebuild instead of re-seeding the whole corpus.
+ *
+ * Ranking uses FTS5's built-in `bm25()` (lower = better; negated here so
+ * higher = better, matching `BM25Index`). Queries are tokenized with the
+ * same tokenizer as `BM25Index` and OR-combined, mirroring its
+ * sum-over-terms semantics.
+ */
+import { type SqlExecutor } from '../sql.js';
+import { type BM25Document, type BM25SearchResult } from './BM25Index.js';
+import type { KeywordIndex } from './KeywordIndex.js';
+export interface Fts5KeywordIndexOptions {
+    /**
+     * Tagged-template SQL executor. On Cloudflare:
+     * `createSqlExecutor(ctx.storage.sql)` from `@kuralle-agents/cf-agent`.
+     * On Node/Bun: a thin wrapper over `bun:sqlite` / `better-sqlite3`.
+     */
+    sql: SqlExecutor;
+    /** FTS5 virtual table name. Default: 'kuralle_keyword_index'. */
+    tableName?: string;
+    /**
+     * FTS5 tokenizer spec. Default: `unicode61 categories 'L* N* Co Mn Mc'`
+     * — unicode61 extended to keep combining marks as token characters, so
+     * Indic scripts (Tamil, Sinhala, Hindi, ...) are not split at vowel
+     * signs. Good for all space-delimited languages. For unsegmented
+     * languages (Chinese, Japanese, Thai) use 'trigram' (substring
+     * matching, query terms must be ≥3 chars).
+     */
+    tokenize?: string;
+}
+export declare class Fts5KeywordIndex implements KeywordIndex {
+    private readonly sql;
+    private readonly table;
+    constructor(options: Fts5KeywordIndexOptions);
+    get size(): number;
+    add(documents: BM25Document[]): void;
+    remove(id: string): boolean;
+    search(query: string, topK?: number): BM25SearchResult[];
+    clear(): void;
+}

package/dist/search/Fts5KeywordIndex.js

@@ -0,0 +1,61 @@
+/**
+ * Fts5KeywordIndex — persistent BM25 keyword index over SQLite FTS5.
+ *
+ * The durable counterpart to the in-memory `BM25Index`: rows live in
+ * SQLite, so on Cloudflare (Durable Object SQLite supports the FTS5
+ * module) the keyword tier survives hibernation — a waking DO opens the
+ * index with ZERO rebuild instead of re-seeding the whole corpus.
+ *
+ * Ranking uses FTS5's built-in `bm25()` (lower = better; negated here so
+ * higher = better, matching `BM25Index`). Queries are tokenized with the
+ * same tokenizer as `BM25Index` and OR-combined, mirroring its
+ * sum-over-terms semantics.
+ */
+import { assertSqlIdentifier, execSql } from '../sql.js';
+import { tokenizeKeywords } from './BM25Index.js';
+const DEFAULT_TOKENIZE = "unicode61 categories 'L* N* Co Mn Mc'";
+export class Fts5KeywordIndex {
+    sql;
+    table;
+    constructor(options) {
+        this.sql = options.sql;
+        this.table = assertSqlIdentifier(options.tableName ?? 'kuralle_keyword_index');
+        const tokenize = options.tokenize ?? DEFAULT_TOKENIZE;
+        if (!/^[A-Za-z0-9_*' ]+$/.test(tokenize)) {
+            throw new Error(`Invalid FTS5 tokenize spec: '${tokenize}'`);
+        }
+        execSql(this.sql, `CREATE VIRTUAL TABLE IF NOT EXISTS ${this.table} ` +
+            `USING fts5(doc_id UNINDEXED, content, tokenize = '${tokenize.replaceAll("'", "''")}')`);
+    }
+    get size() {
+        const rows = execSql(this.sql, `SELECT count(*) AS n FROM ${this.table}`);
+        return rows[0]?.n ?? 0;
+    }
+    add(documents) {
+        for (const doc of documents) {
+            execSql(this.sql, `DELETE FROM ${this.table} WHERE doc_id = ?`, [doc.id]);
+            execSql(this.sql, `INSERT INTO ${this.table} (doc_id, content) VALUES (?, ?)`, [doc.id, doc.text]);
+        }
+    }
+    remove(id) {
+        const existing = execSql(this.sql, `SELECT count(*) AS n FROM ${this.table} WHERE doc_id = ?`, [id]);
+        if ((existing[0]?.n ?? 0) === 0)
+            return false;
+        execSql(this.sql, `DELETE FROM ${this.table} WHERE doc_id = ?`, [id]);
+        return true;
+    }
+    search(query, topK = 10) {
+        const terms = tokenizeKeywords(query);
+        if (terms.length === 0)
+            return [];
+        // Tokens are alphanumeric after tokenization; quoting keeps FTS5 from
+        // interpreting them as query syntax.
+        const match = terms.map((t) => `"${t}"`).join(' OR ');
+        const rows = execSql(this.sql, `SELECT doc_id AS id, -bm25(${this.table}) AS score FROM ${this.table} ` +
+            `WHERE ${this.table} MATCH ? ORDER BY bm25(${this.table}) LIMIT ?`, [match, topK]);
+        return rows.map((r) => ({ id: r.id, score: r.score }));
+    }
+    clear() {
+        execSql(this.sql, `DELETE FROM ${this.table}`);
+    }
+}

package/dist/search/KeywordIndex.d.ts

@@ -0,0 +1,29 @@
+import type { BM25Document, BM25SearchResult } from './BM25Index.js';
+/**
+ * Contract for a BM25-ranked keyword index — the keyword tier of hybrid
+ * retrieval. Implementations: `BM25Index` (in-memory, rebuilt per process)
+ * and `Fts5KeywordIndex` (SQLite FTS5, persistent — survives Durable
+ * Object hibernation with zero rebuild).
+ *
+ * All methods are synchronous: the in-memory index is pure computation and
+ * DO SQLite's `sql.exec` is synchronous, so the contract stays sync to keep
+ * `FusionRetriever` / `KnowledgeFs` hot paths allocation-free.
+ */
+export interface KeywordIndex {
+    /** Number of documents in the index. */
+    readonly size: number;
+    /**
+     * Add documents. A document with an existing ID is overwritten.
+     */
+    add(documents: BM25Document[]): void;
+    /**
+     * Remove a document by ID. Returns true if it existed.
+     */
+    remove(id: string): boolean;
+    /**
+     * Search for documents matching the query, BM25-scored, descending.
+     */
+    search(query: string, topK?: number): BM25SearchResult[];
+    /** Remove all documents. */
+    clear(): void;
+}

package/dist/search/KeywordIndex.js

@@ -0,0 +1 @@
+export {};

package/dist/search/index.d.ts

@@ -1,2 +1,5 @@
-export { BM25Index } from './BM25Index.js';
+export { BM25Index, tokenizeKeywords } from './BM25Index.js';
 export type { BM25Document, BM25SearchResult, BM25IndexOptions } from './BM25Index.js';
+export type { KeywordIndex } from './KeywordIndex.js';
+export { Fts5KeywordIndex } from './Fts5KeywordIndex.js';
+export type { Fts5KeywordIndexOptions } from './Fts5KeywordIndex.js';

package/dist/search/index.js

@@ -1 +1,2 @@
-export { BM25Index } from './BM25Index.js';
+export { BM25Index, tokenizeKeywords } from './BM25Index.js';
+export { Fts5KeywordIndex } from './Fts5KeywordIndex.js';

package/dist/sql.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Minimal tagged-template SQL executor contract.
+ *
+ * Structurally identical to `SqlExecutor` in `@kuralle-agents/cf-agent`
+ * (Durable Object `ctx.storage.sql.exec` wrapped by `createSqlExecutor`),
+ * and trivially implementable over `bun:sqlite` / `better-sqlite3` for
+ * Node-side persistence. Declared here so `@kuralle-agents/rag` stays
+ * dependency-free of the Cloudflare package.
+ */
+export type SqlExecutor = <T = unknown>(strings: TemplateStringsArray, ...values: unknown[]) => T[];
+/**
+ * Run a dynamically-built query through a tagged-template `SqlExecutor`.
+ * `query` uses `?` for each positional parameter. Needed because table
+ * names cannot be bound parameters in SQL.
+ */
+export declare function execSql<T = unknown>(sql: SqlExecutor, query: string, params?: unknown[]): T[];
+/** Validate a SQL identifier (table name) — interpolated, never bound. */
+export declare function assertSqlIdentifier(name: string): string;

package/dist/sql.js

@@ -0,0 +1,20 @@
+/**
+ * Run a dynamically-built query through a tagged-template `SqlExecutor`.
+ * `query` uses `?` for each positional parameter. Needed because table
+ * names cannot be bound parameters in SQL.
+ */
+export function execSql(sql, query, params = []) {
+    const parts = query.split('?');
+    if (parts.length !== params.length + 1) {
+        throw new Error(`execSql: query has ${parts.length - 1} placeholders but ${params.length} params`);
+    }
+    const strings = Object.assign([...parts], { raw: [...parts] });
+    return sql(strings, ...params);
+}
+/** Validate a SQL identifier (table name) — interpolated, never bound. */
+export function assertSqlIdentifier(name) {
+    if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
+        throw new Error(`Invalid SQL identifier: '${name}'`);
+    }
+    return name;
+}

package/dist/types.d.ts

@@ -128,6 +128,14 @@ export interface Embedder {
      * May be undefined if the dimension is not known until the first embed() call.
      */
     readonly dimension?: number;
+    /**
+     * Stable identity of the underlying embedding model, e.g.
+     * `"openai.embedding/text-embedding-3-small"`. Used by the ingest
+     * manifest to lock an index to the model that built it — two different
+     * models of the same dimension produce incompatible vector spaces, so a
+     * silent swap degrades every query without erroring.
+     */
+    readonly id?: string;
 }
 /**
  * A single entry to upsert into a vector index.

package/package.json

@@ -6,7 +6,7 @@
     "url": "git+https://github.com/kuralle/kuralle-agents.git",
     "directory": "packages/kuralle-rag"
   },
-  "version": "0.8.5",
+  "version": "0.10.0",
   "description": "RAG primitives for Kuralle",
   "type": "module",
   "main": "dist/index.js",
@@ -30,7 +30,7 @@
     }
   },
   "dependencies": {
-    "@kuralle-agents/core": "0.8.5"
+    "@kuralle-agents/core": "0.10.0"
   },
   "peerDependencies": {
     "ai": "^6.0.0",
@@ -42,7 +42,7 @@
     "bun-types": "^1.3.0",
     "typescript": "^5.3.0",
     "zod": "^4.0.0",
-    "@kuralle-agents/fs": "0.8.5"
+    "@kuralle-agents/fs": "0.10.0"
   },
   "publishConfig": {
     "access": "public"
@@ -55,7 +55,7 @@
     "prebuild": "rm -rf dist",
     "build": "tsc -p tsconfig.json",
     "clean": "rm -rf dist",
-    "test": "bun test test/chunkers-tokens.test.ts test/vectorStores test/filters test/e2e-loaders.test.ts test/knowledgefs.test.ts test/knowledgefs-grep.test.ts test/knowledgefs-rbac.test.ts test/knowledgefs-agent.test.ts",
+    "test": "bun test test/chunkers-tokens.test.ts test/vectorStores test/filters test/e2e-loaders.test.ts test/knowledgefs.test.ts test/knowledgefs-grep.test.ts test/knowledgefs-rbac.test.ts test/knowledgefs-agent.test.ts test/pipeline-manifest.test.ts test/fts5-keyword-index.test.ts",
     "test:live": "bun test test/e2e-tasktype.test.ts test/e2e-real.test.ts"
   }
 }