@cogitator-ai/rag 0.1.10 → 0.3.0

package/dist/rag-pipeline.js

@@ -1,4 +1,11 @@
 import { RAGPipelineConfigSchema } from './schema.js';
+import { isChunkIndexer } from './retrievers/chunk-indexer.js';
+const STORE_CONCURRENCY = 16;
+function definedEntries(value) {
+    if (!value)
+        return {};
+    return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined));
+}
 export class RAGPipeline {
     config;
     deps;
@@ -10,24 +17,28 @@ export class RAGPipeline {
     async ingest(source) {
         const documents = await this.deps.loader.load(source);
         let totalChunks = 0;
+        let ingestedDocuments = 0;
         for (const doc of documents) {
             const chunks = await this.chunkDocument(doc);
-            if (chunks.length === 0)
-                continue;
-            const texts = chunks.map((c) => c.content);
-            const vectors = await this.deps.embeddingService.embedBatch(texts);
-            if (vectors.length !== chunks.length) {
-                throw new Error(`Embedding count mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
+            if (chunks.length > 0) {
+                const vectors = await this.deps.embeddingService.embedBatch(chunks.map((c) => c.content));
+                if (vectors.length !== chunks.length) {
+                    throw new Error(`Embedding count mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
+                }
+                await this.storeChunks(chunks, vectors, doc);
+                totalChunks += chunks.length;
+                this.stats.chunksStored += chunks.length;
             }
-            await Promise.all(chunks.map((chunk, i) => this.storeChunk(chunk, vectors[i], doc)));
-            totalChunks += chunks.length;
+            ingestedDocuments++;
+            this.stats.documentsIngested++;
         }
-        this.stats.documentsIngested += documents.length;
-        this.stats.chunksStored += totalChunks;
-        return { documents: documents.length, chunks: totalChunks };
+        return { documents: ingestedDocuments, chunks: totalChunks };
     }
     async query(text, options) {
-        const merged = { ...this.config.retrieval, ...options };
+        const merged = {
+            ...this.config.retrieval,
+            ...definedEntries(options),
+        };
         const results = await this.deps.retriever.retrieve(text, merged);
         this.stats.queriesProcessed++;
         if (this.deps.reranker && this.config.reranking?.enabled) {
@@ -39,24 +50,49 @@ export class RAGPipeline {
         return { ...this.stats };
     }
     async chunkDocument(doc) {
-        return this.deps.chunker.chunk(doc.content, doc.id);
+        const chunks = await this.deps.chunker.chunk(doc.content, doc.id);
+        return chunks.filter((chunk) => chunk.content.trim().length > 0);
+    }
+    async storeChunks(chunks, vectors, doc) {
+        let next = 0;
+        const worker = async () => {
+            while (next < chunks.length) {
+                const index = next++;
+                await this.storeChunk(chunks[index], vectors[index], doc);
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(STORE_CONCURRENCY, chunks.length) }, () => worker()));
     }
     async storeChunk(chunk, vector, doc) {
+        const metadata = {
+            ...doc.metadata,
+            ...chunk.metadata,
+            documentId: doc.id,
+            source: doc.source,
+            sourceType: doc.sourceType,
+            order: chunk.order,
+            startOffset: chunk.startOffset,
+            endOffset: chunk.endOffset,
+        };
         const result = await this.deps.embeddingAdapter.addEmbedding({
             sourceId: chunk.id,
             sourceType: 'document',
             vector,
             content: chunk.content,
-            metadata: {
-                ...chunk.metadata,
-                documentId: doc.id,
-                source: doc.source,
-                order: chunk.order,
-            },
+            metadata,
         });
         if (!result.success) {
             throw new Error(result.error);
         }
+        if (isChunkIndexer(this.deps.retriever)) {
+            this.deps.retriever.indexChunk({
+                embeddingId: result.data.id,
+                chunkId: chunk.id,
+                documentId: doc.id,
+                content: chunk.content,
+                metadata,
+            });
+        }
     }
 }
 //# sourceMappingURL=rag-pipeline.js.map

package/dist/rag-pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAWnD,MAAM,OAAO,WAAW;IACL,MAAM,CAAoB;IAC1B,IAAI,CAAkB;IAC/B,KAAK,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IAE/E,YAAY,MAAyB,EAAE,IAAqB,EAAE,aAAa,GAAG,KAAK;QACjF,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7E,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAc;QACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAElC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAEnE,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,SAAS,CACtF,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;YAErF,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,SAAS,CAAC,MAAM,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,WAAW,CAAC;QAEvC,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAkC;QAC1D,MAAM,MAAM,GAA6B,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,GAAG,OAAO,EAAE,CAAC;QAClF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,GAAgB;QAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,MAAgB,EAAE,GAAgB;QAC/E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YAC3D,QAAQ,EAAE,KAAK,CAAC,EAAE;YAClB,UAAU,EAAE,UAAU;YACtB,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAW/D,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAE7B,SAAS,cAAc,CAAmB,KAAoB;IAC5D,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IACtB,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAe,CAAC;AACpG,CAAC;AAED,MAAM,OAAO,WAAW;IACL,MAAM,CAAoB;IAC1B,IAAI,CAAkB;IAC/B,KAAK,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IAE/E,YAAY,MAAyB,EAAE,IAAqB,EAAE,aAAa,GAAG,KAAK;QACjF,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7E,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAc;QACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;gBAE1F,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;oBACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,SAAS,CACtF,CAAC;gBACJ,CAAC;gBAED,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC7C,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC7B,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;YAC3C,CAAC;YACD,iBAAiB,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC;QACjC,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAkC;QAC1D,MAAM,MAAM,GAA6B;YACvC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS;YACxB,GAAG,cAAc,CAAC,OAAO,CAAC;SAC3B,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,GAAgB;QAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,MAAuB,EACvB,OAAmB,EACnB,GAAgB;QAEhB,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAE,EAAE,OAAO,CAAC,KAAK,CAAE,EAAE,GAAG,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CACnF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,MAAgB,EAAE,GAAgB;QAC/E,MAAM,QAAQ,GAA4B;YACxC,GAAG,GAAG,CAAC,QAAQ;YACf,GAAG,KAAK,CAAC,QAAQ;YACjB,UAAU,EAAE,GAAG,CAAC,EAAE;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YAC3D,QAAQ,EAAE,KAAK,CAAC,EAAE;YAClB,UAAU,EAAE,UAAU;YACtB,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ;SACT,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;gBAC7B,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;gBAC3B,OAAO,EAAE,KAAK,CAAC,EAAE;gBACjB,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}

package/dist/rerankers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/rerankers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAA0B,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,cAAc,EAA6B,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAA0B,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,cAAc,EAA6B,MAAM,sBAAsB,CAAC"}

package/dist/retrievers/chunk-indexer.d.ts

@@ -0,0 +1,19 @@
+import type { Retriever } from '@cogitator-ai/types';
+export interface IndexedChunk {
+    embeddingId: string;
+    chunkId: string;
+    documentId: string;
+    content: string;
+    metadata: Record<string, unknown>;
+}
+/**
+ * A retriever that maintains its own index alongside the vector store
+ * (e.g. a local BM25 index). `RAGPipeline.ingest()` calls `indexChunk()` for
+ * every stored chunk.
+ */
+export interface ChunkIndexer {
+    indexChunk(chunk: IndexedChunk): void;
+}
+export declare function isChunkIndexer(retriever: Retriever): retriever is Retriever & ChunkIndexer;
+export declare function resultSource(metadata: Record<string, unknown> | undefined, fallback: string): string;
+//# sourceMappingURL=chunk-indexer.d.ts.map

package/dist/retrievers/chunk-indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-indexer.d.ts","sourceRoot":"","sources":["../../src/retrievers/chunk-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAErD,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,CAAC;CACvC;AAED,wBAAgB,cAAc,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS,IAAI,SAAS,GAAG,YAAY,CAE1F;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,UAG3F"}

package/dist/retrievers/chunk-indexer.js

@@ -0,0 +1,8 @@
+export function isChunkIndexer(retriever) {
+    return typeof retriever.indexChunk === 'function';
+}
+export function resultSource(metadata, fallback) {
+    const source = metadata?.source;
+    return typeof source === 'string' ? source : fallback;
+}
+//# sourceMappingURL=chunk-indexer.js.map

package/dist/retrievers/chunk-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-indexer.js","sourceRoot":"","sources":["../../src/retrievers/chunk-indexer.ts"],"names":[],"mappings":"AAmBA,MAAM,UAAU,cAAc,CAAC,SAAoB;IACjD,OAAO,OAAQ,SAAmC,CAAC,UAAU,KAAK,UAAU,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAA6C,EAAE,QAAgB;IAC1F,MAAM,MAAM,GAAG,QAAQ,EAAE,MAAM,CAAC;IAChC,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxD,CAAC"}

package/dist/retrievers/create-retriever.d.ts

@@ -27,6 +27,7 @@ interface MultiQueryDeps {
     baseRetriever: Retriever;
     expandQuery: (query: string) => Promise<string[]>;
     topK?: number;
+    maxQueries?: number;
 }
 export type CreateRetrieverConfig = SimilarityDeps | MMRDeps | HybridDeps | MultiQueryDeps;
 export declare function createRetriever(config: CreateRetrieverConfig): Retriever;

package/dist/retrievers/create-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAMzD,UAAU,iBAAiB;IACzB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;CACpC;AAED,UAAU,cAAe,SAAQ,iBAAiB;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,OAAQ,SAAQ,iBAAiB;IACzC,QAAQ,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,aAAa,CAAC;IACxB,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,qBAAqB,GAAG,cAAc,GAAG,OAAO,GAAG,UAAU,GAAG,cAAc,CAAC;AAE3F,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,CAuCxE"}
+{"version":3,"file":"create-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAMzD,UAAU,iBAAiB;IACzB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;CACpC;AAED,UAAU,cAAe,SAAQ,iBAAiB;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,OAAQ,SAAQ,iBAAiB;IACzC,QAAQ,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,aAAa,CAAC;IACxB,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,qBAAqB,GAAG,cAAc,GAAG,OAAO,GAAG,UAAU,GAAG,cAAc,CAAC;AAE3F,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,CAwCxE"}

package/dist/retrievers/create-retriever.js

@@ -31,6 +31,7 @@ export function createRetriever(config) {
                 baseRetriever: config.baseRetriever,
                 expandQuery: config.expandQuery,
                 defaultTopK: config.topK,
+                defaultMaxQueries: config.maxQueries,
             });
         default: {
             const _exhaustive = config;

package/dist/retrievers/create-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"create-retriever.js","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAqC9D,MAAM,UAAU,eAAe,CAAC,MAA6B;IAC3D,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,YAAY;YACf,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,KAAK;YACR,OAAO,IAAI,YAAY,CAAC;gBACtB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,aAAa,EAAE,MAAM,CAAC,MAAM;gBAC5B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,OAAO,IAAI,eAAe,CAAC;gBACzB,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,cAAc,EAAE,MAAM,CAAC,OAAO;gBAC9B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,aAAa;YAChB,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,WAAW,EAAE,MAAM,CAAC,IAAI;aACzB,CAAC,CAAC;QAEL,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,MAAM,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"create-retriever.js","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAsCjE,MAAM,UAAU,eAAe,CAAC,MAA6B;IAC3D,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,YAAY;YACf,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,KAAK;YACR,OAAO,IAAI,YAAY,CAAC;gBACtB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,aAAa,EAAE,MAAM,CAAC,MAAM;gBAC5B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,OAAO,IAAI,eAAe,CAAC;gBACzB,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,cAAc,EAAE,MAAM,CAAC,OAAO;gBAC9B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,aAAa;YAChB,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,iBAAiB,EAAE,MAAM,CAAC,UAAU;aACrC,CAAC,CAAC;QAEL,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,MAAM,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/retrievers/hybrid-retriever.d.ts

@@ -1,17 +1,24 @@
 import type { Retriever, RetrievalConfig, RetrievalResult, HybridSearchWeights } from '@cogitator-ai/types';
 import type { HybridSearch } from '@cogitator-ai/memory';
+import { type ChunkIndexer, type IndexedChunk } from './chunk-indexer.js';
 export interface HybridRetrieverConfig {
     hybridSearch: HybridSearch;
     defaultWeights?: HybridSearchWeights;
     defaultTopK?: number;
     defaultThreshold?: number;
 }
-export declare class HybridRetriever implements Retriever {
+/**
+ * Vector + BM25 retrieval fused with reciprocal rank fusion. Chunks ingested through
+ * `RAGPipeline` are added to the HybridSearch keyword index automatically.
+ */
+export declare class HybridRetriever implements Retriever, ChunkIndexer {
     private readonly hybridSearch;
     private readonly defaultWeights?;
     private readonly defaultTopK;
     private readonly defaultThreshold;
+    private readonly indexed;
     constructor(config: HybridRetrieverConfig);
+    indexChunk(chunk: IndexedChunk): void;
     retrieve(query: string, options?: Partial<RetrievalConfig>): Promise<RetrievalResult[]>;
     private toRetrievalResult;
 }

package/dist/retrievers/hybrid-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,eAAe,EACf,mBAAmB,EAEpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEzD,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,YAAY,CAAC;IAC3B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,eAAgB,YAAW,SAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,qBAAqB;IAOnC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAgB7F,OAAO,CAAC,iBAAiB;CAiB1B"}
+{"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,eAAe,EACf,mBAAmB,EAEpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAgB,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAExF,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,YAAY,CAAC;IAC3B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD;;;GAGG;AACH,qBAAa,eAAgB,YAAW,SAAS,EAAE,YAAY;IAC7D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;gBAE/C,MAAM,EAAE,qBAAqB;IAOzC,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAK/B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAgB7F,OAAO,CAAC,iBAAiB;CAoB1B"}

package/dist/retrievers/hybrid-retriever.js

@@ -1,16 +1,26 @@
+import { resultSource } from './chunk-indexer.js';
 const DEFAULT_TOP_K = 10;
 const DEFAULT_THRESHOLD = 0.0;
+/**
+ * Vector + BM25 retrieval fused with reciprocal rank fusion. Chunks ingested through
+ * `RAGPipeline` are added to the HybridSearch keyword index automatically.
+ */
 export class HybridRetriever {
     hybridSearch;
     defaultWeights;
     defaultTopK;
     defaultThreshold;
+    indexed = new Map();
     constructor(config) {
         this.hybridSearch = config.hybridSearch;
         this.defaultWeights = config.defaultWeights;
         this.defaultTopK = config.defaultTopK ?? DEFAULT_TOP_K;
         this.defaultThreshold = config.defaultThreshold ?? DEFAULT_THRESHOLD;
     }
+    indexChunk(chunk) {
+        this.indexed.set(chunk.embeddingId, chunk);
+        this.hybridSearch.indexDocument(chunk.embeddingId, chunk.content);
+    }
     async retrieve(query, options) {
         const result = await this.hybridSearch.search({
             query,
@@ -25,16 +35,19 @@ export class HybridRetriever {
         return result.data.map((entry) => this.toRetrievalResult(entry));
     }
     toRetrievalResult(entry) {
-        const rawDocId = entry.metadata?.documentId;
-        const documentId = typeof rawDocId === 'string' ? rawDocId : entry.sourceId;
+        const known = this.indexed.get(entry.id);
+        const metadata = { ...known?.metadata, ...entry.metadata };
+        const rawDocId = metadata.documentId;
+        const chunkId = known?.chunkId ?? entry.sourceId;
+        const documentId = typeof rawDocId === 'string' ? rawDocId : chunkId;
         return {
-            chunkId: entry.sourceId,
+            chunkId,
             documentId,
             content: entry.content,
             score: entry.score,
-            source: entry.sourceType,
+            source: resultSource(metadata, entry.sourceType),
             metadata: {
-                ...entry.metadata,
+                ...metadata,
                 vectorScore: entry.vectorScore,
                 keywordScore: entry.keywordScore,
             },

package/dist/retrievers/hybrid-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAgBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,eAAe;IACT,YAAY,CAAe;IAC3B,cAAc,CAAuB;IACrC,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA6B;QACvC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC5C,KAAK;YACL,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE,IAAI,CAAC,cAAc;YAC5B,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAmB;QAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;aACjC;SACF,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAwC,MAAM,oBAAoB,CAAC;AASxF,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;GAGG;AACH,MAAM,OAAO,eAAe;IACT,YAAY,CAAe;IAC3B,cAAc,CAAuB;IACrC,WAAW,CAAS;IACpB,gBAAgB,CAAS;IACzB,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE3D,YAAY,MAA6B;QACvC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,UAAU,CAAC,KAAmB;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC5C,KAAK;YACL,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE,IAAI,CAAC,cAAc;YAC5B,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAmB;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC3D,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC;QACrC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,KAAK,CAAC,QAAQ,CAAC;QACjD,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;QAErE,OAAO;YACL,OAAO;YACP,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YAChD,QAAQ,EAAE;gBACR,GAAG,QAAQ;gBACX,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;aACjC;SACF,CAAC;IACJ,CAAC;CACF"}

package/dist/retrievers/index.d.ts

@@ -8,4 +8,6 @@ export { MultiQueryRetriever } from './multi-query-retriever.js';
 export type { MultiQueryRetrieverConfig } from './multi-query-retriever.js';
 export { createRetriever } from './create-retriever.js';
 export type { CreateRetrieverConfig } from './create-retriever.js';
+export { isChunkIndexer } from './chunk-indexer.js';
+export type { ChunkIndexer, IndexedChunk } from './chunk-indexer.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/retrievers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/retrievers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,YAAY,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,YAAY,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/retrievers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,YAAY,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,YAAY,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC"}

package/dist/retrievers/index.js

@@ -3,4 +3,5 @@ export { MMRRetriever } from './mmr-retriever.js';
 export { HybridRetriever } from './hybrid-retriever.js';
 export { MultiQueryRetriever } from './multi-query-retriever.js';
 export { createRetriever } from './create-retriever.js';
+export { isChunkIndexer } from './chunk-indexer.js';
 //# sourceMappingURL=index.js.map

package/dist/retrievers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/retrievers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAE9D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/retrievers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEhE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}

package/dist/retrievers/mmr-retriever.d.ts

@@ -14,6 +14,11 @@ export declare class MMRRetriever implements Retriever {
     private readonly defaultThreshold;
     constructor(config: MMRRetrieverConfig);
     retrieve(query: string, options?: Partial<RetrievalConfig>): Promise<RetrievalResult[]>;
+    /**
+     * Some vector stores (e.g. Qdrant) do not return stored vectors from search.
+     * Missing candidate vectors are re-embedded so diversity can still be computed.
+     */
+    private ensureVectors;
     private selectMMR;
     private toRetrievalResult;
 }

package/dist/retrievers/mmr-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mmr-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,kBAAkB;IACjC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAyBD,qBAAa,YAAa,YAAW,SAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,kBAAkB;IAQhC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAwB7F,OAAO,CAAC,SAAS;IAuCjB,OAAO,CAAC,iBAAiB;CAa1B"}
+{"version":3,"file":"mmr-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,kBAAkB;IACjC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAyBD,qBAAa,YAAa,YAAW,SAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,kBAAkB;IAQhC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAyB7F;;;OAGG;YACW,aAAa;IAsB3B,OAAO,CAAC,SAAS;IAuCjB,OAAO,CAAC,iBAAiB;CAa1B"}

package/dist/retrievers/mmr-retriever.js

@@ -1,3 +1,4 @@
+import { resultSource } from './chunk-indexer.js';
 const DEFAULT_TOP_K = 10;
 const DEFAULT_THRESHOLD = 0.0;
 const DEFAULT_LAMBDA = 0.7;
@@ -47,9 +48,30 @@ export class MMRRetriever {
         if (candidates.length === 0)
             return [];
         const lambda = options?.mmrLambda ?? this.defaultLambda;
-        const selected = this.selectMMR(candidates, topK, lambda);
+        const withVectors = await this.ensureVectors(candidates);
+        const selected = this.selectMMR(withVectors, topK, lambda);
         return selected.map((entry) => this.toRetrievalResult(entry));
     }
+    /**
+     * Some vector stores (e.g. Qdrant) do not return stored vectors from search.
+     * Missing candidate vectors are re-embedded so diversity can still be computed.
+     */
+    async ensureVectors(candidates) {
+        const missing = candidates
+            .map((candidate, index) => ({ candidate, index }))
+            .filter(({ candidate }) => !candidate.vector || candidate.vector.length === 0);
+        if (missing.length === 0)
+            return candidates;
+        const vectors = await this.embedding.embedBatch(missing.map(({ candidate }) => candidate.content));
+        if (vectors.length !== missing.length) {
+            throw new Error(`Embedding count mismatch: got ${vectors.length} vectors for ${missing.length} candidates`);
+        }
+        const result = [...candidates];
+        missing.forEach(({ candidate, index }, i) => {
+            result[index] = { ...candidate, vector: vectors[i] };
+        });
+        return result;
+    }
     selectMMR(candidates, topK, lambda) {
         const remaining = new Set(candidates.map((_, i) => i));
         const selected = [];
@@ -87,7 +109,7 @@ export class MMRRetriever {
             documentId,
             content: entry.content,
             score: entry.mmrScore,
-            source: entry.sourceType,
+            source: resultSource(entry.metadata, entry.sourceType),
             metadata: { ...entry.metadata, originalScore: entry.score },
         };
     }

package/dist/retrievers/mmr-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAI/B,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IACN,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAC/C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAC9D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE,WAAW;YACnB,KAAK;YACL,KAAK,EAAE,IAAI,GAAG,oBAAoB;YAClC,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,MAAM,MAAM,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC;QACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC1D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAChE,CAAC;IAEO,SAAS,CACf,UAA6B,EAC7B,IAAY,EACZ,MAAc;QAEd,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,QAAQ,GAA+C,EAAE,CAAC;QAEhE,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YACjB,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;gBAExC,IAAI,cAAc,GAAG,CAAC,QAAQ,CAAC;gBAC/B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjE,IAAI,GAAG,GAAG,cAAc;wBAAE,cAAc,GAAG,GAAG,CAAC;gBACjD,CAAC;gBACD,IAAI,cAAc,KAAK,CAAC,QAAQ;oBAAE,cAAc,GAAG,CAAC,CAAC;gBAErD,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,cAAc,CAAC;gBAEpE,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;oBACzB,SAAS,GAAG,QAAQ,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,iBAAiB,CAAC,KAA6C;QACrE,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,QAAQ;YACrB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE;SAC5D,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAUlD,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAI/B,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IACN,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAC/C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAC9D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE,WAAW;YACnB,KAAK;YACL,KAAK,EAAE,IAAI,GAAG,oBAAoB;YAClC,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,MAAM,MAAM,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,aAAa,CAAC;QACxD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;QAC3D,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAChE,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,aAAa,CAAC,UAA6B;QACvD,MAAM,OAAO,GAAG,UAAU;aACvB,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;aACjD,MAAM,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;QACjF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,UAAU,CAAC;QAE5C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAC7C,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAClD,CAAC;QACF,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,OAAO,CAAC,MAAM,aAAa,CAC3F,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC;QAC/B,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE;YAC1C,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE,EAAE,CAAC;QACxD,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CACf,UAA6B,EAC7B,IAAY,EACZ,MAAc;QAEd,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,QAAQ,GAA+C,EAAE,CAAC;QAEhE,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YACjB,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;gBAExC,IAAI,cAAc,GAAG,CAAC,QAAQ,CAAC;gBAC/B,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjE,IAAI,GAAG,GAAG,cAAc;wBAAE,cAAc,GAAG,GAAG,CAAC;gBACjD,CAAC;gBACD,IAAI,cAAc,KAAK,CAAC,QAAQ;oBAAE,cAAc,GAAG,CAAC,CAAC;gBAErD,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,cAAc,CAAC;gBAEpE,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;oBACzB,SAAS,GAAG,QAAQ,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,iBAAiB,CAAC,KAA6C;QACrE,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,QAAQ;YACrB,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YACtD,QAAQ,EAAE,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,EAAE;SAC5D,CAAC;IACJ,CAAC;CACF"}

package/dist/retrievers/multi-query-retriever.d.ts

@@ -1,15 +1,26 @@
 import type { Retriever, RetrievalConfig, RetrievalResult } from '@cogitator-ai/types';
+import { type ChunkIndexer, type IndexedChunk } from './chunk-indexer.js';
 export interface MultiQueryRetrieverConfig {
     baseRetriever: Retriever;
     expandQuery: (query: string) => Promise<string[]>;
     defaultTopK?: number;
+    /** Maximum number of generated query variants used in addition to the original query. */
+    defaultMaxQueries?: number;
 }
-export declare class MultiQueryRetriever implements Retriever {
+/**
+ * Expands the query into variants, retrieves for each in parallel and merges results
+ * (best score per chunk). The original query is always searched. If expansion fails the
+ * original query alone is used; if every retrieval fails the first error is thrown.
+ */
+export declare class MultiQueryRetriever implements Retriever, ChunkIndexer {
     private readonly baseRetriever;
     private readonly expandQuery;
     private readonly defaultTopK;
+    private readonly defaultMaxQueries?;
     constructor(config: MultiQueryRetrieverConfig);
+    indexChunk(chunk: IndexedChunk): void;
     retrieve(query: string, options?: Partial<RetrievalConfig>): Promise<RetrievalResult[]>;
+    private buildVariants;
     private deduplicateAndMerge;
 }
 //# sourceMappingURL=multi-query-retriever.d.ts.map

package/dist/retrievers/multi-query-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEvF,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAID,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAEzB,MAAM,EAAE,yBAAyB;IAMvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAsB7F,OAAO,CAAC,mBAAmB;CAY5B"}
+{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACvF,OAAO,EAAkB,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE1F,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,yFAAyF;IACzF,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAID;;;;GAIG;AACH,qBAAa,mBAAoB,YAAW,SAAS,EAAE,YAAY;IACjE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;IACnE,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAS;gBAEhC,MAAM,EAAE,yBAAyB;IAO7C,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAM/B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;YA0B/E,aAAa;IAwB3B,OAAO,CAAC,mBAAmB;CAY5B"}

package/dist/retrievers/multi-query-retriever.js

@@ -1,25 +1,64 @@
+import { isChunkIndexer } from './chunk-indexer.js';
 const DEFAULT_TOP_K = 10;
+/**
+ * Expands the query into variants, retrieves for each in parallel and merges results
+ * (best score per chunk). The original query is always searched. If expansion fails the
+ * original query alone is used; if every retrieval fails the first error is thrown.
+ */
 export class MultiQueryRetriever {
     baseRetriever;
     expandQuery;
     defaultTopK;
+    defaultMaxQueries;
     constructor(config) {
         this.baseRetriever = config.baseRetriever;
         this.expandQuery = config.expandQuery;
         this.defaultTopK = config.defaultTopK ?? DEFAULT_TOP_K;
+        this.defaultMaxQueries = config.defaultMaxQueries;
     }
-    async retrieve(query, options) {
-        const expanded = await this.expandQuery(query);
-        const variants = expanded.length > 0 ? expanded : [query];
-        if (!variants.includes(query)) {
-            variants.unshift(query);
+    indexChunk(chunk) {
+        if (isChunkIndexer(this.baseRetriever)) {
+            this.baseRetriever.indexChunk(chunk);
         }
+    }
+    async retrieve(query, options) {
+        const variants = await this.buildVariants(query, options?.multiQueryCount);
         const settled = await Promise.allSettled(variants.map((variant) => this.baseRetriever.retrieve(variant, options)));
-        const allResults = settled.flatMap((outcome) => outcome.status === 'fulfilled' ? outcome.value : []);
-        const merged = this.deduplicateAndMerge(allResults.flat());
+        const fulfilled = settled.filter((outcome) => outcome.status === 'fulfilled');
+        if (fulfilled.length === 0) {
+            const firstFailure = settled.find((outcome) => outcome.status === 'rejected');
+            throw firstFailure?.reason instanceof Error
+                ? firstFailure.reason
+                : new Error(`MultiQueryRetriever: all retrievals failed: ${String(firstFailure?.reason)}`);
+        }
+        const merged = this.deduplicateAndMerge(fulfilled.flatMap((outcome) => outcome.value));
         const topK = options?.topK ?? this.defaultTopK;
         return merged.slice(0, topK);
     }
+    async buildVariants(query, requestedMax) {
+        let expanded;
+        try {
+            expanded = await this.expandQuery(query);
+        }
+        catch {
+            expanded = [];
+        }
+        const normalized = query.trim().toLowerCase();
+        const seen = new Set([normalized]);
+        const extra = [];
+        for (const variant of expanded) {
+            if (typeof variant !== 'string')
+                continue;
+            const trimmed = variant.trim();
+            const key = trimmed.toLowerCase();
+            if (!trimmed || seen.has(key))
+                continue;
+            seen.add(key);
+            extra.push(trimmed);
+        }
+        const max = requestedMax ?? this.defaultMaxQueries;
+        return [query, ...(max === undefined ? extra : extra.slice(0, max))];
+    }
     deduplicateAndMerge(results) {
         const best = new Map();
         for (const result of results) {

package/dist/retrievers/multi-query-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAQA,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAY;IACzB,WAAW,CAAuC;IAClD,WAAW,CAAS;IAErC,YAAY,MAAiC;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QAE1D,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9B,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC1B,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;QAEF,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC7C,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CACpD,CAAC;QAEF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAE/C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,mBAAmB,CAAC,OAA0B;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;CACF"}
+{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAwC,MAAM,oBAAoB,CAAC;AAU1F,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB;;;;GAIG;AACH,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAY;IACzB,WAAW,CAAuC;IAClD,WAAW,CAAS;IACpB,iBAAiB,CAAU;IAE5C,YAAY,MAAiC;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;QACtC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC,iBAAiB,CAAC;IACpD,CAAC;IAED,UAAU,CAAC,KAAmB;QAC5B,IAAI,cAAc,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC,CAAC;QAE3E,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;QAEF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAC9B,CAAC,OAAO,EAAwD,EAAE,CAChE,OAAO,CAAC,MAAM,KAAK,WAAW,CACjC,CAAC;QACF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAC/B,CAAC,OAAO,EAAoC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,UAAU,CAC7E,CAAC;YACF,MAAM,YAAY,EAAE,MAAM,YAAY,KAAK;gBACzC,CAAC,CAAC,YAAY,CAAC,MAAM;gBACrB,CAAC,CAAC,IAAI,KAAK,CAAC,+CAA+C,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACvF,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAE/C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,KAAa,EAAE,YAAqB;QAC9D,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAC3C,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,GAAG,EAAE,CAAC;QAChB,CAAC;QAED,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC9C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACnC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,OAAO,KAAK,QAAQ;gBAAE,SAAS;YAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAClC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YACxC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,GAAG,GAAG,YAAY,IAAI,IAAI,CAAC,iBAAiB,CAAC;QACnD,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;IACvE,CAAC;IAEO,mBAAmB,CAAC,OAA0B;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;CACF"}

package/dist/retrievers/similarity-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,yBAAyB;IAOvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiB7F,OAAO,CAAC,iBAAiB;CAa1B"}
+{"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,yBAAyB;IAOvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiB7F,OAAO,CAAC,iBAAiB;CAa1B"}

package/dist/retrievers/similarity-retriever.js

@@ -1,3 +1,4 @@
+import { resultSource } from './chunk-indexer.js';
 const DEFAULT_TOP_K = 10;
 const DEFAULT_THRESHOLD = 0.0;
 export class SimilarityRetriever {
@@ -32,7 +33,7 @@ export class SimilarityRetriever {
             documentId,
             content: entry.content,
             score: entry.score,
-            source: entry.sourceType,
+            source: resultSource(entry.metadata, entry.sourceType),
             metadata: entry.metadata,
         };
     }

package/dist/retrievers/similarity-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAgBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,mBAAmB;IACb,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM;YACN,KAAK;YACL,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAoC;QAC5D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AASlD,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,mBAAmB;IACb,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM;YACN,KAAK;YACL,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAoC;QAC5D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YACtD,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}

package/dist/tools.d.ts

@@ -17,7 +17,16 @@ declare const IngestParamsSchema: z.ZodObject<{
 }, z.core.$strip>;
 type IngestParams = z.infer<typeof IngestParamsSchema>;
 export declare function createSearchTool(pipeline: RAGPipeline): RAGTool<SearchParams>;
-export declare function createIngestTool(pipeline: RAGPipeline): RAGTool<IngestParams>;
-export declare function ragTools(pipeline: RAGPipeline): [RAGTool<SearchParams>, RAGTool<IngestParams>];
+export interface IngestToolOptions {
+    /**
+     * Restrict local sources to these directories (symlinks are resolved). When set, any
+     * path outside them is rejected. Strongly recommended when the tool is exposed to an LLM.
+     */
+    allowedRoots?: string[];
+    /** Allow http(s) URLs as sources. Defaults to `true`. */
+    allowUrls?: boolean;
+}
+export declare function createIngestTool(pipeline: RAGPipeline, options?: IngestToolOptions): RAGTool<IngestParams>;
+export declare function ragTools(pipeline: RAGPipeline, ingestOptions?: IngestToolOptions): [RAGTool<SearchParams>, RAGTool<IngestParams>];
 export {};
 //# sourceMappingURL=tools.d.ts.map