@cogitator-ai/rag 0.1.9 → 0.2.0

package/dist/rag-pipeline.js

@@ -1,35 +1,45 @@
 import { RAGPipelineConfigSchema } from './schema.js';
+import { isChunkIndexer } from './retrievers/chunk-indexer.js';
+const STORE_CONCURRENCY = 16;
+function definedEntries(value) {
+    if (!value)
+        return {};
+    return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== undefined));
+}
 export class RAGPipeline {
     config;
     deps;
     stats = { documentsIngested: 0, chunksStored: 0, queriesProcessed: 0 };
-    constructor(config, deps) {
-        this.config = RAGPipelineConfigSchema.parse(config);
+    constructor(config, deps, alreadyParsed = false) {
+        this.config = alreadyParsed ? config : RAGPipelineConfigSchema.parse(config);
         this.deps = deps;
     }
     async ingest(source) {
         const documents = await this.deps.loader.load(source);
         let totalChunks = 0;
+        let ingestedDocuments = 0;
         for (const doc of documents) {
             const chunks = await this.chunkDocument(doc);
-            if (chunks.length === 0)
-                continue;
-            const texts = chunks.map((c) => c.content);
-            const vectors = await this.deps.embeddingService.embedBatch(texts);
-            if (vectors.length !== chunks.length) {
-                throw new Error(`Embedding count mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
-            }
-            for (let i = 0; i < chunks.length; i++) {
-                await this.storeChunk(chunks[i], vectors[i], doc);
+            if (chunks.length > 0) {
+                const vectors = await this.deps.embeddingService.embedBatch(chunks.map((c) => c.content));
+                if (vectors.length !== chunks.length) {
+                    throw new Error(`Embedding count mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
+                }
+                await this.storeChunks(chunks, vectors, doc);
+                totalChunks += chunks.length;
+                this.stats.chunksStored += chunks.length;
             }
-            totalChunks += chunks.length;
+            ingestedDocuments++;
+            this.stats.documentsIngested++;
         }
-        this.stats.documentsIngested += documents.length;
-        this.stats.chunksStored += totalChunks;
-        return { documents: documents.length, chunks: totalChunks };
+        return { documents: ingestedDocuments, chunks: totalChunks };
     }
     async query(text, options) {
-        const results = await this.deps.retriever.retrieve(text, options);
+        const merged = {
+            ...this.config.retrieval,
+            ...definedEntries(options),
+        };
+        const results = await this.deps.retriever.retrieve(text, merged);
         this.stats.queriesProcessed++;
         if (this.deps.reranker && this.config.reranking?.enabled) {
             return this.deps.reranker.rerank(text, results, this.config.reranking.topN);
@@ -40,24 +50,49 @@ export class RAGPipeline {
         return { ...this.stats };
     }
     async chunkDocument(doc) {
-        return this.deps.chunker.chunk(doc.content, doc.id);
+        const chunks = await this.deps.chunker.chunk(doc.content, doc.id);
+        return chunks.filter((chunk) => chunk.content.trim().length > 0);
+    }
+    async storeChunks(chunks, vectors, doc) {
+        let next = 0;
+        const worker = async () => {
+            while (next < chunks.length) {
+                const index = next++;
+                await this.storeChunk(chunks[index], vectors[index], doc);
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(STORE_CONCURRENCY, chunks.length) }, () => worker()));
     }
     async storeChunk(chunk, vector, doc) {
+        const metadata = {
+            ...doc.metadata,
+            ...chunk.metadata,
+            documentId: doc.id,
+            source: doc.source,
+            sourceType: doc.sourceType,
+            order: chunk.order,
+            startOffset: chunk.startOffset,
+            endOffset: chunk.endOffset,
+        };
         const result = await this.deps.embeddingAdapter.addEmbedding({
             sourceId: chunk.id,
             sourceType: 'document',
             vector,
             content: chunk.content,
-            metadata: {
-                ...chunk.metadata,
-                documentId: doc.id,
-                source: doc.source,
-                order: chunk.order,
-            },
+            metadata,
         });
         if (!result.success) {
             throw new Error(result.error);
         }
+        if (isChunkIndexer(this.deps.retriever)) {
+            this.deps.retriever.indexChunk({
+                embeddingId: result.data.id,
+                chunkId: chunk.id,
+                documentId: doc.id,
+                content: chunk.content,
+                metadata,
+            });
+        }
     }
 }
 //# sourceMappingURL=rag-pipeline.js.map

package/dist/rag-pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAWnD,MAAM,OAAO,WAAW;IACL,MAAM,CAAoB;IAC1B,IAAI,CAAkB;IAC/B,KAAK,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IAE/E,YAAY,MAAyB,EAAE,IAAqB;QAC1D,IAAI,CAAC,MAAM,GAAG,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAc;QACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAElC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAEnE,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,SAAS,CACtF,CAAC;YACJ,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAED,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,SAAS,CAAC,MAAM,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,WAAW,CAAC;QAEvC,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAkC;QAC1D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,GAAgB;QAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,MAAgB,EAAE,GAAgB;QAC/E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YAC3D,QAAQ,EAAE,KAAK,CAAC,EAAE;YAClB,UAAU,EAAE,UAAU;YACtB,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAW/D,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAE7B,SAAS,cAAc,CAAmB,KAAoB;IAC5D,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IACtB,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAe,CAAC;AACpG,CAAC;AAED,MAAM,OAAO,WAAW;IACL,MAAM,CAAoB;IAC1B,IAAI,CAAkB;IAC/B,KAAK,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IAE/E,YAAY,MAAyB,EAAE,IAAqB,EAAE,aAAa,GAAG,KAAK;QACjF,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7E,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAc;QACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;gBAE1F,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;oBACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,SAAS,CACtF,CAAC;gBACJ,CAAC;gBAED,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC7C,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC7B,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;YAC3C,CAAC;YACD,iBAAiB,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC;QACjC,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAkC;QAC1D,MAAM,MAAM,GAA6B;YACvC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS;YACxB,GAAG,cAAc,CAAC,OAAO,CAAC;SAC3B,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACjE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,GAAgB;QAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,MAAuB,EACvB,OAAmB,EACnB,GAAgB;QAEhB,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAE,EAAE,OAAO,CAAC,KAAK,CAAE,EAAE,GAAG,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CACnF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,MAAgB,EAAE,GAAgB;QAC/E,MAAM,QAAQ,GAA4B;YACxC,GAAG,GAAG,CAAC,QAAQ;YACf,GAAG,KAAK,CAAC,QAAQ;YACjB,UAAU,EAAE,GAAG,CAAC,EAAE;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;SAC3B,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YAC3D,QAAQ,EAAE,KAAK,CAAC,EAAE;YAClB,UAAU,EAAE,UAAU;YACtB,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ;SACT,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;gBAC7B,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;gBAC3B,OAAO,EAAE,KAAK,CAAC,EAAE;gBACjB,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}

package/dist/rerankers/cohere-reranker.d.ts

@@ -4,8 +4,8 @@ export interface CohereRerankerConfig {
     model?: string;
 }
 export declare class CohereReranker implements Reranker {
-    private apiKey;
-    private model;
+    private readonly apiKey;
+    private readonly model;
     constructor(config: CohereRerankerConfig);
     rerank(query: string, results: RetrievalResult[], topN?: number): Promise<RetrievalResult[]>;
 }

package/dist/rerankers/cohere-reranker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cohere-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/cohere-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAYD,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,MAAM,EAAE,oBAAoB;IAKlC,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC;CAmC9B"}
+{"version":3,"file":"cohere-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/cohere-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAaD,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;gBAEnB,MAAM,EAAE,oBAAoB;IAKlC,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC;CAwC9B"}

package/dist/rerankers/cohere-reranker.js

@@ -1,5 +1,6 @@
 const DEFAULT_MODEL = 'rerank-v3.5';
 const COHERE_RERANK_URL = 'https://api.cohere.com/v2/rerank';
+const DEFAULT_TIMEOUT_MS = 30_000;
 export class CohereReranker {
     apiKey;
     model;
@@ -23,12 +24,16 @@ export class CohereReranker {
                 documents: results.map((r) => r.content),
                 top_n: n,
             }),
+            signal: AbortSignal.timeout(DEFAULT_TIMEOUT_MS),
         });
         if (!response.ok) {
             const text = await response.text();
             throw new Error(`Cohere rerank failed (${response.status}): ${text}`);
         }
         const data = (await response.json());
+        if (!Array.isArray(data.results)) {
+            throw new Error('Cohere rerank returned unexpected response: missing results array');
+        }
         const mapped = data.results
             .filter(({ index }) => index >= 0 && index < results.length)
             .map(({ index, relevance_score }) => ({

package/dist/rerankers/cohere-reranker.js.map

@@ -1 +1 @@
-{"version":3,"file":"cohere-reranker.js","sourceRoot":"","sources":["../../src/rerankers/cohere-reranker.ts"],"names":[],"mappings":"AAcA,MAAM,aAAa,GAAG,aAAa,CAAC;AACpC,MAAM,iBAAiB,GAAG,kCAAkC,CAAC;AAE7D,MAAM,OAAO,cAAc;IACjB,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,MAA4B;QACtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA0B,EAC1B,IAAa;QAEb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,MAAM,CAAC,GAAG,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;QAEjC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,iBAAiB,EAAE;YAC9C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK;gBACL,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACxC,KAAK,EAAE,CAAC;aACT,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAyB,CAAC;QAE7D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO;aACxB,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;aAC3D,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,OAAO,CAAC,KAAK,CAAC;YACjB,KAAK,EAAE,eAAe;SACvB,CAAC,CAAC,CAAC;QAEN,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,CAAC;CACF"}
+{"version":3,"file":"cohere-reranker.js","sourceRoot":"","sources":["../../src/rerankers/cohere-reranker.ts"],"names":[],"mappings":"AAcA,MAAM,aAAa,GAAG,aAAa,CAAC;AACpC,MAAM,iBAAiB,GAAG,kCAAkC,CAAC;AAC7D,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,MAAM,OAAO,cAAc;IACR,MAAM,CAAS;IACf,KAAK,CAAS;IAE/B,YAAY,MAA4B;QACtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA0B,EAC1B,IAAa;QAEb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,MAAM,CAAC,GAAG,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;QAEjC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,iBAAiB,EAAE;YAC9C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK;gBACL,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACxC,KAAK,EAAE,CAAC;aACT,CAAC;YACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,kBAAkB,CAAC;SAChD,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAyB,CAAC;QAE7D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACvF,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO;aACxB,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;aAC3D,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,OAAO,CAAC,KAAK,CAAC;YACjB,KAAK,EAAE,eAAe;SACvB,CAAC,CAAC,CAAC;QAEN,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,CAAC;CACF"}

package/dist/rerankers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,KAAK,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,cAAc,EAAE,KAAK,oBAAoB,EAAE,MAAM,sBAAsB,CAAC"}

package/dist/rerankers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAA0B,MAAM,gBAAgB,CAAC;AACrE,OAAO,EAAE,cAAc,EAA6B,MAAM,mBAAmB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/rerankers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAA0B,MAAM,mBAAmB,CAAC;AACxE,OAAO,EAAE,cAAc,EAA6B,MAAM,sBAAsB,CAAC"}

package/dist/rerankers/llm-reranker.d.ts

@@ -3,7 +3,7 @@ export interface LLMRerankerConfig {
     generateFn: (prompt: string) => Promise<string>;
 }
 export declare class LLMReranker implements Reranker {
-    private generateFn;
+    private readonly generateFn;
     constructor(config: LLMRerankerConfig);
     rerank(query: string, results: RetrievalResult[], topN?: number): Promise<RetrievalResult[]>;
     private buildPrompt;

package/dist/rerankers/llm-reranker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACjD;AAED,qBAAa,WAAY,YAAW,QAAQ;IAC1C,OAAO,CAAC,UAAU,CAAsC;gBAE5C,MAAM,EAAE,iBAAiB;IAI/B,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC;IA0B7B,OAAO,CAAC,WAAW;IAcnB,OAAO,CAAC,WAAW;CAiBpB"}
+{"version":3,"file":"llm-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACjD;AAID,qBAAa,WAAY,YAAW,QAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAsC;gBAErD,MAAM,EAAE,iBAAiB;IAI/B,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC;IA2B7B,OAAO,CAAC,WAAW;IAenB,OAAO,CAAC,WAAW;CA2BpB"}

package/dist/rerankers/llm-reranker.js

@@ -1,3 +1,4 @@
+const MAX_DOCUMENTS_IN_PROMPT = 50;
 export class LLMReranker {
     generateFn;
     constructor(config) {
@@ -21,13 +22,15 @@ export class LLMReranker {
             }));
             return topN ? reranked.slice(0, topN) : reranked;
         }
-        catch {
+        catch (error) {
+            console.warn('[LLMReranker] Reranking failed, returning original order:', error);
             const fallback = [...results];
             return topN ? fallback.slice(0, topN) : fallback;
         }
     }
     buildPrompt(query, results) {
-        const docs = results.map((r, i) => `[${i}] ${r.content}`).join('\n\n');
+        const limited = results.slice(0, MAX_DOCUMENTS_IN_PROMPT);
+        const docs = limited.map((r, i) => `[${i}] ${r.content}`).join('\n\n');
         return [
             "Score each document's relevance to the query on a scale of 0-10.",
             'Return ONLY a JSON array: [{ "index": number, "score": number }]',
@@ -39,18 +42,28 @@ export class LLMReranker {
         ].join('\n');
     }
     parseScores(response, count) {
-        const jsonMatch = /\[[\s\S]*?\]/.exec(response);
-        if (!jsonMatch)
+        const candidates = response.match(/\[[\s\S]*?\]/g);
+        if (!candidates)
             throw new Error('No JSON array found in response');
-        const parsed = JSON.parse(jsonMatch[0]);
-        if (!Array.isArray(parsed))
-            throw new Error('Response is not an array');
-        return parsed.filter((item) => typeof item === 'object' &&
-            item !== null &&
-            typeof item.index === 'number' &&
-            typeof item.score === 'number' &&
-            item.index >= 0 &&
-            item.index < count);
+        for (let i = candidates.length - 1; i >= 0; i--) {
+            try {
+                const parsed = JSON.parse(candidates[i]);
+                if (!Array.isArray(parsed))
+                    continue;
+                const scores = parsed.filter((item) => typeof item === 'object' &&
+                    item !== null &&
+                    typeof item.index === 'number' &&
+                    typeof item.score === 'number' &&
+                    item.index >= 0 &&
+                    item.index < count);
+                if (scores.length > 0)
+                    return scores;
+            }
+            catch {
+                continue;
+            }
+        }
+        throw new Error('No valid JSON array found in response');
     }
 }
 //# sourceMappingURL=llm-reranker.js.map

package/dist/rerankers/llm-reranker.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm-reranker.js","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAMA,MAAM,OAAO,WAAW;IACd,UAAU,CAAsC;IAExD,YAAY,MAAyB;QACnC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA0B,EAC1B,IAAa;QAEb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAE1D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;gBAChD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;gBACrD,GAAG,MAAM;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,EAAE,CAAC,CAAC;aAC/C,CAAC,CAAC,CAAC;YAEJ,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAa,EAAE,OAA0B;QAC3D,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEvE,OAAO;YACL,kEAAkE;YAClE,kEAAkE;YAClE,EAAE;YACF,UAAU,KAAK,EAAE;YACjB,EAAE;YACF,YAAY;YACZ,IAAI;SACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAEO,WAAW,CAAC,QAAgB,EAAE,KAAa;QACjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAExE,OAAO,MAAM,CAAC,MAAM,CAClB,CAAC,IAAI,EAA4C,EAAE,CACjD,OAAO,IAAI,KAAK,QAAQ;YACxB,IAAI,KAAK,IAAI;YACb,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;YAC3D,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;YAC1D,IAA0B,CAAC,KAAK,IAAI,CAAC;YACrC,IAA0B,CAAC,KAAK,GAAG,KAAK,CAC5C,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"llm-reranker.js","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAMA,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,WAAW;IACL,UAAU,CAAsC;IAEjE,YAAY,MAAyB;QACnC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA0B,EAC1B,IAAa;QAEb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAE1D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;gBAChD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;gBACrD,GAAG,MAAM;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,EAAE,CAAC,CAAC;aAC/C,CAAC,CAAC,CAAC;YAEJ,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CAAC,2DAA2D,EAAE,KAAK,CAAC,CAAC;YACjF,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAa,EAAE,OAA0B;QAC3D,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,uBAAuB,CAAC,CAAC;QAC1D,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEvE,OAAO;YACL,kEAAkE;YAClE,kEAAkE;YAClE,EAAE;YACF,UAAU,KAAK,EAAE;YACjB,EAAE;YACF,YAAY;YACZ,IAAI;SACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAEO,WAAW,CAAC,QAAgB,EAAE,KAAa;QACjD,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;QACnD,IAAI,CAAC,UAAU;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAEpE,KAAK,IAAI,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAChD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;oBAAE,SAAS;gBAErC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAC1B,CAAC,IAAI,EAA4C,EAAE,CACjD,OAAO,IAAI,KAAK,QAAQ;oBACxB,IAAI,KAAK,IAAI;oBACb,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;oBAC3D,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;oBAC1D,IAA0B,CAAC,KAAK,IAAI,CAAC;oBACrC,IAA0B,CAAC,KAAK,GAAG,KAAK,CAC5C,CAAC;gBAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;oBAAE,OAAO,MAAM,CAAC;YACvC,CAAC;YAAC,MAAM,CAAC;gBACP,SAAS;YACX,CAAC;QACH,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;CACF"}

package/dist/retrievers/chunk-indexer.d.ts

@@ -0,0 +1,19 @@
+import type { Retriever } from '@cogitator-ai/types';
+export interface IndexedChunk {
+    embeddingId: string;
+    chunkId: string;
+    documentId: string;
+    content: string;
+    metadata: Record<string, unknown>;
+}
+/**
+ * A retriever that maintains its own index alongside the vector store
+ * (e.g. a local BM25 index). `RAGPipeline.ingest()` calls `indexChunk()` for
+ * every stored chunk.
+ */
+export interface ChunkIndexer {
+    indexChunk(chunk: IndexedChunk): void;
+}
+export declare function isChunkIndexer(retriever: Retriever): retriever is Retriever & ChunkIndexer;
+export declare function resultSource(metadata: Record<string, unknown> | undefined, fallback: string): string;
+//# sourceMappingURL=chunk-indexer.d.ts.map

package/dist/retrievers/chunk-indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-indexer.d.ts","sourceRoot":"","sources":["../../src/retrievers/chunk-indexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAErD,MAAM,WAAW,YAAY;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI,CAAC;CACvC;AAED,wBAAgB,cAAc,CAAC,SAAS,EAAE,SAAS,GAAG,SAAS,IAAI,SAAS,GAAG,YAAY,CAE1F;AAED,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,UAG3F"}

package/dist/retrievers/chunk-indexer.js

@@ -0,0 +1,8 @@
+export function isChunkIndexer(retriever) {
+    return typeof retriever.indexChunk === 'function';
+}
+export function resultSource(metadata, fallback) {
+    const source = metadata?.source;
+    return typeof source === 'string' ? source : fallback;
+}
+//# sourceMappingURL=chunk-indexer.js.map

package/dist/retrievers/chunk-indexer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-indexer.js","sourceRoot":"","sources":["../../src/retrievers/chunk-indexer.ts"],"names":[],"mappings":"AAmBA,MAAM,UAAU,cAAc,CAAC,SAAoB;IACjD,OAAO,OAAQ,SAAmC,CAAC,UAAU,KAAK,UAAU,CAAC;AAC/E,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,QAA6C,EAAE,QAAgB;IAC1F,MAAM,MAAM,GAAG,QAAQ,EAAE,MAAM,CAAC;IAChC,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;AACxD,CAAC"}

package/dist/retrievers/create-retriever.d.ts

@@ -26,6 +26,8 @@ interface MultiQueryDeps {
     strategy: 'multi-query';
     baseRetriever: Retriever;
     expandQuery: (query: string) => Promise<string[]>;
+    topK?: number;
+    maxQueries?: number;
 }
 export type CreateRetrieverConfig = SimilarityDeps | MMRDeps | HybridDeps | MultiQueryDeps;
 export declare function createRetriever(config: CreateRetrieverConfig): Retriever;

package/dist/retrievers/create-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAMzD,UAAU,iBAAiB;IACzB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;CACpC;AAED,UAAU,cAAe,SAAQ,iBAAiB;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,OAAQ,SAAQ,iBAAiB;IACzC,QAAQ,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,aAAa,CAAC;IACxB,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACnD;AAED,MAAM,MAAM,qBAAqB,GAAG,cAAc,GAAG,OAAO,GAAG,UAAU,GAAG,cAAc,CAAC;AAE3F,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,CAiCxE"}
+{"version":3,"file":"create-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAMzD,UAAU,iBAAiB;IACzB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;CACpC;AAED,UAAU,cAAe,SAAQ,iBAAiB;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,OAAQ,SAAQ,iBAAiB;IACzC,QAAQ,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,aAAa,CAAC;IACxB,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,qBAAqB,GAAG,cAAc,GAAG,OAAO,GAAG,UAAU,GAAG,cAAc,CAAC;AAE3F,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,CAwCxE"}

package/dist/retrievers/create-retriever.js

@@ -30,7 +30,13 @@ export function createRetriever(config) {
             return new MultiQueryRetriever({
                 baseRetriever: config.baseRetriever,
                 expandQuery: config.expandQuery,
+                defaultTopK: config.topK,
+                defaultMaxQueries: config.maxQueries,
             });
+        default: {
+            const _exhaustive = config;
+            throw new Error(`Unknown retriever strategy: ${JSON.stringify(_exhaustive)}`);
+        }
     }
 }
 //# sourceMappingURL=create-retriever.js.map

package/dist/retrievers/create-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"create-retriever.js","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAoC9D,MAAM,UAAU,eAAe,CAAC,MAA6B;IAC3D,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,YAAY;YACf,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,KAAK;YACR,OAAO,IAAI,YAAY,CAAC;gBACtB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,aAAa,EAAE,MAAM,CAAC,MAAM;gBAC5B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,OAAO,IAAI,eAAe,CAAC;gBACzB,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,cAAc,EAAE,MAAM,CAAC,OAAO;gBAC9B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,aAAa;YAChB,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;aAChC,CAAC,CAAC;IACP,CAAC;AACH,CAAC"}
+{"version":3,"file":"create-retriever.js","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAsCjE,MAAM,UAAU,eAAe,CAAC,MAA6B;IAC3D,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,YAAY;YACf,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,KAAK;YACR,OAAO,IAAI,YAAY,CAAC;gBACtB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,aAAa,EAAE,MAAM,CAAC,MAAM;gBAC5B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,OAAO,IAAI,eAAe,CAAC;gBACzB,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,cAAc,EAAE,MAAM,CAAC,OAAO;gBAC9B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,aAAa;YAChB,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,iBAAiB,EAAE,MAAM,CAAC,UAAU;aACrC,CAAC,CAAC;QAEL,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,MAAM,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,+BAA+B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/retrievers/hybrid-retriever.d.ts

@@ -1,17 +1,24 @@
 import type { Retriever, RetrievalConfig, RetrievalResult, HybridSearchWeights } from '@cogitator-ai/types';
 import type { HybridSearch } from '@cogitator-ai/memory';
+import { type ChunkIndexer, type IndexedChunk } from './chunk-indexer.js';
 export interface HybridRetrieverConfig {
     hybridSearch: HybridSearch;
     defaultWeights?: HybridSearchWeights;
     defaultTopK?: number;
     defaultThreshold?: number;
 }
-export declare class HybridRetriever implements Retriever {
+/**
+ * Vector + BM25 retrieval fused with reciprocal rank fusion. Chunks ingested through
+ * `RAGPipeline` are added to the HybridSearch keyword index automatically.
+ */
+export declare class HybridRetriever implements Retriever, ChunkIndexer {
     private readonly hybridSearch;
     private readonly defaultWeights?;
     private readonly defaultTopK;
     private readonly defaultThreshold;
+    private readonly indexed;
     constructor(config: HybridRetrieverConfig);
+    indexChunk(chunk: IndexedChunk): void;
     retrieve(query: string, options?: Partial<RetrievalConfig>): Promise<RetrievalResult[]>;
     private toRetrievalResult;
 }

package/dist/retrievers/hybrid-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,eAAe,EACf,mBAAmB,EAEpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEzD,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,YAAY,CAAC;IAC3B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,eAAgB,YAAW,SAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,qBAAqB;IAOnC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAgB7F,OAAO,CAAC,iBAAiB;CAiB1B"}
+{"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,eAAe,EACf,mBAAmB,EAEpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAgB,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAExF,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,YAAY,CAAC;IAC3B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD;;;GAGG;AACH,qBAAa,eAAgB,YAAW,SAAS,EAAE,YAAY;IAC7D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;IAC1C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;gBAE/C,MAAM,EAAE,qBAAqB;IAOzC,UAAU,CAAC,KAAK,EAAE,YAAY,GAAG,IAAI;IAK/B,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAgB7F,OAAO,CAAC,iBAAiB;CAoB1B"}

package/dist/retrievers/hybrid-retriever.js

@@ -1,16 +1,26 @@
+import { resultSource } from './chunk-indexer.js';
 const DEFAULT_TOP_K = 10;
 const DEFAULT_THRESHOLD = 0.0;
+/**
+ * Vector + BM25 retrieval fused with reciprocal rank fusion. Chunks ingested through
+ * `RAGPipeline` are added to the HybridSearch keyword index automatically.
+ */
 export class HybridRetriever {
     hybridSearch;
     defaultWeights;
     defaultTopK;
     defaultThreshold;
+    indexed = new Map();
     constructor(config) {
         this.hybridSearch = config.hybridSearch;
         this.defaultWeights = config.defaultWeights;
         this.defaultTopK = config.defaultTopK ?? DEFAULT_TOP_K;
         this.defaultThreshold = config.defaultThreshold ?? DEFAULT_THRESHOLD;
     }
+    indexChunk(chunk) {
+        this.indexed.set(chunk.embeddingId, chunk);
+        this.hybridSearch.indexDocument(chunk.embeddingId, chunk.content);
+    }
     async retrieve(query, options) {
         const result = await this.hybridSearch.search({
             query,
@@ -25,16 +35,19 @@ export class HybridRetriever {
         return result.data.map((entry) => this.toRetrievalResult(entry));
     }
     toRetrievalResult(entry) {
-        const rawDocId = entry.metadata?.documentId;
-        const documentId = typeof rawDocId === 'string' ? rawDocId : entry.sourceId;
+        const known = this.indexed.get(entry.id);
+        const metadata = { ...known?.metadata, ...entry.metadata };
+        const rawDocId = metadata.documentId;
+        const chunkId = known?.chunkId ?? entry.sourceId;
+        const documentId = typeof rawDocId === 'string' ? rawDocId : chunkId;
         return {
-            chunkId: entry.sourceId,
+            chunkId,
             documentId,
             content: entry.content,
             score: entry.score,
-            source: entry.sourceType,
+            source: resultSource(metadata, entry.sourceType),
             metadata: {
-                ...entry.metadata,
+                ...metadata,
                 vectorScore: entry.vectorScore,
                 keywordScore: entry.keywordScore,
             },

package/dist/retrievers/hybrid-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAgBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,eAAe;IACT,YAAY,CAAe;IAC3B,cAAc,CAAuB;IACrC,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA6B;QACvC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC5C,KAAK;YACL,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE,IAAI,CAAC,cAAc;YAC5B,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAmB;QAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;aACjC;SACF,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAQA,OAAO,EAAE,YAAY,EAAwC,MAAM,oBAAoB,CAAC;AASxF,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;GAGG;AACH,MAAM,OAAO,eAAe;IACT,YAAY,CAAe;IAC3B,cAAc,CAAuB;IACrC,WAAW,CAAS;IACpB,gBAAgB,CAAS;IACzB,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE3D,YAAY,MAA6B;QACvC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,UAAU,CAAC,KAAmB;QAC5B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,KAAK,CAAC,WAAW,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC5C,KAAK;YACL,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE,IAAI,CAAC,cAAc;YAC5B,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAmB;QAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,EAAE,GAAG,KAAK,EAAE,QAAQ,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC3D,MAAM,QAAQ,GAAG,QAAQ,CAAC,UAAU,CAAC;QACrC,MAAM,OAAO,GAAG,KAAK,EAAE,OAAO,IAAI,KAAK,CAAC,QAAQ,CAAC;QACjD,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC;QAErE,OAAO;YACL,OAAO;YACP,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,YAAY,CAAC,QAAQ,EAAE,KAAK,CAAC,UAAU,CAAC;YAChD,QAAQ,EAAE;gBACR,GAAG,QAAQ;gBACX,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;aACjC;SACF,CAAC;IACJ,CAAC;CACF"}

package/dist/retrievers/index.d.ts

@@ -8,4 +8,6 @@ export { MultiQueryRetriever } from './multi-query-retriever.js';
 export type { MultiQueryRetrieverConfig } from './multi-query-retriever.js';
 export { createRetriever } from './create-retriever.js';
 export type { CreateRetrieverConfig } from './create-retriever.js';
+export { isChunkIndexer } from './chunk-indexer.js';
+export type { ChunkIndexer, IndexedChunk } from './chunk-indexer.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/retrievers/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/retrievers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,YAAY,EAAE,yBAAyB,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,YAAY,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAC9D,YAAY,EAAE,yBAAyB,EAAE,MAAM,yBAAyB,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,YAAY,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/retrievers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAChE,YAAY,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,YAAY,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AAC5E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC"}

package/dist/retrievers/index.js

@@ -3,4 +3,5 @@ export { MMRRetriever } from './mmr-retriever.js';
 export { HybridRetriever } from './hybrid-retriever.js';
 export { MultiQueryRetriever } from './multi-query-retriever.js';
 export { createRetriever } from './create-retriever.js';
+export { isChunkIndexer } from './chunk-indexer.js';
 //# sourceMappingURL=index.js.map

package/dist/retrievers/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/retrievers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAE7D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE/C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAE9D,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/retrievers/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAEhE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AAEjE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}

package/dist/retrievers/mmr-retriever.d.ts

@@ -14,6 +14,11 @@ export declare class MMRRetriever implements Retriever {
     private readonly defaultThreshold;
     constructor(config: MMRRetrieverConfig);
     retrieve(query: string, options?: Partial<RetrievalConfig>): Promise<RetrievalResult[]>;
+    /**
+     * Some vector stores (e.g. Qdrant) do not return stored vectors from search.
+     * Missing candidate vectors are re-embedded so diversity can still be computed.
+     */
+    private ensureVectors;
     private selectMMR;
     private toRetrievalResult;
 }

package/dist/retrievers/mmr-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mmr-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,kBAAkB;IACjC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAuBD,qBAAa,YAAa,YAAW,SAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,kBAAkB;IAQhC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAuB7F,OAAO,CAAC,SAAS;IAsCjB,OAAO,CAAC,iBAAiB;CAa1B"}
+{"version":3,"file":"mmr-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,kBAAkB;IACjC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAyBD,qBAAa,YAAa,YAAW,SAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,kBAAkB;IAQhC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAyB7F;;;OAGG;YACW,aAAa;IAsB3B,OAAO,CAAC,SAAS;IAuCjB,OAAO,CAAC,iBAAiB;CAa1B"}

package/dist/retrievers/mmr-retriever.js

@@ -1,13 +1,16 @@
+import { resultSource } from './chunk-indexer.js';
 const DEFAULT_TOP_K = 10;
 const DEFAULT_THRESHOLD = 0.0;
 const DEFAULT_LAMBDA = 0.7;
 const CANDIDATE_MULTIPLIER = 3;
 function cosineSimilarity(a, b) {
-    const len = Math.min(a.length, b.length);
+    if (a.length !== b.length) {
+        throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+    }
     let dot = 0;
     let normA = 0;
     let normB = 0;
-    for (let i = 0; i < len; i++) {
+    for (let i = 0; i < a.length; i++) {
         dot += a[i] * b[i];
         normA += a[i] * a[i];
         normB += b[i] * b[i];
@@ -44,11 +47,32 @@ export class MMRRetriever {
         const candidates = result.data;
         if (candidates.length === 0)
             return [];
-        const selected = this.selectMMR(candidates, topK);
+        const lambda = options?.mmrLambda ?? this.defaultLambda;
+        const withVectors = await this.ensureVectors(candidates);
+        const selected = this.selectMMR(withVectors, topK, lambda);
         return selected.map((entry) => this.toRetrievalResult(entry));
     }
-    selectMMR(candidates, topK) {
-        const lambda = this.defaultLambda;
+    /**
+     * Some vector stores (e.g. Qdrant) do not return stored vectors from search.
+     * Missing candidate vectors are re-embedded so diversity can still be computed.
+     */
+    async ensureVectors(candidates) {
+        const missing = candidates
+            .map((candidate, index) => ({ candidate, index }))
+            .filter(({ candidate }) => !candidate.vector || candidate.vector.length === 0);
+        if (missing.length === 0)
+            return candidates;
+        const vectors = await this.embedding.embedBatch(missing.map(({ candidate }) => candidate.content));
+        if (vectors.length !== missing.length) {
+            throw new Error(`Embedding count mismatch: got ${vectors.length} vectors for ${missing.length} candidates`);
+        }
+        const result = [...candidates];
+        missing.forEach(({ candidate, index }, i) => {
+            result[index] = { ...candidate, vector: vectors[i] };
+        });
+        return result;
+    }
+    selectMMR(candidates, topK, lambda) {
         const remaining = new Set(candidates.map((_, i) => i));
         const selected = [];
         while (selected.length < topK && remaining.size > 0) {
@@ -56,12 +80,14 @@ export class MMRRetriever {
             let bestScore = -Infinity;
             for (const idx of remaining) {
                 const relevance = candidates[idx].score;
-                let maxSelectedSim = 0;
+                let maxSelectedSim = -Infinity;
                 for (const sel of selected) {
                     const sim = cosineSimilarity(candidates[idx].vector, sel.vector);
                     if (sim > maxSelectedSim)
                         maxSelectedSim = sim;
                 }
+                if (maxSelectedSim === -Infinity)
+                    maxSelectedSim = 0;
                 const mmrScore = lambda * relevance - (1 - lambda) * maxSelectedSim;
                 if (mmrScore > bestScore) {
                     bestScore = mmrScore;
@@ -83,8 +109,8 @@ export class MMRRetriever {
             documentId,
             content: entry.content,
             score: entry.mmrScore,
-            source: entry.sourceType,
-            metadata: entry.metadata,
+            source: resultSource(entry.metadata, entry.sourceType),
+            metadata: { ...entry.metadata, originalScore: entry.score },
         };
     }
 }