@aiconnect/easy-rag 0.3.1 → 1.0.0

package/dist/query/hybrid.js

@@ -0,0 +1,116 @@
+import { searchVectorStore, searchSparse, getCollection } from './search.js';
+import { generateSparseEmbedding } from '../vector-store/tokenizer.js';
+const RRF_K = 60;
+/**
+ * Auto search: RRF fusion with equal weights (no configuration needed).
+ */
+export function autoSearch(queryEmbedding, queryText, options) {
+    return fusionSearch(queryEmbedding, queryText, {
+        ...options,
+        vectorWeight: 0.5,
+        bm25Weight: 0.5,
+    });
+}
+/**
+ * Weighted search: RRF fusion with configurable weights.
+ */
+export function weightedSearch(queryEmbedding, queryText, options) {
+    return fusionSearch(queryEmbedding, queryText, {
+        ...options,
+        vectorWeight: options.vectorWeight ?? 0.6,
+        bm25Weight: options.bm25Weight ?? 0.4,
+    });
+}
+/**
+ * Core fusion logic: runs dense + sparse queries and merges via RRF.
+ */
+function fusionSearch(queryEmbedding, queryText, options) {
+    const { top, vectorWeight, bm25Weight } = options;
+    const fetchN = top * 2;
+    if (!options.collection) {
+        throw new Error('Collection name is required for hybrid search');
+    }
+    let vectorResults = [];
+    let sparseResults = [];
+    let vectorError = null;
+    let sparseError = null;
+    const collection = getCollection(options.collection);
+    try {
+        vectorResults = searchVectorStore(collection, queryEmbedding, fetchN);
+    }
+    catch (error) {
+        vectorError = error instanceof Error ? error : new Error(String(error));
+    }
+    try {
+        const sparseVector = generateSparseEmbedding(queryText);
+        sparseResults = searchSparse(collection, sparseVector, fetchN);
+    }
+    catch (error) {
+        sparseError = error instanceof Error ? error : new Error(String(error));
+    }
+    collection.closeSync();
+    if (vectorError && sparseError) {
+        throw vectorError;
+    }
+    if (vectorError) {
+        console.warn('Warning: Vector search failed, falling back to sparse-only');
+        return sparseResults.slice(0, top).map((r, idx) => ({
+            ...r,
+            hybridScore: r.similarity,
+            vectorScore: undefined,
+            bm25Score: r.similarity,
+            vectorRank: undefined,
+            bm25Rank: idx + 1,
+        }));
+    }
+    if (sparseError) {
+        console.warn('Warning: Sparse search failed, falling back to vector-only');
+        return vectorResults.slice(0, top).map((r, idx) => ({
+            ...r,
+            hybridScore: r.similarity,
+            vectorScore: r.similarity,
+            bm25Score: undefined,
+            vectorRank: idx + 1,
+            bm25Rank: undefined,
+        }));
+    }
+    const mergedResults = new Map();
+    for (let i = 0; i < vectorResults.length; i++) {
+        const result = vectorResults[i];
+        const key = `${result.metadata.sourcePath}:${result.metadata.chunkIndex}`;
+        const rrfScore = vectorWeight / (RRF_K + i + 1);
+        mergedResults.set(key, {
+            ...result,
+            hybridScore: rrfScore,
+            vectorScore: result.similarity,
+            bm25Score: undefined,
+            vectorRank: i + 1,
+            bm25Rank: undefined,
+        });
+    }
+    for (let i = 0; i < sparseResults.length; i++) {
+        const result = sparseResults[i];
+        const key = `${result.metadata.sourcePath}:${result.metadata.chunkIndex}`;
+        const rrfScore = bm25Weight / (RRF_K + i + 1);
+        const existing = mergedResults.get(key);
+        if (existing) {
+            existing.hybridScore += rrfScore;
+            existing.bm25Score = result.similarity;
+            existing.bm25Rank = i + 1;
+        }
+        else {
+            mergedResults.set(key, {
+                ...result,
+                hybridScore: rrfScore,
+                vectorScore: undefined,
+                bm25Score: result.similarity,
+                vectorRank: undefined,
+                bm25Rank: i + 1,
+            });
+        }
+    }
+    return Array.from(mergedResults.values())
+        .sort((a, b) => b.hybridScore - a.hybridScore)
+        .slice(0, top);
+}
+//# sourceMappingURL=hybrid.js.map

package/dist/query/hybrid.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid.js","sourceRoot":"","sources":["../../src/query/hybrid.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAIvE,MAAM,KAAK,GAAG,EAAE,CAAC;AASjB;;GAEG;AACH,MAAM,UAAU,UAAU,CACxB,cAAwB,EACxB,SAAiB,EACjB,OAAiE;IAEjE,OAAO,YAAY,CAAC,cAAc,EAAE,SAAS,EAAE;QAC7C,GAAG,OAAO;QACV,YAAY,EAAE,GAAG;QACjB,UAAU,EAAE,GAAG;KAChB,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAC5B,cAAwB,EACxB,SAAiB,EACjB,OAA4B;IAE5B,OAAO,YAAY,CAAC,cAAc,EAAE,SAAS,EAAE;QAC7C,GAAG,OAAO;QACV,YAAY,EAAE,OAAO,CAAC,YAAY,IAAI,GAAG;QACzC,UAAU,EAAE,OAAO,CAAC,UAAU,IAAI,GAAG;KACtC,CAAC,CAAC;AACL,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CACnB,cAAwB,EACxB,SAAiB,EACjB,OAA6H;IAE7H,MAAM,EAAE,GAAG,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IAClD,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC;IAEvB,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,aAAa,GAAkB,EAAE,CAAC;IACtC,IAAI,aAAa,GAAkB,EAAE,CAAC;IACtC,IAAI,WAAW,GAAiB,IAAI,CAAC;IACrC,IAAI,WAAW,GAAiB,IAAI,CAAC;IAErC,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAErD,IAAI,CAAC;QACH,aAAa,GAAG,iBAAiB,CAAC,UAAU,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;IACxE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,IAAI,CAAC;QACH,MAAM,YAAY,GAAG,uBAAuB,CAAC,SAAS,CAAC,CAAC;QACxD,aAAa,GAAG,YAAY,CAAC,UAAU,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IACjE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,WAAW,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1E,CAAC;IAED,UAAU,CAAC,SAAS,EAAE,CAAC;IAEvB,IAAI,WAAW,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,WAAW,CAAC;IACpB,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QAC3E,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAClD,GAAG,CAAC;YACJ,WAAW,EAAE,CAAC,CAAC,UAAU;YACzB,WAAW,EAAE,SAAS;YACtB,SAAS,EAAE,CAAC,CAAC,UAAU;YACvB,UAAU,EAAE,SAAS;YACrB,QAAQ,EAAE,GAAG,GAAG,CAAC;SAClB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;QAC3E,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;YAClD,GAAG,CAAC;YACJ,WAAW,EAAE,CAAC,CAAC,UAAU;YACzB,WAAW,EAAE,CAAC,CAAC,UAAU;YACzB,SAAS,EAAE,SAAS;YACpB,UAAU,EAAE,GAAG,GAAG,CAAC;YACnB,QAAQ,EAAE,SAAS;SACpB,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,GAAG,EAA6B,CAAC;IAE3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC1E,MAAM,QAAQ,GAAG,YAAY,GAAG,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE;YACrB,GAAG,MAAM;YACT,WAAW,EAAE,QAAQ;YACrB,WAAW,EAAE,MAAM,CAAC,UAAU;YAC9B,SAAS,EAAE,SAAS;YACpB,UAAU,EAAE,CAAC,GAAG,CAAC;YACjB,QAAQ,EAAE,SAAS;SACpB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC;QAC1E,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9C,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,WAAW,IAAI,QAAQ,CAAC;YACjC,QAAQ,CAAC,SAAS,GAAG,MAAM,CAAC,UAAU,CAAC;YACvC,QAAQ,CAAC,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE;gBACrB,GAAG,MAAM;gBACT,WAAW,EAAE,QAAQ;gBACrB,WAAW,EAAE,SAAS;gBACtB,SAAS,EAAE,MAAM,CAAC,UAAU;gBAC5B,UAAU,EAAE,SAAS;gBACrB,QAAQ,EAAE,CAAC,GAAG,CAAC;aAChB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;SACtC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;SAC7C,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACnB,CAAC"}

package/dist/query/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/query/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,YAAY,EAAE,MAAM,YAAY,CAAC;AAI5D,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAiEzF"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/query/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,YAAY,EAAE,MAAM,YAAY,CAAC;AAQ5D,wBAAsB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAmHzF"}

package/dist/query/index.js

@@ -1,5 +1,8 @@
 import { generateQueryEmbedding } from './embedding.js';
-import { listAllCollections, getCollection, searchChromaDB } from './search.js';
+import { listAllCollections, getCollection, searchVectorStore, searchSparse } from './search.js';
+import { autoSearch, weightedSearch } from './hybrid.js';
+import { generateSparseEmbedding } from '../vector-store/tokenizer.js';
+import { loadConfig } from '../config/index.js';
 export async function executeQuery(question, options) {
     if (!process.env.OPENAI_API_KEY) {
         console.error('Error: OPENAI_API_KEY environment variable is required');
@@ -9,19 +12,56 @@ export async function executeQuery(question, options) {
         console.error('  OPENAI_API_KEY=your-api-key-here easy-rag query "<question>"');
         process.exit(1);
     }
-    const queryEmbedding = await generateQueryEmbedding(question);
-    const allResults = [];
+    const config = await loadConfig();
+    const mode = options.mode ?? config.search_mode ?? 'auto';
+    let allResults = [];
     if (options.collection) {
         try {
-            const collection = await getCollection(options.collection);
-            const results = await searchChromaDB(collection, queryEmbedding, options.top);
-            allResults.push(...results);
+            if (mode === 'auto') {
+                const queryEmbedding = await generateQueryEmbedding(question);
+                const results = autoSearch(queryEmbedding, question, {
+                    top: options.top,
+                    collection: options.collection,
+                });
+                allResults = results;
+            }
+            else if (mode === 'hybrid') {
+                const queryEmbedding = await generateQueryEmbedding(question);
+                const results = weightedSearch(queryEmbedding, question, {
+                    top: options.top,
+                    collection: options.collection,
+                    vectorWeight: config.vector_weight,
+                    bm25Weight: config.bm25_weight,
+                });
+                allResults = results;
+            }
+            else if (mode === 'vector') {
+                const queryEmbedding = await generateQueryEmbedding(question);
+                const collection = getCollection(options.collection);
+                const results = searchVectorStore(collection, queryEmbedding, options.top);
+                collection.closeSync();
+                allResults = results;
+            }
+            else if (mode === 'bm25') {
+                const collection = getCollection(options.collection);
+                const sparseVector = generateSparseEmbedding(question);
+                const results = searchSparse(collection, sparseVector, options.top);
+                collection.closeSync();
+                allResults = results.map((r, idx) => ({
+                    ...r,
+                    hybridScore: r.similarity,
+                    vectorScore: undefined,
+                    bm25Score: r.similarity,
+                    vectorRank: undefined,
+                    bm25Rank: idx + 1,
+                }));
+            }
         }
         catch (error) {
             if (error instanceof Error && error.message.includes('Collection')) {
                 console.error(`Error: Collection "${options.collection}" not found`);
                 console.error('\nAvailable collections:');
-                const collections = await listAllCollections();
+                const collections = listAllCollections();
                 if (collections.length === 0) {
                     console.error('  (no collections found)');
                 }
@@ -34,34 +74,48 @@ export async function executeQuery(question, options) {
         }
     }
     else {
-        const collectionNames = await listAllCollections();
+        const collectionNames = listAllCollections();
         if (collectionNames.length === 0) {
-            console.error('Error: No collections found in ChromaDB');
+            console.error('Error: No collections found');
             console.error('\nPlease run "easy-rag index <folder>" to create a collection first.');
             process.exit(1);
         }
+        if (mode !== 'auto' && mode !== 'hybrid') {
+            console.error('Error: Searching across all collections is only supported in auto/hybrid mode');
+            console.error('Please specify a collection with the -c flag');
+            process.exit(1);
+        }
+        const queryEmbedding = await generateQueryEmbedding(question);
         for (const name of collectionNames) {
             try {
-                const collection = await getCollection(name);
-                const results = await searchChromaDB(collection, queryEmbedding, options.top);
+                const results = mode === 'auto'
+                    ? autoSearch(queryEmbedding, question, { top: options.top, collection: name })
+                    : weightedSearch(queryEmbedding, question, {
+                        top: options.top,
+                        collection: name,
+                        vectorWeight: config.vector_weight,
+                        bm25Weight: config.bm25_weight,
+                    });
                 allResults.push(...results);
             }
             catch (error) {
                 console.error(`Warning: Failed to search collection "${name}": ${error instanceof Error ? error.message : String(error)}`);
             }
         }
+        if (allResults.length > 1) {
+            allResults.sort((a, b) => b.hybridScore - a.hybridScore);
+            allResults = allResults.slice(0, options.top);
+        }
     }
     if (allResults.length === 0) {
         console.log('No results found.');
         return;
     }
-    allResults.sort((a, b) => b.similarity - a.similarity);
-    const topResults = allResults.slice(0, options.top);
     if (options.includeMetadata) {
-        displayResultsWithMetadata(topResults);
+        displayResultsWithMetadata(allResults);
     }
     else {
-        displayResults(topResults);
+        displayResults(allResults);
     }
 }
 function displayResults(results) {
@@ -77,7 +131,18 @@ function displayResultsWithMetadata(results) {
     results.forEach((result, index) => {
         console.log(`--- Result ${index + 1} ---`);
         console.log(`Source: ${result.metadata.sourcePath}`);
-        console.log(`Similarity: ${(result.similarity * 100).toFixed(1)}%`);
+        if (result.hybridScore !== undefined) {
+            console.log(`Hybrid Score: ${result.hybridScore.toFixed(4)}`);
+        }
+        if (result.vectorScore !== undefined) {
+            console.log(`Vector Score: ${result.vectorScore.toFixed(4)} (rank: ${result.vectorRank})`);
+        }
+        if (result.bm25Score !== undefined) {
+            console.log(`BM25 Score: ${result.bm25Score.toFixed(4)} (rank: ${result.bm25Rank})`);
+        }
+        if (result.similarity > 0) {
+            console.log(`Similarity: ${(result.similarity * 100).toFixed(1)}%`);
+        }
         console.log(`Chunk: ${result.metadata.chunkIndex}`);
         console.log(`\nContent:\n${result.content}`);
         console.log();

package/dist/query/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/query/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAEhF,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,OAAqB;IACxE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;QACxE,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxC,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IAE9D,MAAM,UAAU,GAAkB,EAAE,CAAC;IAErC,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;YAC3D,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;YAC9E,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACnE,OAAO,CAAC,KAAK,CAAC,sBAAsB,OAAO,CAAC,UAAU,aAAa,CAAC,CAAC;gBACrE,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAC1C,MAAM,WAAW,GAAG,MAAM,kBAAkB,EAAE,CAAC;gBAC/C,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC7B,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAC5C,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;gBACxD,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,eAAe,GAAG,MAAM,kBAAkB,EAAE,CAAC;QACnD,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,KAAK,CAAC,yCAAyC,CAAC,CAAC;YACzD,OAAO,CAAC,KAAK,CAAC,sEAAsE,CAAC,CAAC;YACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC9E,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,yCAAyC,IAAI,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7H,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;IACvD,MAAM,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IAEpD,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;QAC5B,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACzC,CAAC;SAAM,CAAC;QACN,cAAc,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,OAAsB;IAC5C,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAChC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,0BAA0B,CAAC,OAAsB;IACxD,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAChC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACpE,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/query/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACjG,OAAO,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,QAAgB,EAAE,OAAqB;IACxE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,wDAAwD,CAAC,CAAC;QACxE,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxC,OAAO,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC3D,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAC;QAChF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC;IAE1D,IAAI,UAAU,GAAwC,EAAE,CAAC;IAEzD,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBACpB,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;gBAC9D,MAAM,OAAO,GAAG,UAAU,CAAC,cAAc,EAAE,QAAQ,EAAE;oBACnD,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,UAAU,EAAE,OAAO,CAAC,UAAU;iBAC/B,CAAC,CAAC;gBACH,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC;iBAAM,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;gBAC9D,MAAM,OAAO,GAAG,cAAc,CAAC,cAAc,EAAE,QAAQ,EAAE;oBACvD,GAAG,EAAE,OAAO,CAAC,GAAG;oBAChB,UAAU,EAAE,OAAO,CAAC,UAAU;oBAC9B,YAAY,EAAE,MAAM,CAAC,aAAa;oBAClC,UAAU,EAAE,MAAM,CAAC,WAAW;iBAC/B,CAAC,CAAC;gBACH,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC;iBAAM,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBAC7B,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;gBAC9D,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACrD,MAAM,OAAO,GAAG,iBAAiB,CAAC,UAAU,EAAE,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC3E,UAAU,CAAC,SAAS,EAAE,CAAC;gBACvB,UAAU,GAAG,OAAO,CAAC;YACvB,CAAC;iBAAM,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC3B,MAAM,UAAU,GAAG,aAAa,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;gBACrD,MAAM,YAAY,GAAG,uBAAuB,CAAC,QAAQ,CAAC,CAAC;gBACvD,MAAM,OAAO,GAAG,YAAY,CAAC,UAAU,EAAE,YAAY,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;gBACpE,UAAU,CAAC,SAAS,EAAE,CAAC;gBACvB,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;oBACpC,GAAG,CAAC;oBACJ,WAAW,EAAE,CAAC,CAAC,UAAU;oBACzB,WAAW,EAAE,SAAS;oBACtB,SAAS,EAAE,CAAC,CAAC,UAAU;oBACvB,UAAU,EAAE,SAAS;oBACrB,QAAQ,EAAE,GAAG,GAAG,CAAC;iBAClB,CAAC,CAAC,CAAC;YACN,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACnE,OAAO,CAAC,KAAK,CAAC,sBAAsB,OAAO,CAAC,UAAU,aAAa,CAAC,CAAC;gBACrE,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAC1C,MAAM,WAAW,GAAG,kBAAkB,EAAE,CAAC;gBACzC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC7B,OAAO,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;gBAC5C,CAAC;qBAAM,CAAC;oBACN,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;gBACxD,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;SAAM,CAAC;QACN,MAAM,eAAe,GAAG,kBAAkB,EAAE,CAAC;QAC7C,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC7C,OAAO,CAAC,KAAK,CAAC,sEAAsE,CAAC,CAAC;YACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACzC,OAAO,CAAC,KAAK,CAAC,+EAA+E,CAAC,CAAC;YAC/F,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,sBAAsB,CAAC,QAAQ,CAAC,CAAC;QAE9D,KAAK,MAAM,IAAI,IAAI,eAAe,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,IAAI,KAAK,MAAM;oBAC7B,CAAC,CAAC,UAAU,CAAC,cAAc,EAAE,QAAQ,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;oBAC9E,CAAC,CAAC,cAAc,CAAC,cAAc,EAAE,QAAQ,EAAE;wBACvC,GAAG,EAAE,OAAO,CAAC,GAAG;wBAChB,UAAU,EAAE,IAAI;wBAChB,YAAY,EAAE,MAAM,CAAC,aAAa;wBAClC,UAAU,EAAE,MAAM,CAAC,WAAW;qBAC/B,CAAC,CAAC;gBACP,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,yCAAyC,IAAI,MAAM,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC7H,CAAC;QACH,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,UAAkC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;YAClF,UAAU,GAAI,UAAkC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;QAC5B,0BAA0B,CAAC,UAAiC,CAAC,CAAC;IAChE,CAAC;SAAM,CAAC;QACN,cAAc,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,OAAsB;IAC5C,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAChC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,0BAA0B,CAAC,OAA4B;IAC9D,OAAO,CAAC,GAAG,EAAE,CAAC;IACd,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAChC,OAAO,CAAC,GAAG,CAAC,cAAc,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACrD,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChE,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC;QAC7F,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACnC,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACtE,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,UAAU,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7C,OAAO,CAAC,GAAG,EAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/query/search.d.ts

@@ -1,6 +1,9 @@
-import { type Collection } from 'chromadb';
 import type { QueryResult } from './types.js';
-export declare function listAllCollections(): Promise<string[]>;
-export declare function getCollection(collectionName: string): Promise<Collection>;
-export declare function searchChromaDB(collection: Collection, queryEmbedding: number[], top: number): Promise<QueryResult[]>;
+import { openCollection } from '../vector-store/index.js';
+type ZVecCollection = ReturnType<typeof openCollection>;
+export declare function listAllCollections(): string[];
+export declare function getCollection(collectionName: string): ZVecCollection;
+export declare function searchVectorStore(collection: ZVecCollection, queryEmbedding: number[], top: number): QueryResult[];
+export declare function searchSparse(collection: ZVecCollection, sparseVector: Record<number, number>, top: number): QueryResult[];
+export {};
 //# sourceMappingURL=search.d.ts.map

package/dist/query/search.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/query/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,UAAU,EAAE,MAAM,UAAU,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAM9C,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC,CAG5D;AAED,wBAAsB,aAAa,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CAK/E;AAED,wBAAsB,cAAc,CAClC,UAAU,EAAE,UAAU,EACtB,cAAc,EAAE,MAAM,EAAE,EACxB,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,WAAW,EAAE,CAAC,CAiCxB"}
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/query/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAmB,MAAM,0BAA0B,CAAC;AAE3E,KAAK,cAAc,GAAG,UAAU,CAAC,OAAO,cAAc,CAAC,CAAC;AAExD,wBAAgB,kBAAkB,IAAI,MAAM,EAAE,CAE7C;AAED,wBAAgB,aAAa,CAAC,cAAc,EAAE,MAAM,GAAG,cAAc,CAEpE;AAED,wBAAgB,iBAAiB,CAC/B,UAAU,EAAE,cAAc,EAC1B,cAAc,EAAE,MAAM,EAAE,EACxB,GAAG,EAAE,MAAM,GACV,WAAW,EAAE,CAqBf;AAED,wBAAgB,YAAY,CAC1B,UAAU,EAAE,cAAc,EAC1B,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EACpC,GAAG,EAAE,MAAM,GACV,WAAW,EAAE,CAqBf"}

package/dist/query/search.js

@@ -1,45 +1,50 @@
-import { ChromaClient } from 'chromadb';
-const client = new ChromaClient({
-    path: process.env.CHROMA_URL || 'http://localhost:8000',
-});
-export async function listAllCollections() {
-    const collections = await client.listCollections();
-    return collections.map((c) => String(c));
+import { openCollection, listCollections } from '../vector-store/index.js';
+export function listAllCollections() {
+    return listCollections().map((c) => c.name);
 }
-export async function getCollection(collectionName) {
-    const sanitized = collectionName
-        .replace(/[^a-zA-Z0-9_-]/g, '_')
-        .toLowerCase();
-    return await client.getCollection({ name: sanitized });
+export function getCollection(collectionName) {
+    return openCollection(collectionName);
 }
-export async function searchChromaDB(collection, queryEmbedding, top) {
-    const result = await collection.query({
-        queryEmbeddings: [queryEmbedding],
-        nResults: top,
+export function searchVectorStore(collection, queryEmbedding, top) {
+    const results = collection.querySync({
+        fieldName: 'embedding',
+        vector: queryEmbedding,
+        topk: top,
+        outputFields: ['content', 'sourcePath', 'contentType', 'chunkIndex', 'sectionTitle', 'startPosition', 'endPosition', 'rowNumber'],
     });
-    const queryResults = [];
-    if (result.documents && result.documents[0]) {
-        for (let i = 0; i < result.documents[0].length; i++) {
-            const content = result.documents[0][i];
-            const metadata = result.metadatas?.[0]?.[i];
-            const distance = result.distances?.[0]?.[i];
-            if (content && metadata) {
-                queryResults.push({
-                    content,
-                    metadata: {
-                        sourcePath: String(metadata.sourcePath),
-                        contentType: String(metadata.contentType),
-                        chunkIndex: Number(metadata.chunkIndex),
-                        sectionTitle: metadata.sectionTitle !== undefined ? String(metadata.sectionTitle) : undefined,
-                        startPosition: metadata.startPosition !== undefined ? Number(metadata.startPosition) : undefined,
-                        endPosition: metadata.endPosition !== undefined ? Number(metadata.endPosition) : undefined,
-                        rowNumber: metadata.rowNumber !== undefined ? Number(metadata.rowNumber) : undefined,
-                    },
-                    similarity: distance !== undefined ? Math.max(0, 1 - distance) : 0,
-                });
-            }
-        }
-    }
-    return queryResults;
+    return results.map((doc) => ({
+        content: String(doc.fields.content ?? ''),
+        metadata: {
+            sourcePath: String(doc.fields.sourcePath),
+            contentType: String(doc.fields.contentType),
+            chunkIndex: Number(doc.fields.chunkIndex),
+            sectionTitle: doc.fields.sectionTitle !== undefined ? String(doc.fields.sectionTitle) : undefined,
+            startPosition: doc.fields.startPosition !== undefined ? Number(doc.fields.startPosition) : undefined,
+            endPosition: doc.fields.endPosition !== undefined ? Number(doc.fields.endPosition) : undefined,
+            rowNumber: doc.fields.rowNumber !== undefined ? Number(doc.fields.rowNumber) : undefined,
+        },
+        similarity: doc.score,
+    }));
+}
+export function searchSparse(collection, sparseVector, top) {
+    const results = collection.querySync({
+        fieldName: 'sparse_embedding',
+        vector: sparseVector,
+        topk: top,
+        outputFields: ['content', 'sourcePath', 'contentType', 'chunkIndex', 'sectionTitle', 'startPosition', 'endPosition', 'rowNumber'],
+    });
+    return results.map((doc) => ({
+        content: String(doc.fields.content ?? ''),
+        metadata: {
+            sourcePath: String(doc.fields.sourcePath),
+            contentType: String(doc.fields.contentType),
+            chunkIndex: Number(doc.fields.chunkIndex),
+            sectionTitle: doc.fields.sectionTitle !== undefined ? String(doc.fields.sectionTitle) : undefined,
+            startPosition: doc.fields.startPosition !== undefined ? Number(doc.fields.startPosition) : undefined,
+            endPosition: doc.fields.endPosition !== undefined ? Number(doc.fields.endPosition) : undefined,
+            rowNumber: doc.fields.rowNumber !== undefined ? Number(doc.fields.rowNumber) : undefined,
+        },
+        similarity: doc.score,
+    }));
 }
 //# sourceMappingURL=search.js.map

package/dist/query/search.js.map

@@ -1 +1 @@
-{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/query/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAmB,MAAM,UAAU,CAAC;AAGzD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC;IAC9B,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,uBAAuB;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,MAAM,WAAW,GAAG,MAAM,MAAM,CAAC,eAAe,EAAE,CAAC;IACnD,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,cAAsB;IACxD,MAAM,SAAS,GAAG,cAAc;SAC7B,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,WAAW,EAAE,CAAC;IACjB,OAAO,MAAM,MAAM,CAAC,aAAa,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;AACzD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,UAAsB,EACtB,cAAwB,EACxB,GAAW;IAEX,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC;QACpC,eAAe,EAAE,CAAC,cAAc,CAAC;QACjC,QAAQ,EAAE,GAAG;KACd,CAAC,CAAC;IAEH,MAAM,YAAY,GAAkB,EAAE,CAAC;IAEvC,IAAI,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,MAAM,OAAO,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAA4B,CAAC;YACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAE5C,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,YAAY,CAAC,IAAI,CAAC;oBAChB,OAAO;oBACP,QAAQ,EAAE;wBACR,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;wBACvC,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC;wBACzC,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;wBACvC,YAAY,EAAE,QAAQ,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS;wBAC7F,aAAa,EAAE,QAAQ,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;wBAChG,WAAW,EAAE,QAAQ,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;wBAC1F,SAAS,EAAE,QAAQ,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;qBACrF;oBACD,UAAU,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;iBACnE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,YAAY,CAAC;AACtB,CAAC"}
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/query/search.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAC;AAI3E,MAAM,UAAU,kBAAkB;IAChC,OAAO,eAAe,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,cAAsB;IAClD,OAAO,cAAc,CAAC,cAAc,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,UAA0B,EAC1B,cAAwB,EACxB,GAAW;IAEX,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC;QACnC,SAAS,EAAE,WAAW;QACtB,MAAM,EAAE,cAAc;QACtB,IAAI,EAAE,GAAG;QACT,YAAY,EAAE,CAAC,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW,CAAC;KAClI,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAuD,EAAE,EAAE,CAAC,CAAC;QAC/E,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QACzC,QAAQ,EAAE;YACR,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC;YACzC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC;YAC3C,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC;YACzC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS;YACjG,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;YACpG,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9F,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;SACzF;QACD,UAAU,EAAE,GAAG,CAAC,KAAK;KACtB,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,UAA0B,EAC1B,YAAoC,EACpC,GAAW;IAEX,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC;QACnC,SAAS,EAAE,kBAAkB;QAC7B,MAAM,EAAE,YAAY;QACpB,IAAI,EAAE,GAAG;QACT,YAAY,EAAE,CAAC,SAAS,EAAE,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,cAAc,EAAE,eAAe,EAAE,aAAa,EAAE,WAAW,CAAC;KAClI,CAAC,CAAC;IAEH,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,GAAuD,EAAE,EAAE,CAAC,CAAC;QAC/E,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QACzC,QAAQ,EAAE;YACR,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC;YACzC,WAAW,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC;YAC3C,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC;YACzC,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS;YACjG,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS;YACpG,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS;YAC9F,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS;SACzF;QACD,UAAU,EAAE,GAAG,CAAC,KAAK;KACtB,CAAC,CAAC,CAAC;AACN,CAAC"}

package/dist/query/types.d.ts

@@ -11,9 +11,17 @@ export interface QueryResult {
     };
     similarity: number;
 }
+export interface HybridQueryResult extends QueryResult {
+    hybridScore: number;
+    vectorScore?: number;
+    bm25Score?: number;
+    vectorRank?: number;
+    bm25Rank?: number;
+}
 export interface QueryOptions {
     top: number;
     includeMetadata: boolean;
     collection?: string;
+    mode?: 'auto' | 'hybrid' | 'vector' | 'bm25';
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/query/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/query/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE;QACR,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/query/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE;QACR,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,EAAE,MAAM,CAAC;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IACF,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAkB,SAAQ,WAAW;IACpD,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC9C"}

package/dist/vector-store/index.d.ts

@@ -1,4 +1,5 @@
-export { getOrCreateCollection, storeEmbeddings, initializeChromaDB, listCollections, deleteCollection } from './chromadb.js';
-export { ensureChromaDataDirectory } from './utils.js';
+export { getOrCreateCollection, openCollection, storeEmbeddings, listCollections, deleteCollection, getDimensionForModel, } from './zvec.js';
+export type { SparseChunk } from './zvec.js';
+export { ensureCollectionsDirectory, getCollectionPath, sanitizeCollectionName } from './utils.js';
 export type { VectorChunk, CollectionInfo } from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/vector-store/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/vector-store/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,kBAAkB,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC9H,OAAO,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AACvD,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/vector-store/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,WAAW,CAAC;AACnB,YAAY,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAC7C,OAAO,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC;AACnG,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC"}

package/dist/vector-store/index.js

@@ -1,3 +1,3 @@
-export { getOrCreateCollection, storeEmbeddings, initializeChromaDB, listCollections, deleteCollection } from './chromadb.js';
-export { ensureChromaDataDirectory } from './utils.js';
+export { getOrCreateCollection, openCollection, storeEmbeddings, listCollections, deleteCollection, getDimensionForModel, } from './zvec.js';
+export { ensureCollectionsDirectory, getCollectionPath, sanitizeCollectionName } from './utils.js';
 //# sourceMappingURL=index.js.map

package/dist/vector-store/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/vector-store/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,kBAAkB,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAC9H,OAAO,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/vector-store/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,qBAAqB,EACrB,cAAc,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,WAAW,CAAC;AAEnB,OAAO,EAAE,0BAA0B,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAC"}

package/dist/vector-store/tokenizer.d.ts

@@ -0,0 +1,3 @@
+export declare function tokenize(text: string): string[];
+export declare function generateSparseEmbedding(text: string): Record<number, number>;
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/vector-store/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/vector-store/tokenizer.ts"],"names":[],"mappings":"AA6DA,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAK/C;AAWD,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAoB5E"}

package/dist/vector-store/tokenizer.js

@@ -0,0 +1,91 @@
+const ENGLISH_STOPWORDS = new Set([
+    'a', 'an', 'the', 'and', 'or', 'but', 'is', 'are', 'was', 'were', 'be', 'been', 'being',
+    'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'could', 'should', 'may',
+    'might', 'must', 'shall', 'can', 'to', 'of', 'in', 'for', 'on', 'with', 'at', 'by',
+    'from', 'as', 'into', 'through', 'during', 'before', 'after', 'above', 'below',
+    'between', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when',
+    'where', 'why', 'how', 'all', 'each', 'few', 'more', 'most', 'other', 'some', 'such',
+    'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 'just',
+    'now', 'he', 'him', 'his', 'himself', 'she', 'her', 'hers', 'herself', 'it', 'its',
+    'itself', 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'who',
+    'whom', 'this', 'that', 'these', 'those', 'i', 'me', 'my', 'myself', 'we', 'us',
+    'our', 'ours', 'ourselves', 'you', 'your', 'yours', 'yourself', 'yourselves', 'over',
+]);
+const PORTUGUESE_STOPWORDS = new Set([
+    'a', 'agora', 'ainda', 'alguem', 'algum', 'alguma', 'algumas', 'alguns', 'ampla',
+    'amplas', 'amplo', 'amplos', 'ante', 'antes', 'ao', 'aos', 'apenas', 'apos',
+    'aquela', 'aquelas', 'aquele', 'aqueles', 'as', 'asqueroso', 'atras', 'ate', 'aí',
+    'baixo', 'bastante', 'bem', 'boa', 'boas', 'bom', 'bons', 'breve', 'cada', 'caminho',
+    'cara', 'caras', 'certeza', 'clara', 'claras', 'claro', 'claros', 'com', 'como',
+    'comprido', 'comprida', 'compridas', 'compridos', 'conhecido', 'conhecida', 'conhecidas',
+    'conhecidos', 'consigo', 'contigo', 'contra', 'contudo', 'custa', 'da', 'daquele',
+    'daquela', 'daquelas', 'daqueles', 'das', 'de', 'debate', 'dentro', 'depois',
+    'desde', 'deixado', 'ele', 'eles', 'ela', 'elas', 'em', 'enquanto', 'entao', 'entre',
+    'era', 'eram', 'essa', 'essas', 'esse', 'esses', 'esta', 'estamos', 'estas', 'estava',
+    'estavam', 'este', 'esteja', 'estejam', 'estejamos', 'estes', 'estiveste', 'estivestes',
+    'estou', 'estavamos', 'eu', 'faco', 'fais', 'famos', 'fazem', 'fazemos', 'fazendo',
+    'faz', 'farao', 'farei', 'faras', 'faremos', 'faria', 'faríamos', 'fazeis', 'feita',
+    'feitas', 'feito', 'feitos', 'foi', 'fomos', 'for', 'fora', 'foram', 'formas', 'fosse',
+    'fossem', 'fossemos', 'fui', 'geral', 'gerais', 'ha', 'haja', 'hajam', 'hajamos',
+    'havemos', 'havia', 'hei', 'hoje', 'houve', 'houvemos', 'houvesse', 'houvessem',
+    'houvessemos', 'isso', 'isto', 'ja', 'la', 'lhe', 'lhes', 'logo', 'longo', 'longa',
+    'longas', 'longos', 'mais', 'mas', 'me', 'medo', 'menor', 'menores', 'meu', 'meus',
+    'mil', 'momento', 'muito', 'muitos', 'na', 'nada', 'nao', 'nas', 'nem', 'nenhum',
+    'nessa', 'nessas', 'nesta', 'neste', 'ninguem', 'no', 'nos', 'nosotros', 'nosso',
+    'nossa', 'nossos', 'nossas', 'noutras', 'noutros', 'num', 'numa', 'numas', 'nuns',
+    'o', 'os', 'ou', 'outro', 'outros', 'pela', 'pelas', 'pelo', 'pelos', 'pequena',
+    'pequenas', 'pequeno', 'pequenos', 'perante', 'pode', 'podem', 'podemos', 'poder',
+    'podia', 'podias', 'porem', 'porque', 'posso', 'pouca', 'poucas', 'pouco', 'poucos',
+    'primeira', 'primeiro', 'propria', 'proprio', 'propios', 'qual', 'quais',
+    'quando', 'quanto', 'quantos', 'quarta', 'quarto', 'quatro', 'que', 'quem', 'quer',
+    'queres', 'queria', 'queriam', 'querias', 'quero', 'quisesse',
+    'quiseste', 'quiser', 'quiseram', 'quisera',
+    'queremos', 'quisermos', 'quis', 'se', 'segunda',
+    'segundo', 'sei', 'seis', 'seja', 'sejam', 'sejamos', 'sem', 'semelhante', 'sendo',
+    'ser', 'serei', 'seremos', 'seria', 'seriam', 'seriamos', 'seu', 'seus',
+    'sinto', 'sobre', 'sois', 'somente', 'somos', 'sou', 'sua', 'suas',
+    'tao', 'tem', 'tamanho', 'tanta', 'tantas', 'tanto', 'tantos', 'te', 'temos',
+    'tenho', 'tens', 'ter', 'teria', 'teriam', 'teriamos', 'teremos', 'tente',
+    'tentamos', 'tentei', 'tentaram', 'tentaste', 'tentastes', 'tes', 'teu',
+    'teus', 'teve', 'tinha', 'tinham', 'tinhamos', 'tive', 'tivemos', 'tiver', 'tiveram',
+    'tivera', 'tivermos', 'tivesse', 'tivessem', 'tivessemos', 'tiveste',
+    'tivestes', 'toda', 'todas', 'todo', 'todos', 'trabalhar', 'trabalha', 'trabalham',
+    'trabalhamos', 'tre', 'tres', 'tu', 'tua', 'tuas', 'ultimo', 'uniao',
+    'um', 'uma', 'umas', 'uns', 'vai', 'vais', 'vaivem', 'vao', 'ver', 'verde',
+    'vem', 'vendo', 'vez', 'vezes', 'viagem', 'voce', 'voces',
+    'vos', 'vosso', 'vossa', 'vossos', 'vossas', 'vontade', 'vs', 'zero',
+]);
+const STOPWORDS = new Set([...ENGLISH_STOPWORDS, ...PORTUGUESE_STOPWORDS]);
+export function tokenize(text) {
+    const lowercased = text.toLowerCase();
+    const noPunctuation = lowercased.replace(/[^\p{L}\p{N}\s]/gu, ' ');
+    const tokens = noPunctuation.split(/\s+/).filter(token => token.length > 0);
+    return tokens.filter(token => !STOPWORDS.has(token));
+}
+function hashToken(token) {
+    let hash = 0;
+    for (let i = 0; i < token.length; i++) {
+        const char = token.charCodeAt(i);
+        hash = ((hash << 5) - hash + char) | 0;
+    }
+    return hash >>> 0;
+}
+export function generateSparseEmbedding(text) {
+    const tokens = tokenize(text);
+    if (tokens.length === 0) {
+        return {};
+    }
+    const termFrequencies = new Map();
+    for (const token of tokens) {
+        termFrequencies.set(token, (termFrequencies.get(token) ?? 0) + 1);
+    }
+    const sparse = {};
+    for (const [token, count] of termFrequencies) {
+        const tf = count / tokens.length;
+        const hash = hashToken(token);
+        const existing = sparse[hash];
+        sparse[hash] = existing !== undefined ? existing + tf : tf;
+    }
+    return sparse;
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/vector-store/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/vector-store/tokenizer.ts"],"names":[],"mappings":"AAAA,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IACvF,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK;IACpF,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI;IAClF,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO;IAC9E,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IAC/E,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IACpF,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IAC9E,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK;IAClF,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK;IACjF,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI;IAC/E,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM;CACrF,CAAC,CAAC;AAEH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAAC;IACnC,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,OAAO;IAChF,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM;IAC3E,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI;IACjF,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS;IACpF,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM;IAC/E,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,YAAY;IACxF,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS;IACjF,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ;IAC5E,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO;IACpF,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ;IACrF,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,WAAW,EAAE,OAAO,EAAE,WAAW,EAAE,YAAY;IACvF,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,SAAS;IAClF,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO;IACnF,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO;IACtF,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;IAChF,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW;IAC/E,aAAa,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO;IAClF,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM;IAClF,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,QAAQ;IAChF,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,UAAU,EAAE,OAAO;IAChF,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IACjF,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS;IAC/E,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO;IACjF,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ;IACnF,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO;IACxE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM;IAClF,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,UAAU;IAC7D,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS;IAC3C,UAAU,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS;IAChD,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO;IAClF,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM;IACvE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM;IAClE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO;IAC5E,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,OAAO;IACzE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK;IACvE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS;IACpF,QAAQ,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,YAAY,EAAE,SAAS;IACpE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW;IAClF,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,OAAO;IACpE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO;IAC1E,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO;IACzD,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM;CACrE,CAAC,CAAC;AAEH,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,iBAAiB,EAAE,GAAG,oBAAoB,CAAC,CAAC,CAAC;AAE3E,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACtC,MAAM,aAAa,GAAG,UAAU,CAAC,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC;IACnE,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC5E,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACjC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,IAAI,KAAK,CAAC,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;IAClD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpE,CAAC;IAED,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,eAAe,EAAE,CAAC;QAC7C,MAAM,EAAE,GAAG,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;QACjC,MAAM,IAAI,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}