@matperez/coderag 0.1.24

package/dist/storage.test.js

@@ -0,0 +1,157 @@
+/**
+ * Tests for in-memory storage
+ */
+import { beforeEach, describe, expect, it } from 'vitest';
+import { MemoryStorage } from './storage.js';
+describe('MemoryStorage', () => {
+    let storage;
+    beforeEach(() => {
+        storage = new MemoryStorage();
+    });
+    describe('storeFile', () => {
+        it('should store a file', async () => {
+            const file = {
+                path: 'test.ts',
+                content: 'console.log("test");',
+                size: 100,
+                mtime: Date.now(),
+                language: 'TypeScript',
+                hash: 'abc123',
+            };
+            await storage.storeFile(file);
+            const retrieved = await storage.getFile('test.ts');
+            expect(retrieved).toEqual(file);
+        });
+        it('should overwrite existing file', async () => {
+            const file1 = {
+                path: 'test.ts',
+                content: 'old content',
+                size: 100,
+                mtime: Date.now(),
+                hash: 'hash1',
+            };
+            const file2 = {
+                path: 'test.ts',
+                content: 'new content',
+                size: 150,
+                mtime: Date.now(),
+                hash: 'hash2',
+            };
+            await storage.storeFile(file1);
+            await storage.storeFile(file2);
+            const retrieved = await storage.getFile('test.ts');
+            expect(retrieved?.content).toBe('new content');
+            expect(retrieved?.hash).toBe('hash2');
+        });
+    });
+    describe('getFile', () => {
+        it('should retrieve stored file', async () => {
+            const file = {
+                path: 'test.ts',
+                content: 'test',
+                size: 100,
+                mtime: Date.now(),
+                hash: 'abc',
+            };
+            await storage.storeFile(file);
+            const retrieved = await storage.getFile('test.ts');
+            expect(retrieved).toEqual(file);
+        });
+        it('should return null for non-existent file', async () => {
+            const retrieved = await storage.getFile('nonexistent.ts');
+            expect(retrieved).toBeNull();
+        });
+    });
+    describe('getAllFiles', () => {
+        it('should return all stored files', async () => {
+            const files = [
+                { path: 'file1.ts', content: 'a', size: 1, mtime: Date.now(), hash: 'h1' },
+                { path: 'file2.ts', content: 'b', size: 2, mtime: Date.now(), hash: 'h2' },
+                { path: 'file3.ts', content: 'c', size: 3, mtime: Date.now(), hash: 'h3' },
+            ];
+            for (const file of files) {
+                await storage.storeFile(file);
+            }
+            const allFiles = await storage.getAllFiles();
+            expect(allFiles).toHaveLength(3);
+            expect(allFiles).toEqual(expect.arrayContaining(files));
+        });
+        it('should return empty array when no files stored', async () => {
+            const allFiles = await storage.getAllFiles();
+            expect(allFiles).toEqual([]);
+        });
+    });
+    describe('deleteFile', () => {
+        it('should delete existing file', async () => {
+            const file = {
+                path: 'test.ts',
+                content: 'test',
+                size: 100,
+                mtime: Date.now(),
+                hash: 'abc',
+            };
+            await storage.storeFile(file);
+            await storage.deleteFile('test.ts');
+            const retrieved = await storage.getFile('test.ts');
+            expect(retrieved).toBeNull();
+        });
+        it('should not throw when deleting non-existent file', async () => {
+            await expect(async () => {
+                await storage.deleteFile('nonexistent.ts');
+            }).not.toThrow();
+        });
+    });
+    describe('clear', () => {
+        it('should remove all files', async () => {
+            const files = [
+                { path: 'file1.ts', content: 'a', size: 1, mtime: Date.now(), hash: 'h1' },
+                { path: 'file2.ts', content: 'b', size: 2, mtime: Date.now(), hash: 'h2' },
+            ];
+            for (const file of files) {
+                await storage.storeFile(file);
+            }
+            await storage.clear();
+            const allFiles = await storage.getAllFiles();
+            expect(allFiles).toHaveLength(0);
+        });
+    });
+    describe('count', () => {
+        it('should return correct file count', async () => {
+            expect(await storage.count()).toBe(0);
+            await storage.storeFile({
+                path: 'file1.ts',
+                content: 'a',
+                size: 1,
+                mtime: Date.now(),
+                hash: 'h1',
+            });
+            expect(await storage.count()).toBe(1);
+            await storage.storeFile({
+                path: 'file2.ts',
+                content: 'b',
+                size: 2,
+                mtime: Date.now(),
+                hash: 'h2',
+            });
+            expect(await storage.count()).toBe(2);
+            await storage.deleteFile('file1.ts');
+            expect(await storage.count()).toBe(1);
+        });
+    });
+    describe('exists', () => {
+        it('should return true for existing file', async () => {
+            await storage.storeFile({
+                path: 'test.ts',
+                content: 'test',
+                size: 100,
+                mtime: Date.now(),
+                hash: 'abc',
+            });
+            expect(await storage.exists('test.ts')).toBe(true);
+        });
+        it('should return false for non-existent file', async () => {
+            expect(await storage.exists('nonexistent.ts')).toBe(false);
+        });
+    });
+});
+//# sourceMappingURL=storage.test.js.map

package/dist/storage.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"storage.test.js","sourceRoot":"","sources":["../src/storage.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AACzD,OAAO,EAAqB,aAAa,EAAE,MAAM,cAAc,CAAA;AAE/D,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAsB,CAAA;IAE1B,UAAU,CAAC,GAAG,EAAE;QACf,OAAO,GAAG,IAAI,aAAa,EAAE,CAAA;IAC9B,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;QAC1B,EAAE,CAAC,qBAAqB,EAAE,KAAK,IAAI,EAAE;YACpC,MAAM,IAAI,GAAiB;gBAC1B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,sBAAsB;gBAC/B,IAAI,EAAE,GAAG;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE;gBACjB,QAAQ,EAAE,YAAY;gBACtB,IAAI,EAAE,QAAQ;aACd,CAAA;YAED,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YAC7B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;YAElD,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAChC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,KAAK,GAAiB;gBAC3B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,aAAa;gBACtB,IAAI,EAAE,GAAG;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE;gBACjB,IAAI,EAAE,OAAO;aACb,CAAA;YAED,MAAM,KAAK,GAAiB;gBAC3B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,aAAa;gBACtB,IAAI,EAAE,GAAG;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE;gBACjB,IAAI,EAAE,OAAO;aACb,CAAA;YAED,MAAM,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;YAC9B,MAAM,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;YAE9B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;YAClD,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;YAC9C,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,SAAS,EAAE,GAAG,EAAE;QACxB,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,IAAI,GAAiB;gBAC1B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,MAAM;gBACf,IAAI,EAAE,GAAG;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE;gBACjB,IAAI,EAAE,KAAK;aACX,CAAA;YAED,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YAC7B,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;YAElD,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAChC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAA;YACzD,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAA;QAC7B,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,KAAK,GAAmB;gBAC7B,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;gBAC1E,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;gBAC1E,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;aAC1E,CAAA;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YAC9B,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,CAAA;YAC5C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;YAChC,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,CAAA;QACxD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,CAAA;YAC5C,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAC7B,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,IAAI,GAAiB;gBAC1B,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,MAAM;gBACf,IAAI,EAAE,GAAG;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE;gBACjB,IAAI,EAAE,KAAK;aACX,CAAA;YAED,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YAC7B,MAAM,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,CAAA;YAEnC,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAA;YAClD,MAAM,CAAC,SAAS,CAAC,CAAC,QAAQ,EAAE,CAAA;QAC7B,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,MAAM,CAAC,KAAK,IAAI,EAAE;gBACvB,MAAM,OAAO,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAA;YAC3C,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAA;QACjB,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YACxC,MAAM,KAAK,GAAmB;gBAC7B,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;gBAC1E,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE;aAC1E,CAAA;YAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBAC1B,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YAC9B,CAAC;YAED,MAAM,OAAO,CAAC,KAAK,EAAE,CAAA;YAErB,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,WAAW,EAAE,CAAA;YAC5C,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACjC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;QACtB,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YACjD,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAErC,MAAM,OAAO,CAAC,SAAS,CAAC;gBACvB,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,GAAG;gBACZ,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE;gBACjB,IAAI,EAAE,IAAI;aACV,CAAC,CAAA;YACF,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAErC,MAAM,OAAO,CAAC,SAAS,CAAC;gBACvB,IAAI,EAAE,UAAU;gBAChB,OAAO,EAAE,GAAG;gBACZ,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE;gBACjB,IAAI,EAAE,IAAI;aACV,CAAC,CAAA;YACF,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAErC,MAAM,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,CAAA;YACpC,MAAM,CAAC,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,QAAQ,EAAE,GAAG,EAAE;QACvB,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,OAAO,CAAC,SAAS,CAAC;gBACvB,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,MAAM;gBACf,IAAI,EAAE,GAAG;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE;gBACjB,IAAI,EAAE,KAAK;aACX,CAAC,CAAA;YAEF,MAAM,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;YAC1D,MAAM,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC3D,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/tfidf.d.ts

@@ -0,0 +1,97 @@
+/**
+ * BM25 (Best Matching 25) implementation
+ * Using StarCoder2 tokenizer for code-aware tokenization
+ *
+ * BM25 improves on TF-IDF with:
+ * 1. Term frequency saturation (k1 parameter) - diminishing returns for repeated terms
+ * 2. Document length normalization (b parameter) - adjusts for document length
+ */
+import { initializeTokenizer } from './code-tokenizer.js';
+export { initializeTokenizer };
+export interface DocumentVector {
+    uri: string;
+    terms: Map<string, number>;
+    rawTerms: Map<string, number>;
+    magnitude: number;
+}
+export interface SearchIndex {
+    documents: DocumentVector[];
+    idf: Map<string, number>;
+    totalDocuments: number;
+    metadata: {
+        generatedAt: string;
+        version: string;
+    };
+}
+/**
+ * Tokenize code using StarCoder2 (async)
+ */
+export declare function tokenize(text: string): Promise<string[]>;
+/**
+ * Build TF-IDF search index from documents (async - uses StarCoder2)
+ */
+export declare function buildSearchIndex(documents: Array<{
+    uri: string;
+    content: string;
+}>): Promise<SearchIndex>;
+/**
+ * Calculate cosine similarity between query and document
+ */
+export declare function calculateCosineSimilarity(queryVector: Map<string, number>, docVector: DocumentVector): number;
+/**
+ * Process query into TF-IDF vector (async - uses StarCoder2)
+ */
+export declare function processQuery(query: string, idf: Map<string, number>): Promise<Map<string, number>>;
+/**
+ * SQL-based search result from storage
+ * Uses pre-computed magnitude and token count for BM25 scoring
+ */
+export interface StorageSearchResult {
+    path: string;
+    matchedTerms: Map<string, {
+        tfidf: number;
+        rawFreq: number;
+    }>;
+    magnitude: number;
+    tokenCount: number;
+}
+/**
+ * Search documents using BM25 scoring (SQL-based storage)
+ *
+ * BM25 formula: score(D,Q) = Σ IDF(qi) * (f(qi,D) * (k1+1)) / (f(qi,D) + k1 * (1 - b + b * |D|/avgdl))
+ *
+ * Where:
+ * - f(qi,D) = raw frequency of term qi in document D
+ * - |D| = document length (token count)
+ * - avgdl = average document length
+ * - k1 = term frequency saturation (default: 1.2)
+ * - b = length normalization (default: 0.75)
+ */
+export declare function searchDocumentsFromStorage(query: string, candidates: StorageSearchResult[], idf: Map<string, number>, options?: {
+    limit?: number;
+    minScore?: number;
+    avgDocLength?: number;
+}): Promise<Array<{
+    uri: string;
+    score: number;
+    matchedTerms: string[];
+}>>;
+/**
+ * Get query tokens (exported for SQL-based search) - async
+ */
+export declare function getQueryTokens(query: string): Promise<string[]>;
+/**
+ * Search documents using BM25 scoring (in-memory index)
+ *
+ * For in-memory search, document length is calculated from rawTerms.
+ * Average document length is calculated from all documents in the index.
+ */
+export declare function searchDocuments(query: string, index: SearchIndex, options?: {
+    limit?: number;
+    minScore?: number;
+}): Promise<Array<{
+    uri: string;
+    score: number;
+    matchedTerms: string[];
+}>>;
+//# sourceMappingURL=tfidf.d.ts.map

package/dist/tfidf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.d.ts","sourceRoot":"","sources":["../src/tfidf.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,mBAAmB,EAAiC,MAAM,qBAAqB,CAAA;AAGxF,OAAO,EAAE,mBAAmB,EAAE,CAAA;AA2B9B,MAAM,WAAW,cAAc;IAC9B,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC1B,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC7B,SAAS,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,WAAW;IAC3B,SAAS,EAAE,cAAc,EAAE,CAAA;IAC3B,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACxB,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE;QACT,WAAW,EAAE,MAAM,CAAA;QACnB,OAAO,EAAE,MAAM,CAAA;KACf,CAAA;CACD;AAED;;GAEG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAE9D;AAmFD;;GAEG;AACH,wBAAsB,gBAAgB,CACrC,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,GAChD,OAAO,CAAC,WAAW,CAAC,CAsCtB;AAED;;GAEG;AACH,wBAAgB,yBAAyB,CACxC,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EAChC,SAAS,EAAE,cAAc,GACvB,MAAM,CAiBR;AAED;;GAEG;AACH,wBAAsB,YAAY,CACjC,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GACtB,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAG9B;AAkBD;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IACnC,IAAI,EAAE,MAAM,CAAA;IACZ,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAC7D,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,0BAA0B,CAC/C,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,mBAAmB,EAAE,EACjC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,EACxB,OAAO,GAAE;IACR,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;CAChB,GACJ,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAAC,CAiExE;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAErE;AAED;;;;;GAKG;AACH,wBAAsB,eAAe,CACpC,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE;IACR,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,MAAM,CAAA;CACZ,GACJ,OAAO,CAAC,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,YAAY,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAAC,CAkExE"}

package/dist/tfidf.js

@@ -0,0 +1,308 @@
+/**
+ * BM25 (Best Matching 25) implementation
+ * Using StarCoder2 tokenizer for code-aware tokenization
+ *
+ * BM25 improves on TF-IDF with:
+ * 1. Term frequency saturation (k1 parameter) - diminishing returns for repeated terms
+ * 2. Document length normalization (b parameter) - adjusts for document length
+ */
+import { initializeTokenizer, tokenize as starcoderTokenize } from './code-tokenizer.js';
+// Re-export tokenize for external use
+export { initializeTokenizer };
+// BM25 parameters (Elasticsearch/Lucene defaults)
+const BM25_K1 = 1.2; // Term frequency saturation (1.2-2.0 typical)
+const BM25_B = 0.75; // Length normalization (0.75 typical, 0 = no normalization, 1 = full normalization)
+// Query token cache - avoids re-tokenizing the same query (CPU optimization)
+const queryTokenCache = new Map();
+const QUERY_CACHE_MAX_SIZE = 100;
+async function getCachedQueryTokens(query) {
+    const cached = queryTokenCache.get(query);
+    if (cached)
+        return cached;
+    // Tokenize and dedupe
+    const tokens = [...new Set(await tokenize(query))];
+    // LRU-style eviction: remove oldest if full
+    if (queryTokenCache.size >= QUERY_CACHE_MAX_SIZE) {
+        const firstKey = queryTokenCache.keys().next().value;
+        if (firstKey)
+            queryTokenCache.delete(firstKey);
+    }
+    queryTokenCache.set(query, tokens);
+    return tokens;
+}
+/**
+ * Tokenize code using StarCoder2 (async)
+ */
+export async function tokenize(text) {
+    return starcoderTokenize(text);
+}
+/**
+ * Calculate Term Frequency (TF)
+ */
+function calculateTF(termFrequency) {
+    const totalTerms = Array.from(termFrequency.values()).reduce((sum, freq) => sum + freq, 0);
+    const tf = new Map();
+    for (const [term, freq] of termFrequency.entries()) {
+        tf.set(term, freq / totalTerms);
+    }
+    return tf;
+}
+/**
+ * Calculate Inverse Document Frequency (IDF)
+ */
+function calculateIDF(documents, totalDocuments) {
+    const documentFrequency = new Map();
+    // Count how many documents contain each term
+    for (const doc of documents) {
+        const uniqueTerms = new Set(doc.keys());
+        for (const term of uniqueTerms) {
+            documentFrequency.set(term, (documentFrequency.get(term) || 0) + 1);
+        }
+    }
+    // Calculate IDF for each term using smoothed formula
+    // Standard formula: log(N/df) = 0 when term appears in ALL documents
+    // Smoothed formula: log((N+1)/(df+1)) + 1 ensures no term gets IDF=0
+    const idf = new Map();
+    for (const [term, docFreq] of documentFrequency.entries()) {
+        idf.set(term, Math.log((totalDocuments + 1) / (docFreq + 1)) + 1);
+    }
+    return idf;
+}
+/**
+ * Calculate TF-IDF scores for a document
+ */
+function calculateTFIDF(tf, idf) {
+    const tfidf = new Map();
+    for (const [term, tfScore] of tf.entries()) {
+        const idfScore = idf.get(term) || 0;
+        tfidf.set(term, tfScore * idfScore);
+    }
+    return tfidf;
+}
+/**
+ * Calculate vector magnitude for cosine similarity
+ */
+function calculateMagnitude(vector) {
+    let sum = 0;
+    for (const value of vector.values()) {
+        sum += value * value;
+    }
+    return Math.sqrt(sum);
+}
+/**
+ * Extract term frequencies from content (async - uses StarCoder2)
+ */
+async function extractTermFrequencies(content) {
+    const tokens = await tokenize(content);
+    const frequencies = new Map();
+    for (const token of tokens) {
+        frequencies.set(token, (frequencies.get(token) || 0) + 1);
+    }
+    return frequencies;
+}
+/**
+ * Build TF-IDF search index from documents (async - uses StarCoder2)
+ */
+export async function buildSearchIndex(documents) {
+    // Extract term frequencies for all documents
+    const documentTerms = await Promise.all(documents.map(async (doc) => ({
+        uri: doc.uri,
+        terms: await extractTermFrequencies(doc.content),
+    })));
+    // Calculate IDF scores
+    const idf = calculateIDF(documentTerms.map((d) => d.terms), documents.length);
+    // Calculate TF-IDF for each document
+    const documentVectors = documentTerms.map((doc) => {
+        const tf = calculateTF(doc.terms);
+        const tfidf = calculateTFIDF(tf, idf);
+        const magnitude = calculateMagnitude(tfidf);
+        return {
+            uri: doc.uri,
+            terms: tfidf,
+            rawTerms: doc.terms,
+            magnitude,
+        };
+    });
+    return {
+        documents: documentVectors,
+        idf,
+        totalDocuments: documents.length,
+        metadata: {
+            generatedAt: new Date().toISOString(),
+            version: '1.0.0',
+        },
+    };
+}
+/**
+ * Calculate cosine similarity between query and document
+ */
+export function calculateCosineSimilarity(queryVector, docVector) {
+    let dotProduct = 0;
+    // Calculate dot product
+    for (const [term, queryScore] of queryVector.entries()) {
+        const docScore = docVector.terms.get(term) || 0;
+        dotProduct += queryScore * docScore;
+    }
+    // Calculate query magnitude
+    const queryMagnitude = calculateMagnitude(queryVector);
+    if (queryMagnitude === 0 || docVector.magnitude === 0) {
+        return 0;
+    }
+    return dotProduct / (queryMagnitude * docVector.magnitude);
+}
+/**
+ * Process query into TF-IDF vector (async - uses StarCoder2)
+ */
+export async function processQuery(query, idf) {
+    const terms = await tokenize(query);
+    return processQueryWithTokens(terms, idf);
+}
+/**
+ * Process query from pre-tokenized terms (CPU optimization - avoids re-tokenizing)
+ */
+function processQueryWithTokens(tokens, idf) {
+    const queryVector = new Map();
+    for (const term of tokens) {
+        const idfValue = idf.get(term) || 0;
+        if (idfValue > 0) {
+            queryVector.set(term, idfValue);
+        }
+    }
+    return queryVector;
+}
+/**
+ * Search documents using BM25 scoring (SQL-based storage)
+ *
+ * BM25 formula: score(D,Q) = Σ IDF(qi) * (f(qi,D) * (k1+1)) / (f(qi,D) + k1 * (1 - b + b * |D|/avgdl))
+ *
+ * Where:
+ * - f(qi,D) = raw frequency of term qi in document D
+ * - |D| = document length (token count)
+ * - avgdl = average document length
+ * - k1 = term frequency saturation (default: 1.2)
+ * - b = length normalization (default: 0.75)
+ */
+export async function searchDocumentsFromStorage(query, candidates, idf, options = {}) {
+    const { limit = 10, minScore = 0 } = options;
+    // Get query tokens (cached)
+    const queryTokens = await getCachedQueryTokens(query);
+    if (queryTokens.length === 0) {
+        return [];
+    }
+    // Calculate average document length if not provided
+    // Fallback to average of candidates (less accurate but works without global stats)
+    let avgDocLength = options.avgDocLength;
+    if (!avgDocLength || avgDocLength === 0) {
+        const totalTokens = candidates.reduce((sum, c) => sum + (c.tokenCount || 0), 0);
+        avgDocLength = candidates.length > 0 ? totalTokens / candidates.length : 1;
+    }
+    // Ensure avgDocLength is at least 1 to avoid division by zero
+    avgDocLength = Math.max(avgDocLength, 1);
+    // Score each candidate using BM25
+    const results = [];
+    let minThreshold = minScore;
+    for (const candidate of candidates) {
+        // Get matched terms
+        const matchedTerms = [];
+        for (const term of queryTokens) {
+            if (candidate.matchedTerms.has(term)) {
+                matchedTerms.push(term);
+            }
+        }
+        if (matchedTerms.length === 0)
+            continue;
+        // BM25 scoring
+        const docLen = candidate.tokenCount || 1;
+        let score = 0;
+        for (const term of matchedTerms) {
+            const docData = candidate.matchedTerms.get(term);
+            if (!docData)
+                continue;
+            const termFreq = docData.rawFreq;
+            const termIdf = idf.get(term) || 0;
+            // BM25 term score: IDF * (tf * (k1+1)) / (tf + k1 * (1 - b + b * docLen/avgdl))
+            const numerator = termFreq * (BM25_K1 + 1);
+            const denominator = termFreq + BM25_K1 * (1 - BM25_B + (BM25_B * docLen) / avgDocLength);
+            score += termIdf * (numerator / denominator);
+        }
+        if (score < minThreshold)
+            continue;
+        results.push({ uri: `file://${candidate.path}`, score, matchedTerms });
+        // Bounded results (optimization for large candidate sets)
+        if (results.length >= limit * 2) {
+            results.sort((a, b) => b.score - a.score);
+            results.length = limit;
+            minThreshold = results[results.length - 1].score;
+        }
+    }
+    return results.sort((a, b) => b.score - a.score).slice(0, limit);
+}
+/**
+ * Get query tokens (exported for SQL-based search) - async
+ */
+export async function getQueryTokens(query) {
+    return getCachedQueryTokens(query);
+}
+/**
+ * Search documents using BM25 scoring (in-memory index)
+ *
+ * For in-memory search, document length is calculated from rawTerms.
+ * Average document length is calculated from all documents in the index.
+ */
+export async function searchDocuments(query, index, options = {}) {
+    const { limit = 10, minScore = 0 } = options;
+    // Process query with cached tokens (CPU optimization)
+    const queryTokens = await getCachedQueryTokens(query);
+    if (queryTokens.length === 0) {
+        return [];
+    }
+    // Calculate average document length from index
+    let totalTokens = 0;
+    for (const doc of index.documents) {
+        for (const freq of doc.rawTerms.values()) {
+            totalTokens += freq;
+        }
+    }
+    const avgDocLength = index.documents.length > 0 ? totalTokens / index.documents.length : 1;
+    // Score documents using BM25
+    const results = [];
+    let minThreshold = minScore;
+    for (const doc of index.documents) {
+        // Get matched terms
+        const matchedTerms = [];
+        for (const token of queryTokens) {
+            if (doc.rawTerms.has(token)) {
+                matchedTerms.push(token);
+            }
+        }
+        if (matchedTerms.length === 0)
+            continue;
+        // Calculate document length (sum of all term frequencies)
+        let docLen = 0;
+        for (const freq of doc.rawTerms.values()) {
+            docLen += freq;
+        }
+        docLen = Math.max(docLen, 1); // Avoid division by zero
+        // BM25 scoring
+        let score = 0;
+        for (const term of matchedTerms) {
+            const termFreq = doc.rawTerms.get(term) || 0;
+            const termIdf = index.idf.get(term) || 0;
+            // BM25 term score
+            const numerator = termFreq * (BM25_K1 + 1);
+            const denominator = termFreq + BM25_K1 * (1 - BM25_B + (BM25_B * docLen) / avgDocLength);
+            score += termIdf * (numerator / denominator);
+        }
+        if (score < minThreshold)
+            continue;
+        results.push({ uri: doc.uri, score, matchedTerms });
+        // Bounded results (optimization)
+        if (results.length >= limit * 2) {
+            results.sort((a, b) => b.score - a.score);
+            results.length = limit;
+            minThreshold = results[results.length - 1].score;
+        }
+    }
+    return results.sort((a, b) => b.score - a.score).slice(0, limit);
+}
+//# sourceMappingURL=tfidf.js.map

package/dist/tfidf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.js","sourceRoot":"","sources":["../src/tfidf.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,mBAAmB,EAAE,QAAQ,IAAI,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAExF,sCAAsC;AACtC,OAAO,EAAE,mBAAmB,EAAE,CAAA;AAE9B,kDAAkD;AAClD,MAAM,OAAO,GAAG,GAAG,CAAA,CAAC,8CAA8C;AAClE,MAAM,MAAM,GAAG,IAAI,CAAA,CAAC,oFAAoF;AAExG,6EAA6E;AAC7E,MAAM,eAAe,GAAG,IAAI,GAAG,EAAoB,CAAA;AACnD,MAAM,oBAAoB,GAAG,GAAG,CAAA;AAEhC,KAAK,UAAU,oBAAoB,CAAC,KAAa;IAChD,MAAM,MAAM,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;IACzC,IAAI,MAAM;QAAE,OAAO,MAAM,CAAA;IAEzB,sBAAsB;IACtB,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAElD,4CAA4C;IAC5C,IAAI,eAAe,CAAC,IAAI,IAAI,oBAAoB,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,CAAA;QACpD,IAAI,QAAQ;YAAE,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAC/C,CAAC;IAED,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,MAAM,CAAA;AACd,CAAC;AAmBD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IAC1C,OAAO,iBAAiB,CAAC,IAAI,CAAC,CAAA;AAC/B,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,aAAkC;IACtD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAA;IAC1F,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEpC,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;QACpD,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,UAAU,CAAC,CAAA;IAChC,CAAC;IAED,OAAO,EAAE,CAAA;AACV,CAAC;AAED;;GAEG;AACH,SAAS,YAAY,CACpB,SAAgC,EAChC,cAAsB;IAEtB,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEnD,6CAA6C;IAC7C,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC7B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;QACvC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACpE,CAAC;IACF,CAAC;IAED,qDAAqD;IACrD,qEAAqE;IACrE,qEAAqE;IACrE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAA;IACrC,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,iBAAiB,CAAC,OAAO,EAAE,EAAE,CAAC;QAC3D,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAClE,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,SAAS,cAAc,CAAC,EAAuB,EAAE,GAAwB;IACxE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEvC,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,GAAG,QAAQ,CAAC,CAAA;IACpC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,MAA2B;IACtD,IAAI,GAAG,GAAG,CAAC,CAAA;IACX,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QACrC,GAAG,IAAI,KAAK,GAAG,KAAK,CAAA;IACrB,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;AACtB,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,sBAAsB,CAAC,OAAe;IACpD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAA;IACtC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1D,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,SAAkD;IAElD,6CAA6C;IAC7C,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACtC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7B,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,KAAK,EAAE,MAAM,sBAAsB,CAAC,GAAG,CAAC,OAAO,CAAC;KAChD,CAAC,CAAC,CACH,CAAA;IAED,uBAAuB;IACvB,MAAM,GAAG,GAAG,YAAY,CACvB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,EACjC,SAAS,CAAC,MAAM,CAChB,CAAA;IAED,qCAAqC;IACrC,MAAM,eAAe,GAAqB,aAAa,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACnE,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACjC,MAAM,KAAK,GAAG,cAAc,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;QACrC,MAAM,SAAS,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAA;QAE3C,OAAO;YACN,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,GAAG,CAAC,KAAK;YACnB,SAAS;SACT,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,OAAO;QACN,SAAS,EAAE,eAAe;QAC1B,GAAG;QACH,cAAc,EAAE,SAAS,CAAC,MAAM;QAChC,QAAQ,EAAE;YACT,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACrC,OAAO,EAAE,OAAO;SAChB;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,yBAAyB,CACxC,WAAgC,EAChC,SAAyB;IAEzB,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,wBAAwB;IACxB,KAAK,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;QACxD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/C,UAAU,IAAI,UAAU,GAAG,QAAQ,CAAA;IACpC,CAAC;IAED,4BAA4B;IAC5B,MAAM,cAAc,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAA;IAEtD,IAAI,cAAc,KAAK,CAAC,IAAI,SAAS,CAAC,SAAS,KAAK,CAAC,EAAE,CAAC;QACvD,OAAO,CAAC,CAAA;IACT,CAAC;IAED,OAAO,UAAU,GAAG,CAAC,cAAc,GAAG,SAAS,CAAC,SAAS,CAAC,CAAA;AAC3D,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,KAAa,EACb,GAAwB;IAExB,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;IACnC,OAAO,sBAAsB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;AAC1C,CAAC;AAED;;GAEG;AACH,SAAS,sBAAsB,CAAC,MAAgB,EAAE,GAAwB;IACzE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;IAE7C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YAClB,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;QAChC,CAAC;IACF,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAaD;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC/C,KAAa,EACb,UAAiC,EACjC,GAAwB,EACxB,UAII,EAAE;IAEN,MAAM,EAAE,KAAK,GAAG,EAAE,EAAE,QAAQ,GAAG,CAAC,EAAE,GAAG,OAAO,CAAA;IAE5C,4BAA4B;IAC5B,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,KAAK,CAAC,CAAA;IAErD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAA;IACV,CAAC;IAED,oDAAoD;IACpD,mFAAmF;IACnF,IAAI,YAAY,GAAG,OAAO,CAAC,YAAY,CAAA;IACvC,IAAI,CAAC,YAAY,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QAC/E,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAC3E,CAAC;IACD,8DAA8D;IAC9D,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAA;IAExC,kCAAkC;IAClC,MAAM,OAAO,GAAkE,EAAE,CAAA;IACjF,IAAI,YAAY,GAAG,QAAQ,CAAA;IAE3B,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACpC,oBAAoB;QACpB,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAChC,IAAI,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACxB,CAAC;QACF,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAEvC,eAAe;QACf,MAAM,MAAM,GAAG,SAAS,CAAC,UAAU,IAAI,CAAC,CAAA;QACxC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YAChD,IAAI,CAAC,OAAO;gBAAE,SAAQ;YAEtB,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAA;YAChC,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAElC,gFAAgF;YAChF,MAAM,SAAS,GAAG,QAAQ,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAA;YAC1C,MAAM,WAAW,GAAG,QAAQ,GAAG,OAAO,GAAG,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,YAAY,CAAC,CAAA;YACxF,KAAK,IAAI,OAAO,GAAG,CAAC,SAAS,GAAG,WAAW,CAAC,CAAA;QAC7C,CAAC;QAED,IAAI,KAAK,GAAG,YAAY;YAAE,SAAQ;QAElC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,SAAS,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAA;QAEtE,0DAA0D;QAC1D,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;YACzC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAA;YACtB,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACjD,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;AACjE,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,KAAa;IACjD,OAAO,oBAAoB,CAAC,KAAK,CAAC,CAAA;AACnC,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,KAAa,EACb,KAAkB,EAClB,UAGI,EAAE;IAEN,MAAM,EAAE,KAAK,GAAG,EAAE,EAAE,QAAQ,GAAG,CAAC,EAAE,GAAG,OAAO,CAAA;IAE5C,sDAAsD;IACtD,MAAM,WAAW,GAAG,MAAM,oBAAoB,CAAC,KAAK,CAAC,CAAA;IAErD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,CAAA;IACV,CAAC;IAED,+CAA+C;IAC/C,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACnC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,WAAW,IAAI,IAAI,CAAA;QACpB,CAAC;IACF,CAAC;IACD,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAE1F,6BAA6B;IAC7B,MAAM,OAAO,GAAkE,EAAE,CAAA;IACjF,IAAI,YAAY,GAAG,QAAQ,CAAA;IAE3B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACnC,oBAAoB;QACpB,MAAM,YAAY,GAAa,EAAE,CAAA;QACjC,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC7B,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YACzB,CAAC;QACF,CAAC;QAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAEvC,0DAA0D;QAC1D,IAAI,MAAM,GAAG,CAAC,CAAA;QACd,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,MAAM,IAAI,IAAI,CAAA;QACf,CAAC;QACD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA,CAAC,yBAAyB;QAEtD,eAAe;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC5C,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAExC,kBAAkB;YAClB,MAAM,SAAS,GAAG,QAAQ,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAA;YAC1C,MAAM,WAAW,GAAG,QAAQ,GAAG,OAAO,GAAG,CAAC,CAAC,GAAG,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,YAAY,CAAC,CAAA;YACxF,KAAK,IAAI,OAAO,GAAG,CAAC,SAAS,GAAG,WAAW,CAAC,CAAA;QAC7C,CAAC;QAED,IAAI,KAAK,GAAG,YAAY;YAAE,SAAQ;QAElC,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAA;QAEnD,iCAAiC;QACjC,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;YACzC,OAAO,CAAC,MAAM,GAAG,KAAK,CAAA;YACtB,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK,CAAA;QACjD,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;AACjE,CAAC"}

package/dist/tfidf.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for TF-IDF search functionality (StarCoder2 tokenizer)
+ */
+export {};
+//# sourceMappingURL=tfidf.test.d.ts.map

package/dist/tfidf.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tfidf.test.d.ts","sourceRoot":"","sources":["../src/tfidf.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}