@matperez/coderag 0.1.24

package/dist/hybrid-search.test.js

@@ -0,0 +1,252 @@
+/**
+ * Hybrid Search Tests
+ */
+import fs from 'node:fs';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { createMockProvider } from './embeddings.js';
+import { hybridSearch, keywordSearch, semanticSearch } from './hybrid-search.js';
+import { CodebaseIndexer } from './indexer.js';
+import { MemoryStorage } from './storage.js';
+describe('Hybrid Search', () => {
+    let tempDir;
+    let indexer;
+    let embeddingProvider;
+    beforeEach(async () => {
+        // Create temporary test directory
+        tempDir = `/tmp/hybrid-search-test-${Date.now()}`;
+        fs.mkdirSync(tempDir, { recursive: true });
+        // Create test files
+        const testFiles = [
+            {
+                name: 'auth.ts',
+                content: `
+export function authenticate(username: string, password: string) {
+  // User authentication logic
+  return validateCredentials(username, password);
+}
+
+export function validateCredentials(user: string, pass: string): boolean {
+  // Check user credentials
+  return true;
+}
+        `,
+            },
+            {
+                name: 'database.ts',
+                content: `
+export class DatabaseConnection {
+  connect() {
+    // Database connection logic
+  }
+
+  query(sql: string) {
+    // Execute SQL query
+  }
+}
+        `,
+            },
+            {
+                name: 'api.ts',
+                content: `
+export function handleRequest(req: Request) {
+  // API request handler
+  return processRequest(req);
+}
+
+export function processRequest(request: Request) {
+  // Process API request
+  return { success: true };
+}
+        `,
+            },
+        ];
+        for (const file of testFiles) {
+            const filePath = path.join(tempDir, file.name);
+            fs.writeFileSync(filePath, file.content);
+        }
+        // Initialize embedding provider
+        embeddingProvider = createMockProvider(128);
+        // Initialize indexer with embeddings
+        indexer = new CodebaseIndexer({
+            codebaseRoot: tempDir,
+            storage: new MemoryStorage(),
+            embeddingProvider,
+        });
+        // Index the codebase
+        await indexer.index();
+    });
+    afterEach(() => {
+        // Clean up temp directory
+        if (fs.existsSync(tempDir)) {
+            fs.rmSync(tempDir, { recursive: true, force: true });
+        }
+    });
+    describe('hybridSearch', () => {
+        it('should combine vector and TF-IDF results', async () => {
+            const results = await hybridSearch('authentication', indexer, {
+                limit: 5,
+                vectorWeight: 0.7,
+            });
+            expect(results.length).toBeGreaterThan(0);
+            expect(results.length).toBeLessThanOrEqual(5);
+            // Should have at least one result with hybrid method
+            const hasHybrid = results.some((r) => r.method === 'hybrid');
+            const hasVector = results.some((r) => r.method === 'vector');
+            const hasTfidf = results.some((r) => r.method === 'tfidf');
+            // At least one of these should be true
+            expect(hasHybrid || hasVector || hasTfidf).toBe(true);
+            // All results should have valid scores
+            expect(results.every((r) => r.score >= 0)).toBe(true);
+            // Results should be sorted by score
+            for (let i = 1; i < results.length; i++) {
+                expect(results[i - 1].score).toBeGreaterThanOrEqual(results[i].score);
+            }
+        });
+        it('should respect limit parameter', async () => {
+            const results = await hybridSearch('function', indexer, { limit: 2 });
+            expect(results.length).toBeLessThanOrEqual(2);
+        });
+        it('should filter by minimum score', async () => {
+            const results = await hybridSearch('authentication', indexer, {
+                minScore: 0.5,
+            });
+            expect(results.every((r) => r.score >= 0.5)).toBe(true);
+        });
+        it('should include content when requested', async () => {
+            const results = await hybridSearch('database', indexer, {
+                includeContent: true,
+                limit: 1,
+            });
+            if (results.length > 0) {
+                expect(results[0].content).toBeDefined();
+                expect(typeof results[0].content).toBe('string');
+            }
+        });
+        it('should adjust scores based on vector weight', async () => {
+            const highVectorWeight = await hybridSearch('authentication', indexer, {
+                vectorWeight: 0.9,
+                limit: 5,
+            });
+            const lowVectorWeight = await hybridSearch('authentication', indexer, {
+                vectorWeight: 0.1,
+                limit: 5,
+            });
+            // Both should return results
+            expect(highVectorWeight.length).toBeGreaterThan(0);
+            expect(lowVectorWeight.length).toBeGreaterThan(0);
+            // Scores might differ based on weighting
+            // (This is a soft check as exact scores depend on the mock provider)
+        });
+        it('should fallback to TF-IDF when vector search unavailable', async () => {
+            // Create indexer without embeddings
+            const simpleIndexer = new CodebaseIndexer({
+                codebaseRoot: tempDir,
+                storage: new MemoryStorage(),
+            });
+            await simpleIndexer.index();
+            const results = await hybridSearch('authentication', simpleIndexer);
+            expect(results.length).toBeGreaterThan(0);
+            expect(results.every((r) => r.method === 'tfidf')).toBe(true);
+        });
+        it('should handle empty query', async () => {
+            const results = await hybridSearch('', indexer);
+            // Empty query should still return results (all docs with low scores)
+            expect(Array.isArray(results)).toBe(true);
+        });
+        it('should handle query with no matches', async () => {
+            const results = await hybridSearch('zzznonexistenttermzzz', indexer);
+            // Should return empty or very low-scored results
+            expect(Array.isArray(results)).toBe(true);
+        });
+    });
+    describe('semanticSearch', () => {
+        // FIXME: Skipping due to HNSW initialization timing issues with small datasets
+        it.skip('should use vector search only (weight = 1.0)', async () => {
+            const results = await semanticSearch('authentication', indexer, {
+                limit: 5,
+                minScore: 0, // Allow all results for small test dataset
+            });
+            expect(results.length).toBeGreaterThan(0);
+            // All results should be from vector search
+            expect(results.every((r) => r.method === 'vector' || r.method === 'hybrid')).toBe(true);
+        });
+        it('should return relevant semantic matches', async () => {
+            const results = await semanticSearch('user login credentials', indexer);
+            // Should find auth.ts which has related content
+            expect(results.length).toBeGreaterThan(0);
+            // At least one result should have reasonable similarity
+            expect(results.some((r) => r.score > 0)).toBe(true);
+        });
+    });
+    describe('keywordSearch', () => {
+        it('should use TF-IDF search only (weight = 0.0)', async () => {
+            const results = await keywordSearch('authenticate', indexer, { limit: 5 });
+            expect(results.length).toBeGreaterThan(0);
+            // All results should be from TF-IDF search
+            expect(results.every((r) => r.method === 'tfidf')).toBe(true);
+        });
+        it('should return keyword matches', async () => {
+            const results = await keywordSearch('authenticate', indexer);
+            expect(results.length).toBeGreaterThan(0);
+            // Should have matched terms
+            expect(results.some((r) => r.matchedTerms && r.matchedTerms.length > 0)).toBe(true);
+        });
+        it('should match exact keywords', async () => {
+            const results = await keywordSearch('DatabaseConnection', indexer);
+            expect(results.length).toBeGreaterThan(0);
+            // Should find database.ts
+            expect(results.some((r) => r.path.includes('database.ts'))).toBe(true);
+        });
+    });
+    describe('comparison', () => {
+        // FIXME: Skipping due to HNSW initialization timing issues with small datasets
+        it.skip('hybrid vs semantic vs keyword should return different results', async () => {
+            const query = 'authentication';
+            const hybridResults = await hybridSearch(query, indexer, {
+                vectorWeight: 0.7,
+                limit: 5,
+                minScore: 0,
+            });
+            const semanticResults = await semanticSearch(query, indexer, {
+                limit: 5,
+                minScore: 0, // Allow all results for small test dataset
+            });
+            const keywordResults = await keywordSearch(query, indexer, { limit: 5 });
+            // All should return some results
+            expect(hybridResults.length).toBeGreaterThan(0);
+            expect(semanticResults.length).toBeGreaterThan(0);
+            expect(keywordResults.length).toBeGreaterThan(0);
+            // Methods should be different
+            const _hybridMethods = new Set(hybridResults.map((r) => r.method));
+            const semanticMethods = new Set(semanticResults.map((r) => r.method));
+            const keywordMethods = new Set(keywordResults.map((r) => r.method));
+            expect(semanticMethods.has('vector') || semanticMethods.has('hybrid')).toBe(true);
+            expect(keywordMethods.has('tfidf')).toBe(true);
+        });
+    });
+    describe('result quality', () => {
+        it('should return results with all expected fields', async () => {
+            const results = await hybridSearch('function', indexer, {
+                includeContent: true,
+                limit: 1,
+            });
+            if (results.length > 0) {
+                const result = results[0];
+                expect(result.path).toBeDefined();
+                expect(typeof result.path).toBe('string');
+                expect(result.score).toBeDefined();
+                expect(typeof result.score).toBe('number');
+                expect(result.method).toBeDefined();
+                expect(['vector', 'tfidf', 'hybrid']).toContain(result.method);
+            }
+        });
+        it('should have normalized scores', async () => {
+            const results = await hybridSearch('authentication', indexer, { limit: 10 });
+            expect(results.length).toBeGreaterThan(0);
+            // Scores should be in reasonable range (0 to ~2 after normalization + weighting)
+            expect(results.every((r) => r.score >= 0 && r.score <= 10)).toBe(true);
+        });
+    });
+});
+//# sourceMappingURL=hybrid-search.test.js.map

package/dist/hybrid-search.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hybrid-search.test.js","sourceRoot":"","sources":["../src/hybrid-search.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,SAAS,CAAA;AACxB,OAAO,IAAI,MAAM,WAAW,CAAA;AAC5B,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AAChF,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAE5C,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC9B,IAAI,OAAe,CAAA;IACnB,IAAI,OAAwB,CAAA;IAC5B,IAAI,iBAAsB,CAAA;IAE1B,UAAU,CAAC,KAAK,IAAI,EAAE;QACrB,kCAAkC;QAClC,OAAO,GAAG,2BAA2B,IAAI,CAAC,GAAG,EAAE,EAAE,CAAA;QACjD,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAE1C,oBAAoB;QACpB,MAAM,SAAS,GAAG;YACjB;gBACC,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE;;;;;;;;;;SAUJ;aACL;YACD;gBACC,IAAI,EAAE,aAAa;gBACnB,OAAO,EAAE;;;;;;;;;;SAUJ;aACL;YACD;gBACC,IAAI,EAAE,QAAQ;gBACd,OAAO,EAAE;;;;;;;;;;SAUJ;aACL;SACD,CAAA;QAED,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YAC9C,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;QACzC,CAAC;QAED,gCAAgC;QAChC,iBAAiB,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAA;QAE3C,qCAAqC;QACrC,OAAO,GAAG,IAAI,eAAe,CAAC;YAC7B,YAAY,EAAE,OAAO;YACrB,OAAO,EAAE,IAAI,aAAa,EAAE;YAC5B,iBAAiB;SACjB,CAAC,CAAA;QAEF,qBAAqB;QACrB,MAAM,OAAO,CAAC,KAAK,EAAE,CAAA;IACtB,CAAC,CAAC,CAAA;IAEF,SAAS,CAAC,GAAG,EAAE;QACd,0BAA0B;QAC1B,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5B,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QACrD,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,gBAAgB,EAAE,OAAO,EAAE;gBAC7D,KAAK,EAAE,CAAC;gBACR,YAAY,EAAE,GAAG;aACjB,CAAC,CAAA;YAEF,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACzC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAA;YAE7C,qDAAqD;YACrD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;YAC5D,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;YAC5D,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAA;YAE1D,uCAAuC;YACvC,MAAM,CAAC,SAAS,IAAI,SAAS,IAAI,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAErD,uCAAuC;YACvC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAErD,oCAAoC;YACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACzC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,sBAAsB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;YACtE,CAAC;QACF,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;YAErE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAA;QAC9C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;YAC/C,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,gBAAgB,EAAE,OAAO,EAAE;gBAC7D,QAAQ,EAAE,GAAG;aACb,CAAC,CAAA;YAEF,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxD,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,UAAU,EAAE,OAAO,EAAE;gBACvD,cAAc,EAAE,IAAI;gBACpB,KAAK,EAAE,CAAC;aACR,CAAC,CAAA;YAEF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;gBACxC,MAAM,CAAC,OAAO,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;YACjD,CAAC;QACF,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,6CAA6C,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,gBAAgB,GAAG,MAAM,YAAY,CAAC,gBAAgB,EAAE,OAAO,EAAE;gBACtE,YAAY,EAAE,GAAG;gBACjB,KAAK,EAAE,CAAC;aACR,CAAC,CAAA;YAEF,MAAM,eAAe,GAAG,MAAM,YAAY,CAAC,gBAAgB,EAAE,OAAO,EAAE;gBACrE,YAAY,EAAE,GAAG;gBACjB,KAAK,EAAE,CAAC;aACR,CAAC,CAAA;YAEF,6BAA6B;YAC7B,MAAM,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAClD,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAEjD,yCAAyC;YACzC,qEAAqE;QACtE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACzE,oCAAoC;YACpC,MAAM,aAAa,GAAG,IAAI,eAAe,CAAC;gBACzC,YAAY,EAAE,OAAO;gBACrB,OAAO,EAAE,IAAI,aAAa,EAAE;aAC5B,CAAC,CAAA;YAEF,MAAM,aAAa,CAAC,KAAK,EAAE,CAAA;YAE3B,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,gBAAgB,EAAE,aAAa,CAAC,CAAA;YAEnE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACzC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;YAE/C,qEAAqE;YACrE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACpD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,uBAAuB,EAAE,OAAO,CAAC,CAAA;YAEpE,iDAAiD;YACjD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC1C,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC/B,+EAA+E;QAC/E,EAAE,CAAC,IAAI,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAClE,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,gBAAgB,EAAE,OAAO,EAAE;gBAC/D,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,EAAE,2CAA2C;aACxD,CAAC,CAAA;YAEF,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACzC,2CAA2C;YAC3C,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxF,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,wBAAwB,EAAE,OAAO,CAAC,CAAA;YAEvE,gDAAgD;YAChD,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACzC,wDAAwD;YACxD,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpD,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;QAC9B,EAAE,CAAC,8CAA8C,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;YAE1E,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACzC,2CAA2C;YAC3C,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,cAAc,EAAE,OAAO,CAAC,CAAA;YAE5D,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACzC,4BAA4B;YAC5B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,CAAC,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpF,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,oBAAoB,EAAE,OAAO,CAAC,CAAA;YAElE,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACzC,0BAA0B;YAC1B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACvE,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC3B,+EAA+E;QAC/E,EAAE,CAAC,IAAI,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YACnF,MAAM,KAAK,GAAG,gBAAgB,CAAA;YAE9B,MAAM,aAAa,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,OAAO,EAAE;gBACxD,YAAY,EAAE,GAAG;gBACjB,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC;aACX,CAAC,CAAA;YACF,MAAM,eAAe,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE;gBAC5D,KAAK,EAAE,CAAC;gBACR,QAAQ,EAAE,CAAC,EAAE,2CAA2C;aACxD,CAAC,CAAA;YACF,MAAM,cAAc,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAA;YAExE,iCAAiC;YACjC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAC/C,MAAM,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACjD,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAEhD,8BAA8B;YAC9B,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;YAClE,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;YACrE,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;YAEnE,MAAM,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACjF,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,UAAU,EAAE,OAAO,EAAE;gBACvD,cAAc,EAAE,IAAI;gBACpB,KAAK,EAAE,CAAC;aACR,CAAC,CAAA;YAEF,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;gBACzB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAA;gBACjC,MAAM,CAAC,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBACzC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;gBAClC,MAAM,CAAC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;gBAC1C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAA;gBACnC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAC/D,CAAC;QACF,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,+BAA+B,EAAE,KAAK,IAAI,EAAE;YAC9C,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,gBAAgB,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAA;YAE5E,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACzC,iFAAiF;YACjF,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACvE,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/incremental-tfidf.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Incremental TF-IDF Index Manager
+ * Efficiently updates TF-IDF vectors without full rebuild
+ */
+import type { DocumentVector } from './tfidf.js';
+export interface IncrementalUpdate {
+    type: 'add' | 'update' | 'delete';
+    uri: string;
+    oldDocument?: DocumentVector;
+    newContent?: string;
+}
+export interface IncrementalStats {
+    totalDocuments: number;
+    affectedDocuments: number;
+    affectedTerms: number;
+    updateTime: number;
+}
+/**
+ * Incremental TF-IDF Index Manager
+ *
+ * ALGORITHM:
+ * 1. Track document frequency (df) for each term
+ * 2. When document changes:
+ *    - Update df for affected terms
+ *    - Recalculate IDF for affected terms
+ *    - Update TF-IDF only for affected documents
+ * 3. Avoid full rebuild unless necessary
+ */
+export declare class IncrementalTFIDF {
+    private documents;
+    private idf;
+    private documentFrequency;
+    private totalDocuments;
+    constructor(initialDocuments?: DocumentVector[], initialIdf?: Map<string, number>);
+    /**
+     * Apply multiple updates efficiently
+     */
+    applyUpdates(updates: IncrementalUpdate[]): Promise<IncrementalStats>;
+    /**
+     * Add a new document
+     */
+    private addDocumentInternal;
+    /**
+     * Update an existing document
+     */
+    private updateDocumentInternal;
+    /**
+     * Delete a document
+     */
+    private deleteDocumentInternal;
+    /**
+     * Extract terms from content (async - uses StarCoder2)
+     */
+    private extractTerms;
+    /**
+     * Calculate TF for a document
+     */
+    private calculateTF;
+    /**
+     * Recalculate TF-IDF for a document
+     */
+    private recalculateDocumentTFIDF;
+    /**
+     * Get current index state
+     */
+    getIndex(): {
+        documents: DocumentVector[];
+        idf: Map<string, number>;
+        totalDocuments: number;
+    };
+    /**
+     * Check if full rebuild is recommended
+     * Returns true if changes are too extensive for incremental update
+     */
+    shouldFullRebuild(updates: IncrementalUpdate[]): Promise<boolean>;
+}
+//# sourceMappingURL=incremental-tfidf.d.ts.map

package/dist/incremental-tfidf.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental-tfidf.d.ts","sourceRoot":"","sources":["../src/incremental-tfidf.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAGhD,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,WAAW,CAAC,EAAE,cAAc,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,gBAAgB;IAChC,cAAc,EAAE,MAAM,CAAA;IACtB,iBAAiB,EAAE,MAAM,CAAA;IACzB,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,gBAAgB;IAC5B,OAAO,CAAC,SAAS,CAA6B;IAC9C,OAAO,CAAC,GAAG,CAAqB;IAChC,OAAO,CAAC,iBAAiB,CAAqB;IAC9C,OAAO,CAAC,cAAc,CAAQ;gBAG7B,gBAAgB,GAAE,cAAc,EAAO,EACvC,UAAU,GAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAa;IAmB5C;;OAEG;IACG,YAAY,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAwE3E;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAyB3B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAoC9B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAiB9B;;OAEG;YACW,YAAY;IAW1B;;OAEG;IACH,OAAO,CAAC,WAAW;IAWnB;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAkBhC;;OAEG;IACH,QAAQ,IAAI;QACX,SAAS,EAAE,cAAc,EAAE,CAAA;QAC3B,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACxB,cAAc,EAAE,MAAM,CAAA;KACtB;IAQD;;;OAGG;IACG,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC;CA8BvE"}

package/dist/incremental-tfidf.js

@@ -0,0 +1,248 @@
+/**
+ * Incremental TF-IDF Index Manager
+ * Efficiently updates TF-IDF vectors without full rebuild
+ */
+import { tokenize } from './tfidf.js'; // async tokenize using StarCoder2
+/**
+ * Incremental TF-IDF Index Manager
+ *
+ * ALGORITHM:
+ * 1. Track document frequency (df) for each term
+ * 2. When document changes:
+ *    - Update df for affected terms
+ *    - Recalculate IDF for affected terms
+ *    - Update TF-IDF only for affected documents
+ * 3. Avoid full rebuild unless necessary
+ */
+export class IncrementalTFIDF {
+    documents;
+    idf;
+    documentFrequency;
+    totalDocuments;
+    constructor(initialDocuments = [], initialIdf = new Map()) {
+        this.documents = new Map();
+        this.idf = new Map(initialIdf);
+        this.documentFrequency = new Map();
+        this.totalDocuments = initialDocuments.length;
+        // Initialize from existing index
+        for (const doc of initialDocuments) {
+            this.documents.set(doc.uri, doc);
+            // Calculate document frequency from existing documents
+            for (const term of doc.rawTerms.keys()) {
+                const df = this.documentFrequency.get(term) || 0;
+                this.documentFrequency.set(term, df + 1);
+            }
+        }
+    }
+    /**
+     * Apply multiple updates efficiently
+     */
+    async applyUpdates(updates) {
+        const startTime = Date.now();
+        const affectedTerms = new Set();
+        const affectedDocuments = new Set();
+        // Phase 1: Process all changes and collect affected terms
+        for (const update of updates) {
+            switch (update.type) {
+                case 'add':
+                    if (update.newContent) {
+                        const terms = await this.extractTerms(update.newContent);
+                        this.addDocumentInternal(update.uri, terms, affectedTerms);
+                        affectedDocuments.add(update.uri);
+                    }
+                    break;
+                case 'update':
+                    if (update.oldDocument && update.newContent) {
+                        const oldTerms = update.oldDocument.rawTerms;
+                        const newTerms = await this.extractTerms(update.newContent);
+                        this.updateDocumentInternal(update.uri, oldTerms, newTerms, affectedTerms);
+                        affectedDocuments.add(update.uri);
+                    }
+                    break;
+                case 'delete':
+                    if (update.oldDocument) {
+                        this.deleteDocumentInternal(update.uri, update.oldDocument.rawTerms, affectedTerms);
+                        affectedDocuments.delete(update.uri);
+                    }
+                    break;
+            }
+        }
+        // Phase 2: Recalculate IDF for affected terms using smoothed formula
+        // Smoothed IDF: log((N+1)/(df+1)) + 1 ensures no term gets IDF=0
+        for (const term of affectedTerms) {
+            const df = this.documentFrequency.get(term) || 0;
+            if (df > 0) {
+                this.idf.set(term, Math.log((this.totalDocuments + 1) / (df + 1)) + 1);
+            }
+            else {
+                this.idf.delete(term);
+                this.documentFrequency.delete(term);
+            }
+        }
+        // Phase 3: Update TF-IDF for all documents containing affected terms
+        for (const doc of this.documents.values()) {
+            let needsUpdate = false;
+            // Check if this document contains any affected terms
+            for (const term of doc.rawTerms.keys()) {
+                if (affectedTerms.has(term)) {
+                    needsUpdate = true;
+                    break;
+                }
+            }
+            if (needsUpdate) {
+                this.recalculateDocumentTFIDF(doc);
+                affectedDocuments.add(doc.uri);
+            }
+        }
+        return {
+            totalDocuments: this.totalDocuments,
+            affectedDocuments: affectedDocuments.size,
+            affectedTerms: affectedTerms.size,
+            updateTime: Date.now() - startTime,
+        };
+    }
+    /**
+     * Add a new document
+     */
+    addDocumentInternal(uri, terms, affectedTerms) {
+        this.totalDocuments++;
+        // Update document frequency
+        for (const term of terms.keys()) {
+            const df = this.documentFrequency.get(term) || 0;
+            this.documentFrequency.set(term, df + 1);
+            affectedTerms.add(term);
+        }
+        // Create document vector (TF-IDF will be calculated later)
+        const doc = {
+            uri,
+            terms: new Map(),
+            rawTerms: terms,
+            magnitude: 0,
+        };
+        this.documents.set(uri, doc);
+    }
+    /**
+     * Update an existing document
+     */
+    updateDocumentInternal(uri, oldTerms, newTerms, affectedTerms) {
+        // Collect all affected terms (old + new)
+        for (const term of oldTerms.keys()) {
+            affectedTerms.add(term);
+        }
+        for (const term of newTerms.keys()) {
+            affectedTerms.add(term);
+        }
+        // Update document frequency
+        // Remove old terms
+        for (const term of oldTerms.keys()) {
+            const df = this.documentFrequency.get(term) || 0;
+            this.documentFrequency.set(term, Math.max(0, df - 1));
+        }
+        // Add new terms
+        for (const term of newTerms.keys()) {
+            const df = this.documentFrequency.get(term) || 0;
+            this.documentFrequency.set(term, df + 1);
+        }
+        // Update document
+        const doc = this.documents.get(uri);
+        if (doc) {
+            doc.rawTerms = newTerms;
+            doc.terms.clear();
+            doc.magnitude = 0;
+        }
+    }
+    /**
+     * Delete a document
+     */
+    deleteDocumentInternal(uri, terms, affectedTerms) {
+        this.totalDocuments--;
+        // Update document frequency
+        for (const term of terms.keys()) {
+            const df = this.documentFrequency.get(term) || 0;
+            this.documentFrequency.set(term, Math.max(0, df - 1));
+            affectedTerms.add(term);
+        }
+        this.documents.delete(uri);
+    }
+    /**
+     * Extract terms from content (async - uses StarCoder2)
+     */
+    async extractTerms(content) {
+        const tokens = await tokenize(content);
+        const frequencies = new Map();
+        for (const token of tokens) {
+            frequencies.set(token, (frequencies.get(token) || 0) + 1);
+        }
+        return frequencies;
+    }
+    /**
+     * Calculate TF for a document
+     */
+    calculateTF(termFrequency) {
+        const totalTerms = Array.from(termFrequency.values()).reduce((sum, freq) => sum + freq, 0);
+        const tf = new Map();
+        for (const [term, freq] of termFrequency.entries()) {
+            tf.set(term, freq / totalTerms);
+        }
+        return tf;
+    }
+    /**
+     * Recalculate TF-IDF for a document
+     */
+    recalculateDocumentTFIDF(doc) {
+        const tf = this.calculateTF(doc.rawTerms);
+        doc.terms.clear();
+        // Calculate TF-IDF for each term
+        for (const [term, tfScore] of tf.entries()) {
+            const idfScore = this.idf.get(term) || 0;
+            doc.terms.set(term, tfScore * idfScore);
+        }
+        // Recalculate magnitude
+        let sumSquares = 0;
+        for (const tfidf of doc.terms.values()) {
+            sumSquares += tfidf * tfidf;
+        }
+        doc.magnitude = Math.sqrt(sumSquares);
+    }
+    /**
+     * Get current index state
+     */
+    getIndex() {
+        return {
+            documents: Array.from(this.documents.values()),
+            idf: new Map(this.idf),
+            totalDocuments: this.totalDocuments,
+        };
+    }
+    /**
+     * Check if full rebuild is recommended
+     * Returns true if changes are too extensive for incremental update
+     */
+    async shouldFullRebuild(updates) {
+        const changeRatio = updates.length / Math.max(this.totalDocuments, 1);
+        // If >20% of documents changed, recommend full rebuild
+        if (changeRatio > 0.2) {
+            return true;
+        }
+        // Count new unique terms
+        const newTerms = new Set();
+        for (const update of updates) {
+            if (update.type === 'add' || update.type === 'update') {
+                if (update.newContent) {
+                    const tokens = await tokenize(update.newContent);
+                    for (const token of tokens) {
+                        if (!this.documentFrequency.has(token)) {
+                            newTerms.add(token);
+                        }
+                    }
+                }
+            }
+        }
+        // If too many new terms, recommend full rebuild
+        if (newTerms.size > 1000) {
+            return true;
+        }
+        return false;
+    }
+}
+//# sourceMappingURL=incremental-tfidf.js.map

package/dist/incremental-tfidf.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental-tfidf.js","sourceRoot":"","sources":["../src/incremental-tfidf.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA,CAAC,kCAAkC;AAgBxE;;;;;;;;;;GAUG;AACH,MAAM,OAAO,gBAAgB;IACpB,SAAS,CAA6B;IACtC,GAAG,CAAqB;IACxB,iBAAiB,CAAqB;IACtC,cAAc,CAAQ;IAE9B,YACC,mBAAqC,EAAE,EACvC,aAAkC,IAAI,GAAG,EAAE;QAE3C,IAAI,CAAC,SAAS,GAAG,IAAI,GAAG,EAAE,CAAA;QAC1B,IAAI,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAA;QAC9B,IAAI,CAAC,iBAAiB,GAAG,IAAI,GAAG,EAAE,CAAA;QAClC,IAAI,CAAC,cAAc,GAAG,gBAAgB,CAAC,MAAM,CAAA;QAE7C,iCAAiC;QACjC,KAAK,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACpC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;YAEhC,uDAAuD;YACvD,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBACxC,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;gBAChD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,CAAA;YACzC,CAAC;QACF,CAAC;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QAC5B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAU,CAAA;QACvC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAA;QAE3C,0DAA0D;QAC1D,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,QAAQ,MAAM,CAAC,IAAI,EAAE,CAAC;gBACrB,KAAK,KAAK;oBACT,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;wBACvB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;wBACxD,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,aAAa,CAAC,CAAA;wBAC1D,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;oBAClC,CAAC;oBACD,MAAK;gBAEN,KAAK,QAAQ;oBACZ,IAAI,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;wBAC7C,MAAM,QAAQ,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAA;wBAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;wBAC3D,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAA;wBAC1E,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;oBAClC,CAAC;oBACD,MAAK;gBAEN,KAAK,QAAQ;oBACZ,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;wBACxB,IAAI,CAAC,sBAAsB,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,WAAW,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAA;wBACnF,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;oBACrC,CAAC;oBACD,MAAK;YACP,CAAC;QACF,CAAC;QAED,qEAAqE;QACrE,iEAAiE;QACjE,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChD,IAAI,EAAE,GAAG,CAAC,EAAE,CAAC;gBACZ,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACvE,CAAC;iBAAM,CAAC;gBACP,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACrB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACpC,CAAC;QACF,CAAC;QAED,qEAAqE;QACrE,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,WAAW,GAAG,KAAK,CAAA;YAEvB,qDAAqD;YACrD,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;gBACxC,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7B,WAAW,GAAG,IAAI,CAAA;oBAClB,MAAK;gBACN,CAAC;YACF,CAAC;YAED,IAAI,WAAW,EAAE,CAAC;gBACjB,IAAI,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAA;gBAClC,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC/B,CAAC;QACF,CAAC;QAED,OAAO;YACN,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,iBAAiB,EAAE,iBAAiB,CAAC,IAAI;YACzC,aAAa,EAAE,aAAa,CAAC,IAAI;YACjC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SAClC,CAAA;IACF,CAAC;IAED;;OAEG;IACK,mBAAmB,CAC1B,GAAW,EACX,KAA0B,EAC1B,aAA0B;QAE1B,IAAI,CAAC,cAAc,EAAE,CAAA;QAErB,4BAA4B;QAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACjC,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,CAAA;YACxC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACxB,CAAC;QAED,2DAA2D;QAC3D,MAAM,GAAG,GAAmB;YAC3B,GAAG;YACH,KAAK,EAAE,IAAI,GAAG,EAAE;YAChB,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,CAAC;SACZ,CAAA;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;IAC7B,CAAC;IAED;;OAEG;IACK,sBAAsB,CAC7B,GAAW,EACX,QAA6B,EAC7B,QAA6B,EAC7B,aAA0B;QAE1B,yCAAyC;QACzC,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACxB,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACxB,CAAC;QAED,4BAA4B;QAC5B,mBAAmB;QACnB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;QACtD,CAAC;QAED,gBAAgB;QAChB,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;YACpC,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,CAAA;QACzC,CAAC;QAED,kBAAkB;QAClB,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACnC,IAAI,GAAG,EAAE,CAAC;YACT,GAAG,CAAC,QAAQ,GAAG,QAAQ,CAAA;YACvB,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;YACjB,GAAG,CAAC,SAAS,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED;;OAEG;IACK,sBAAsB,CAC7B,GAAW,EACX,KAA0B,EAC1B,aAA0B;QAE1B,IAAI,CAAC,cAAc,EAAE,CAAA;QAErB,4BAA4B;QAC5B,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;YACjC,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAChD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,CAAA;YACrD,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACxB,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC3B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY,CAAC,OAAe;QACzC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAA;QACtC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE7C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QAC1D,CAAC;QAED,OAAO,WAAW,CAAA;IACnB,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,aAAkC;QACrD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,GAAG,GAAG,IAAI,EAAE,CAAC,CAAC,CAAA;QAC1F,MAAM,EAAE,GAAG,IAAI,GAAG,EAAkB,CAAA;QAEpC,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;YACpD,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,GAAG,UAAU,CAAC,CAAA;QAChC,CAAC;QAED,OAAO,EAAE,CAAA;IACV,CAAC;IAED;;OAEG;IACK,wBAAwB,CAAC,GAAmB;QACnD,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QACzC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QAEjB,iCAAiC;QACjC,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACxC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,GAAG,QAAQ,CAAC,CAAA;QACxC,CAAC;QAED,wBAAwB;QACxB,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,UAAU,IAAI,KAAK,GAAG,KAAK,CAAA;QAC5B,CAAC;QACD,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACtC,CAAC;IAED;;OAEG;IACH,QAAQ;QAKP,OAAO;YACN,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YAC9C,GAAG,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;YACtB,cAAc,EAAE,IAAI,CAAC,cAAc;SACnC,CAAA;IACF,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,iBAAiB,CAAC,OAA4B;QACnD,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC,CAAA;QAErE,uDAAuD;QACvD,IAAI,WAAW,GAAG,GAAG,EAAE,CAAC;YACvB,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,yBAAyB;QACzB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACvD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;oBACvB,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,CAAA;oBAChD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBAC5B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;4BACxC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;wBACpB,CAAC;oBACF,CAAC;gBACF,CAAC;YACF,CAAC;QACF,CAAC;QAED,gDAAgD;QAChD,IAAI,QAAQ,CAAC,IAAI,GAAG,IAAI,EAAE,CAAC;YAC1B,OAAO,IAAI,CAAA;QACZ,CAAC;QAED,OAAO,KAAK,CAAA;IACb,CAAC;CACD"}

package/dist/incremental-tfidf.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for Incremental TF-IDF implementation (StarCoder2 tokenizer)
+ */
+export {};
+//# sourceMappingURL=incremental-tfidf.test.d.ts.map

package/dist/incremental-tfidf.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental-tfidf.test.d.ts","sourceRoot":"","sources":["../src/incremental-tfidf.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}