@matperez/coderag 0.1.24

package/dist/incremental-tfidf.test.js

@@ -0,0 +1,276 @@
+/**
+ * Tests for Incremental TF-IDF implementation (StarCoder2 tokenizer)
+ */
+import { beforeAll, describe, expect, it } from 'vitest';
+import { IncrementalTFIDF } from './incremental-tfidf.js';
+import { buildSearchIndex, initializeTokenizer } from './tfidf.js';
+// Skip if running in CI without model cache
+const shouldSkip = process.env.CI === 'true' && !process.env.HF_HOME;
+describe('IncrementalTFIDF', () => {
+    beforeAll(async () => {
+        if (shouldSkip)
+            return;
+        await initializeTokenizer();
+    }, 60000);
+    describe('initialization', () => {
+        it.skipIf(shouldSkip)('should initialize from existing index', async () => {
+            const documents = [
+                { uri: 'file://doc1.txt', content: 'hello world' },
+                { uri: 'file://doc2.txt', content: 'hello there' },
+            ];
+            const index = await buildSearchIndex(documents);
+            const incrementalEngine = new IncrementalTFIDF(index.documents, index.idf);
+            const result = incrementalEngine.getIndex();
+            expect(result.totalDocuments).toBe(2);
+            expect(result.documents.length).toBe(2);
+            expect(result.idf.size).toBeGreaterThan(0);
+        });
+        it('should initialize with empty index', () => {
+            const incrementalEngine = new IncrementalTFIDF([], new Map());
+            const result = incrementalEngine.getIndex();
+            expect(result.totalDocuments).toBe(0);
+            expect(result.documents.length).toBe(0);
+        });
+    });
+    describe('add document', () => {
+        it.skipIf(shouldSkip)('should add new document and update IDF', async () => {
+            const initialDocs = [{ uri: 'file://doc1.txt', content: 'hello world' }];
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            const stats = await engine.applyUpdates([
+                {
+                    type: 'add',
+                    uri: 'file://doc2.txt',
+                    newContent: 'hello there world',
+                },
+            ]);
+            expect(stats.affectedDocuments).toBeGreaterThan(0);
+            expect(stats.affectedTerms).toBeGreaterThan(0);
+            const result = engine.getIndex();
+            expect(result.totalDocuments).toBe(2);
+            expect(result.documents.length).toBe(2);
+        });
+        it.skipIf(shouldSkip)('should handle adding multiple documents', async () => {
+            const engine = new IncrementalTFIDF([], new Map());
+            const stats = await engine.applyUpdates([
+                {
+                    type: 'add',
+                    uri: 'file://doc1.txt',
+                    newContent: 'hello world',
+                },
+                {
+                    type: 'add',
+                    uri: 'file://doc2.txt',
+                    newContent: 'goodbye world',
+                },
+            ]);
+            expect(stats.affectedDocuments).toBe(2);
+            const result = engine.getIndex();
+            expect(result.totalDocuments).toBe(2);
+            expect(result.documents.length).toBe(2);
+        });
+    });
+    describe('update document', () => {
+        it.skipIf(shouldSkip)('should update existing document and recalculate IDF', async () => {
+            const initialDocs = [
+                { uri: 'file://doc1.txt', content: 'hello world' },
+                { uri: 'file://doc2.txt', content: 'hello there' },
+            ];
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            const oldDoc = index.documents.find((d) => d.uri === 'file://doc1.txt');
+            const stats = await engine.applyUpdates([
+                {
+                    type: 'update',
+                    uri: 'file://doc1.txt',
+                    oldDocument: oldDoc,
+                    newContent: 'goodbye universe',
+                },
+            ]);
+            expect(stats.affectedDocuments).toBeGreaterThan(0);
+            expect(stats.affectedTerms).toBeGreaterThan(0);
+            const result = engine.getIndex();
+            expect(result.totalDocuments).toBe(2);
+        });
+        it.skipIf(shouldSkip)('should update affected documents when term frequencies change', async () => {
+            const initialDocs = [
+                { uri: 'file://doc1.txt', content: 'rare word' },
+                { uri: 'file://doc2.txt', content: 'common word common' },
+            ];
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            const oldDoc = index.documents.find((d) => d.uri === 'file://doc1.txt');
+            await engine.applyUpdates([
+                {
+                    type: 'update',
+                    uri: 'file://doc1.txt',
+                    oldDocument: oldDoc,
+                    newContent: 'word word word',
+                },
+            ]);
+            const result = engine.getIndex();
+            expect(result.totalDocuments).toBe(2);
+        });
+    });
+    describe('delete document', () => {
+        it.skipIf(shouldSkip)('should delete document and update IDF', async () => {
+            const initialDocs = [
+                { uri: 'file://doc1.txt', content: 'hello world' },
+                { uri: 'file://doc2.txt', content: 'hello there' },
+            ];
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            const oldDoc = index.documents.find((d) => d.uri === 'file://doc1.txt');
+            const stats = await engine.applyUpdates([
+                {
+                    type: 'delete',
+                    uri: 'file://doc1.txt',
+                    oldDocument: oldDoc,
+                },
+            ]);
+            expect(stats.affectedTerms).toBeGreaterThan(0);
+            const result = engine.getIndex();
+            expect(result.totalDocuments).toBe(1);
+            expect(result.documents.length).toBe(1);
+            expect(result.documents.find((d) => d.uri === 'file://doc1.txt')).toBeUndefined();
+        });
+        it.skipIf(shouldSkip)('should handle deleting last occurrence of terms', async () => {
+            const initialDocs = [
+                { uri: 'file://doc1.txt', content: 'unique word' },
+                { uri: 'file://doc2.txt', content: 'common word' },
+            ];
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            const oldDoc = index.documents.find((d) => d.uri === 'file://doc1.txt');
+            await engine.applyUpdates([
+                {
+                    type: 'delete',
+                    uri: 'file://doc1.txt',
+                    oldDocument: oldDoc,
+                },
+            ]);
+            const result = engine.getIndex();
+            expect(result.totalDocuments).toBe(1);
+        });
+    });
+    describe('mixed operations', () => {
+        it.skipIf(shouldSkip)('should handle add, update, and delete in single batch', async () => {
+            const initialDocs = [
+                { uri: 'file://doc1.txt', content: 'hello world' },
+                { uri: 'file://doc2.txt', content: 'hello there' },
+            ];
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            const doc1 = index.documents.find((d) => d.uri === 'file://doc1.txt');
+            const doc2 = index.documents.find((d) => d.uri === 'file://doc2.txt');
+            const stats = await engine.applyUpdates([
+                {
+                    type: 'delete',
+                    uri: 'file://doc1.txt',
+                    oldDocument: doc1,
+                },
+                {
+                    type: 'update',
+                    uri: 'file://doc2.txt',
+                    oldDocument: doc2,
+                    newContent: 'greetings universe',
+                },
+                {
+                    type: 'add',
+                    uri: 'file://doc3.txt',
+                    newContent: 'new document here',
+                },
+            ]);
+            expect(stats.affectedDocuments).toBeGreaterThan(0);
+            expect(stats.affectedTerms).toBeGreaterThan(0);
+            const result = engine.getIndex();
+            expect(result.totalDocuments).toBe(2); // doc1 deleted, doc3 added
+            expect(result.documents.length).toBe(2);
+        });
+    });
+    describe('shouldFullRebuild', () => {
+        it.skipIf(shouldSkip)('should recommend full rebuild when >20% of documents change', async () => {
+            const initialDocs = Array.from({ length: 100 }, (_, i) => ({
+                uri: `file://doc${i}.txt`,
+                content: 'hello world',
+            }));
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            // Create updates for 25 documents (25%)
+            const updates = Array.from({ length: 25 }, (_, i) => ({
+                type: 'update',
+                uri: `file://doc${i}.txt`,
+                oldDocument: index.documents[i],
+                newContent: 'updated content',
+            }));
+            expect(await engine.shouldFullRebuild(updates)).toBe(true);
+        });
+        it.skipIf(shouldSkip)('should not recommend full rebuild when <20% of documents change', async () => {
+            const initialDocs = Array.from({ length: 100 }, (_, i) => ({
+                uri: `file://doc${i}.txt`,
+                content: 'hello world',
+            }));
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            // Create updates for 15 documents (15%)
+            const updates = Array.from({ length: 15 }, (_, i) => ({
+                type: 'update',
+                uri: `file://doc${i}.txt`,
+                oldDocument: index.documents[i],
+                newContent: 'updated content',
+            }));
+            expect(await engine.shouldFullRebuild(updates)).toBe(false);
+        });
+        it.skipIf(shouldSkip)('should recommend full rebuild when adding >1000 new terms', async () => {
+            const initialDocs = [{ uri: 'file://doc1.txt', content: 'hello world' }];
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            // Create content with >1000 unique terms
+            const newTerms = Array.from({ length: 1100 }, (_, i) => `term${i}`).join(' ');
+            const updates = [
+                {
+                    type: 'add',
+                    uri: 'file://doc2.txt',
+                    newContent: newTerms,
+                },
+            ];
+            expect(await engine.shouldFullRebuild(updates)).toBe(true);
+        });
+    });
+    describe('TF-IDF calculations', () => {
+        it.skipIf(shouldSkip)('should calculate correct TF-IDF values after update', async () => {
+            const initialDocs = [
+                { uri: 'file://doc1.txt', content: 'word word word' },
+                { uri: 'file://doc2.txt', content: 'word other' },
+            ];
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            const result = engine.getIndex();
+            // IDF values should be computed
+            expect(result.idf.size).toBeGreaterThan(0);
+        });
+        it.skipIf(shouldSkip)('should maintain vector magnitudes', async () => {
+            const initialDocs = [{ uri: 'file://doc1.txt', content: 'hello world' }];
+            const index = await buildSearchIndex(initialDocs);
+            const engine = new IncrementalTFIDF(index.documents, index.idf);
+            await engine.applyUpdates([
+                {
+                    type: 'add',
+                    uri: 'file://doc2.txt',
+                    newContent: 'hello there',
+                },
+            ]);
+            const result = engine.getIndex();
+            for (const doc of result.documents) {
+                // Calculate magnitude from terms
+                let sumSquares = 0;
+                for (const tfidf of doc.terms.values()) {
+                    sumSquares += tfidf * tfidf;
+                }
+                const expectedMagnitude = Math.sqrt(sumSquares);
+                expect(doc.magnitude).toBeCloseTo(expectedMagnitude, 10);
+            }
+        });
+    });
+});
+//# sourceMappingURL=incremental-tfidf.test.js.map

package/dist/incremental-tfidf.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"incremental-tfidf.test.js","sourceRoot":"","sources":["../src/incremental-tfidf.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAA;AACzD,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAA;AAElE,4CAA4C;AAC5C,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAA;AAEpE,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;IACjC,SAAS,CAAC,KAAK,IAAI,EAAE;QACpB,IAAI,UAAU;YAAE,OAAM;QACtB,MAAM,mBAAmB,EAAE,CAAA;IAC5B,CAAC,EAAE,KAAK,CAAC,CAAA;IAET,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,SAAS,GAAG;gBACjB,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;gBAClD,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;aAClD,CAAA;YAED,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,SAAS,CAAC,CAAA;YAC/C,MAAM,iBAAiB,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE1E,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,CAAA;YAC3C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACrC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAC3C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;YAC7C,MAAM,iBAAiB,GAAG,IAAI,gBAAgB,CAAC,EAAE,EAAE,IAAI,GAAG,EAAE,CAAC,CAAA;YAE7D,MAAM,MAAM,GAAG,iBAAiB,CAAC,QAAQ,EAAE,CAAA;YAC3C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACrC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC7B,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,WAAW,GAAG,CAAC,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAA;YACxE,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC;gBACvC;oBACC,IAAI,EAAE,KAAK;oBACX,GAAG,EAAE,iBAAiB;oBACtB,UAAU,EAAE,mBAAmB;iBAC/B;aACD,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAClD,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAE9C,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACrC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YAC3E,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,EAAE,EAAE,IAAI,GAAG,EAAE,CAAC,CAAA;YAElD,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC;gBACvC;oBACC,IAAI,EAAE,KAAK;oBACX,GAAG,EAAE,iBAAiB;oBACtB,UAAU,EAAE,aAAa;iBACzB;gBACD;oBACC,IAAI,EAAE,KAAK;oBACX,GAAG,EAAE,iBAAiB;oBACtB,UAAU,EAAE,eAAe;iBAC3B;aACD,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAEvC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACrC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;YACvF,MAAM,WAAW,GAAG;gBACnB,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;gBAClD,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;aAClD,CAAA;YAED,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,iBAAiB,CAAE,CAAA;YAExE,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC;gBACvC;oBACC,IAAI,EAAE,QAAQ;oBACd,GAAG,EAAE,iBAAiB;oBACtB,WAAW,EAAE,MAAM;oBACnB,UAAU,EAAE,kBAAkB;iBAC9B;aACD,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAClD,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAE9C,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;YACjG,MAAM,WAAW,GAAG;gBACnB,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,WAAW,EAAE;gBAChD,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,oBAAoB,EAAE;aACzD,CAAA;YAED,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,iBAAiB,CAAE,CAAA;YAExE,MAAM,MAAM,CAAC,YAAY,CAAC;gBACzB;oBACC,IAAI,EAAE,QAAQ;oBACd,GAAG,EAAE,iBAAiB;oBACtB,WAAW,EAAE,MAAM;oBACnB,UAAU,EAAE,gBAAgB;iBAC5B;aACD,CAAC,CAAA;YAEF,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAChC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,WAAW,GAAG;gBACnB,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;gBAClD,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;aAClD,CAAA;YAED,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,iBAAiB,CAAE,CAAA;YAExE,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC;gBACvC;oBACC,IAAI,EAAE,QAAQ;oBACd,GAAG,EAAE,iBAAiB;oBACtB,WAAW,EAAE,MAAM;iBACnB;aACD,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAE9C,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACrC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACvC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,iBAAiB,CAAC,CAAC,CAAC,aAAa,EAAE,CAAA;QAClF,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;YACnF,MAAM,WAAW,GAAG;gBACnB,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;gBAClD,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;aAClD,CAAA;YAED,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,iBAAiB,CAAE,CAAA;YAExE,MAAM,MAAM,CAAC,YAAY,CAAC;gBACzB;oBACC,IAAI,EAAE,QAAQ;oBACd,GAAG,EAAE,iBAAiB;oBACtB,WAAW,EAAE,MAAM;iBACnB;aACD,CAAC,CAAA;YAEF,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;YACzF,MAAM,WAAW,GAAG;gBACnB,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;gBAClD,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE;aAClD,CAAA;YAED,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,iBAAiB,CAAE,CAAA;YACtE,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,iBAAiB,CAAE,CAAA;YAEtE,MAAM,KAAK,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC;gBACvC;oBACC,IAAI,EAAE,QAAQ;oBACd,GAAG,EAAE,iBAAiB;oBACtB,WAAW,EAAE,IAAI;iBACjB;gBACD;oBACC,IAAI,EAAE,QAAQ;oBACd,GAAG,EAAE,iBAAiB;oBACtB,WAAW,EAAE,IAAI;oBACjB,UAAU,EAAE,oBAAoB;iBAChC;gBACD;oBACC,IAAI,EAAE,KAAK;oBACX,GAAG,EAAE,iBAAiB;oBACtB,UAAU,EAAE,mBAAmB;iBAC/B;aACD,CAAC,CAAA;YAEF,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAClD,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YAE9C,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAChC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA,CAAC,2BAA2B;YACjE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACxC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC/F,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC1D,GAAG,EAAE,aAAa,CAAC,MAAM;gBACzB,OAAO,EAAE,aAAa;aACtB,CAAC,CAAC,CAAA;YAEH,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,wCAAwC;YACxC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrD,IAAI,EAAE,QAAiB;gBACvB,GAAG,EAAE,aAAa,CAAC,MAAM;gBACzB,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC/B,UAAU,EAAE,iBAAiB;aAC7B,CAAC,CAAC,CAAA;YAEH,MAAM,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC3D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;YACnG,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC1D,GAAG,EAAE,aAAa,CAAC,MAAM;gBACzB,OAAO,EAAE,aAAa;aACtB,CAAC,CAAC,CAAA;YAEH,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,wCAAwC;YACxC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrD,IAAI,EAAE,QAAiB;gBACvB,GAAG,EAAE,aAAa,CAAC,MAAM;gBACzB,WAAW,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC/B,UAAU,EAAE,iBAAiB;aAC7B,CAAC,CAAC,CAAA;YAEH,MAAM,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAC5D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;YAC7F,MAAM,WAAW,GAAG,CAAC,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAA;YAExE,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,yCAAyC;YACzC,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAE7E,MAAM,OAAO,GAAG;gBACf;oBACC,IAAI,EAAE,KAAc;oBACpB,GAAG,EAAE,iBAAiB;oBACtB,UAAU,EAAE,QAAQ;iBACpB;aACD,CAAA;YAED,MAAM,CAAC,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC3D,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,qDAAqD,EAAE,KAAK,IAAI,EAAE;YACvF,MAAM,WAAW,GAAG;gBACnB,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,gBAAgB,EAAE;gBACrD,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,YAAY,EAAE;aACjD,CAAA;YAED,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAEhC,gCAAgC;YAChC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QAC3C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YACrE,MAAM,WAAW,GAAG,CAAC,EAAE,GAAG,EAAE,iBAAiB,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC,CAAA;YAExE,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAA;YACjD,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;YAE/D,MAAM,MAAM,CAAC,YAAY,CAAC;gBACzB;oBACC,IAAI,EAAE,KAAK;oBACX,GAAG,EAAE,iBAAiB;oBACtB,UAAU,EAAE,aAAa;iBACzB;aACD,CAAC,CAAA;YAEF,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAA;YAEhC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACpC,iCAAiC;gBACjC,IAAI,UAAU,GAAG,CAAC,CAAA;gBAClB,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;oBACxC,UAAU,IAAI,KAAK,GAAG,KAAK,CAAA;gBAC5B,CAAC;gBACD,MAAM,iBAAiB,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBAE/C,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,WAAW,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAA;YACzD,CAAC;QACF,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/index.d.ts

@@ -0,0 +1,18 @@
+/**
+ * @codebase-search/core
+ * Core library for intelligent codebase search using TF-IDF
+ */
+export { type ASTChunkOptions, type ChunkResult, chunkCodeByAST, chunkCodeByASTSimple, getSupportedLanguages, } from './ast-chunking.js';
+export { getCoderagDataDir, type ProjectMetadata } from './db/client.js';
+export { chunkText, composeProviders, cosineSimilarity, createDefaultConfig, createEmbeddingProvider, createMockProvider, createOpenAIProvider, type EmbeddingConfig, type EmbeddingProvider, generateMockEmbedding, getDefaultEmbeddingProvider, normalizeVector, } from './embeddings.js';
+export { type HybridSearchOptions, type HybridSearchResult, hybridSearch, keywordSearch, semanticSearch, } from './hybrid-search.js';
+export { type IncrementalStats, IncrementalTFIDF, type IncrementalUpdate, } from './incremental-tfidf.js';
+export { CodebaseIndexer, type FileChangeEvent, type FileDiff, type IndexerOptions, type IndexingStatus, type SearchResult, } from './indexer.js';
+export { type EmbeddedLanguageConfig, EXTENSION_TO_LANGUAGE, getLanguageConfig, getLanguageConfigByExtension, getLanguageFromPath, getSupportedExtensions, isLanguageSupported, LANGUAGE_REGISTRY, type LanguageConfig, } from './language-config.js';
+export { type CacheEntry, type CacheStats, createCacheKey, LRUCache } from './search-cache.js';
+export { type CodebaseFile, MemoryStorage } from './storage.js';
+export { PersistentStorage } from './storage-persistent.js';
+export { buildSearchIndex, calculateCosineSimilarity, type DocumentVector, processQuery, type SearchIndex, searchDocuments, } from './tfidf.js';
+export { detectLanguage, isTextFile, loadGitignore, type ScanOptions, type ScanResult, scanFiles, simpleHash, } from './utils.js';
+export { type VectorDocument, type VectorSearchResult, VectorStorage, type VectorStorageOptions, type VectorStorageStats, } from './vector-storage.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACN,KAAK,eAAe,EACpB,KAAK,WAAW,EAChB,cAAc,EACd,oBAAoB,EACpB,qBAAqB,GACrB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,iBAAiB,EAAE,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAA;AACxE,OAAO,EACN,SAAS,EACT,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,oBAAoB,EACpB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,qBAAqB,EACrB,2BAA2B,EAC3B,eAAe,GACf,MAAM,iBAAiB,CAAA;AACxB,OAAO,EACN,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,YAAY,EACZ,aAAa,EACb,cAAc,GACd,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EACN,KAAK,gBAAgB,EACrB,gBAAgB,EAChB,KAAK,iBAAiB,GACtB,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EACN,eAAe,EACf,KAAK,eAAe,EACpB,KAAK,QAAQ,EACb,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,YAAY,GACjB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,KAAK,sBAAsB,EAC3B,qBAAqB,EACrB,iBAAiB,EACjB,4BAA4B,EAC5B,mBAAmB,EACnB,sBAAsB,EACtB,mBAAmB,EACnB,iBAAiB,EACjB,KAAK,cAAc,GACnB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,KAAK,UAAU,EAAE,KAAK,UAAU,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAC9F,OAAO,EAAE,KAAK,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAA;AAC3D,OAAO,EACN,gBAAgB,EAChB,yBAAyB,EACzB,KAAK,cAAc,EACnB,YAAY,EACZ,KAAK,WAAW,EAChB,eAAe,GACf,MAAM,YAAY,CAAA;AACnB,OAAO,EACN,cAAc,EACd,UAAU,EACV,aAAa,EACb,KAAK,WAAW,EAChB,KAAK,UAAU,EACf,SAAS,EACT,UAAU,GACV,MAAM,YAAY,CAAA;AACnB,OAAO,EACN,KAAK,cAAc,EACnB,KAAK,kBAAkB,EACvB,aAAa,EACb,KAAK,oBAAoB,EACzB,KAAK,kBAAkB,GACvB,MAAM,qBAAqB,CAAA"}

package/dist/index.js

@@ -0,0 +1,19 @@
+/**
+ * @codebase-search/core
+ * Core library for intelligent codebase search using TF-IDF
+ */
+export { chunkCodeByAST, chunkCodeByASTSimple, getSupportedLanguages, } from './ast-chunking.js';
+export { getCoderagDataDir } from './db/client.js';
+export { chunkText, composeProviders, cosineSimilarity, createDefaultConfig, createEmbeddingProvider, createMockProvider, createOpenAIProvider, generateMockEmbedding, getDefaultEmbeddingProvider, normalizeVector, } from './embeddings.js';
+export { hybridSearch, keywordSearch, semanticSearch, } from './hybrid-search.js';
+export { IncrementalTFIDF, } from './incremental-tfidf.js';
+// Re-export main components
+export { CodebaseIndexer, } from './indexer.js';
+export { EXTENSION_TO_LANGUAGE, getLanguageConfig, getLanguageConfigByExtension, getLanguageFromPath, getSupportedExtensions, isLanguageSupported, LANGUAGE_REGISTRY, } from './language-config.js';
+export { createCacheKey, LRUCache } from './search-cache.js';
+export { MemoryStorage } from './storage.js';
+export { PersistentStorage } from './storage-persistent.js';
+export { buildSearchIndex, calculateCosineSimilarity, processQuery, searchDocuments, } from './tfidf.js';
+export { detectLanguage, isTextFile, loadGitignore, scanFiles, simpleHash, } from './utils.js';
+export { VectorStorage, } from './vector-storage.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAGN,cAAc,EACd,oBAAoB,EACpB,qBAAqB,GACrB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,iBAAiB,EAAwB,MAAM,gBAAgB,CAAA;AACxE,OAAO,EACN,SAAS,EACT,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,oBAAoB,EAGpB,qBAAqB,EACrB,2BAA2B,EAC3B,eAAe,GACf,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAGN,YAAY,EACZ,aAAa,EACb,cAAc,GACd,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAEN,gBAAgB,GAEhB,MAAM,wBAAwB,CAAA;AAC/B,4BAA4B;AAC5B,OAAO,EACN,eAAe,GAMf,MAAM,cAAc,CAAA;AACrB,OAAO,EAEN,qBAAqB,EACrB,iBAAiB,EACjB,4BAA4B,EAC5B,mBAAmB,EACnB,sBAAsB,EACtB,mBAAmB,EACnB,iBAAiB,GAEjB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAoC,cAAc,EAAE,QAAQ,EAAE,MAAM,mBAAmB,CAAA;AAC9F,OAAO,EAAqB,aAAa,EAAE,MAAM,cAAc,CAAA;AAC/D,OAAO,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAA;AAC3D,OAAO,EACN,gBAAgB,EAChB,yBAAyB,EAEzB,YAAY,EAEZ,eAAe,GACf,MAAM,YAAY,CAAA;AACnB,OAAO,EACN,cAAc,EACd,UAAU,EACV,aAAa,EAGb,SAAS,EACT,UAAU,GACV,MAAM,YAAY,CAAA;AACnB,OAAO,EAGN,aAAa,GAGb,MAAM,qBAAqB,CAAA"}

package/dist/indexer.d.ts

@@ -0,0 +1,205 @@
+/**
+ * Codebase indexer service
+ * Uses chunk-level indexing for better search granularity
+ */
+import type { EmbeddingProvider } from './embeddings.js';
+import { type Storage } from './storage.js';
+import { type SearchIndex } from './tfidf.js';
+import { type FileMetadata } from './utils.js';
+import { VectorStorage } from './vector-storage.js';
+export interface IndexerOptions {
+    codebaseRoot?: string;
+    maxFileSize?: number;
+    storage?: Storage;
+    onProgress?: (current: number, total: number, file: string) => void;
+    watch?: boolean;
+    onFileChange?: (event: FileChangeEvent) => void;
+    embeddingProvider?: EmbeddingProvider;
+    vectorBatchSize?: number;
+    indexingBatchSize?: number;
+    lowMemoryMode?: boolean;
+    /** Skip files whose hash matches existing (default true). Set false for benchmarking. */
+    skipUnchanged?: boolean;
+    /** Tokenize chunk contents in parallel sub-batches (default false). Set true to enable. */
+    useParallelTokenize?: boolean;
+}
+export interface FileChangeEvent {
+    type: 'add' | 'change' | 'unlink';
+    path: string;
+    timestamp: number;
+}
+export interface IndexingStatus {
+    isIndexing: boolean;
+    progress: number;
+    totalFiles: number;
+    processedFiles: number;
+    totalChunks: number;
+    indexedChunks: number;
+    currentFile?: string;
+}
+/**
+ * Result of comparing filesystem with database
+ */
+export interface FileDiff {
+    added: FileMetadata[];
+    changed: FileMetadata[];
+    deleted: string[];
+    unchanged: number;
+}
+export declare class CodebaseIndexer {
+    private codebaseRoot;
+    private maxFileSize;
+    private storage;
+    private searchIndex;
+    private incrementalEngine;
+    private pendingFileChanges;
+    private searchCache;
+    private watcher;
+    private isWatching;
+    private onFileChangeCallback?;
+    private pendingUpdates;
+    private ignoreFilter;
+    private status;
+    private vectorStorage?;
+    private embeddingProvider?;
+    private vectorBatchSize;
+    private indexingBatchSize;
+    private lowMemoryMode;
+    constructor(options?: IndexerOptions);
+    /**
+     * Get current indexing status
+     */
+    getStatus(): IndexingStatus;
+    /**
+     * Compare filesystem with database to find changes
+     * Used for incremental updates after long periods of inactivity
+     */
+    private diffFilesystem;
+    /**
+     * Process incremental changes (add, update, delete files)
+     * Uses chunk-level indexing with SQL-based updates
+     */
+    private processIncrementalChanges;
+    /**
+     * Get search index
+     */
+    getSearchIndex(): SearchIndex | null;
+    /**
+     * Index the codebase
+     */
+    index(options?: IndexerOptions): Promise<void>;
+    /**
+     * Start watching for file changes
+     * Uses @parcel/watcher which provides native FSEvents on macOS
+     */
+    startWatch(): Promise<void>;
+    /**
+     * Check if a file should be ignored
+     */
+    private shouldIgnore;
+    /**
+     * Stop watching for file changes
+     */
+    stopWatch(): Promise<void>;
+    /**
+     * Close indexer and release all resources
+     * Should be called when the indexer is no longer needed
+     */
+    close(): Promise<void>;
+    /**
+     * Handle file change events with debouncing
+     */
+    private handleFileChange;
+    /**
+     * Process file change and update index
+     */
+    private processFileChange;
+    /**
+     * Rebuild search index from current storage
+     * Uses incremental update when possible for performance
+     */
+    private rebuildSearchIndex;
+    /**
+     * Full rebuild of search index (fallback when incremental not possible)
+     * For persistent storage, this rebuilds the chunk-level index
+     */
+    private fullRebuildSearchIndex;
+    /**
+     * Persist search index to storage
+     * NOTE: For PersistentStorage, chunk-based indexing happens inline during index()
+     * This method is only used for in-memory storage fallback
+     */
+    private persistSearchIndex;
+    /**
+     * Check if currently watching for changes
+     */
+    isWatchEnabled(): boolean;
+    /**
+     * Search the codebase
+     * Returns chunk-level results when using persistent storage (SQL-based search)
+     */
+    search(query: string, options?: {
+        limit?: number;
+        includeContent?: boolean;
+        fileExtensions?: string[];
+        pathFilter?: string;
+        excludePaths?: string[];
+        contextLines?: number;
+        maxSnippetChars?: number;
+        maxSnippetBlocks?: number;
+    }): Promise<SearchResult[]>;
+    /**
+     * Chunk-based search with BM25 scoring
+     * Returns chunk content directly (no separate snippet extraction needed)
+     */
+    private searchChunks;
+    /**
+     * Extract code block snippets from content around matched terms
+     *
+     * Returns the most relevant code blocks (not just lines) with context.
+     * Blocks are ranked by term density (more matched terms = higher score).
+     */
+    private extractSnippet;
+    /**
+     * Get file content
+     */
+    getFileContent(filePath: string): Promise<string | null>;
+    /**
+     * Get total indexed files count
+     */
+    getIndexedCount(): Promise<number>;
+    /**
+     * Get vector storage (for hybrid search)
+     */
+    getVectorStorage(): VectorStorage | undefined;
+    /**
+     * Get embedding provider (for hybrid search)
+     */
+    getEmbeddingProvider(): EmbeddingProvider | undefined;
+    /**
+     * Build vector index from file metadata (Memory optimization)
+     * Generates embeddings per CHUNK, not per file
+     */
+    private buildVectorIndexFromMetadata;
+    /**
+     * Update vectors for a single file (chunk-level)
+     * Deletes old chunks and adds new ones
+     */
+    private updateFileVector;
+    /**
+     * Delete vectors for a file (all chunks)
+     */
+    private deleteFileVector;
+}
+export interface SearchResult {
+    path: string;
+    score: number;
+    matchedTerms: string[];
+    language?: string;
+    size: number;
+    snippet?: string;
+    chunkType?: string;
+    startLine?: number;
+    endLine?: number;
+}
+//# sourceMappingURL=indexer.d.ts.map

package/dist/indexer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../src/indexer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAGxD,OAAO,EAAoC,KAAK,OAAO,EAAE,MAAM,cAAc,CAAA;AAE7E,OAAO,EAAoC,KAAK,WAAW,EAAY,MAAM,YAAY,CAAA;AACzF,OAAO,EAEN,KAAK,YAAY,EAOjB,MAAM,YAAY,CAAA;AACnB,OAAO,EAAuB,aAAa,EAAE,MAAM,qBAAqB,CAAA;AAExE,MAAM,WAAW,cAAc;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;IACnE,KAAK,CAAC,EAAE,OAAO,CAAA;IACf,YAAY,CAAC,EAAE,CAAC,KAAK,EAAE,eAAe,KAAK,IAAI,CAAA;IAC/C,iBAAiB,CAAC,EAAE,iBAAiB,CAAA;IACrC,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,yFAAyF;IACzF,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,2FAA2F;IAC3F,mBAAmB,CAAC,EAAE,OAAO,CAAA;CAC7B;AAED,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,KAAK,GAAG,QAAQ,GAAG,QAAQ,CAAA;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,cAAc;IAC9B,UAAU,EAAE,OAAO,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,YAAY,EAAE,CAAA;IACrB,OAAO,EAAE,YAAY,EAAE,CAAA;IACvB,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CACjB;AAED,qBAAa,eAAe;IAC3B,OAAO,CAAC,YAAY,CAAQ;IAC5B,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAA2B;IAC9C,OAAO,CAAC,iBAAiB,CAAgC;IACzD,OAAO,CAAC,kBAAkB,CAA0B;IACpD,OAAO,CAAC,WAAW,CAA0B;IAC7C,OAAO,CAAC,OAAO,CAA2D;IAC1E,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,oBAAoB,CAAC,CAAkC;IAC/D,OAAO,CAAC,cAAc,CAAoC;IAC1D,OAAO,CAAC,YAAY,CAAsB;IAC1C,OAAO,CAAC,MAAM,CAOb;IACD,OAAO,CAAC,aAAa,CAAC,CAAe;IACrC,OAAO,CAAC,iBAAiB,CAAC,CAAmB;IAC7C,OAAO,CAAC,eAAe,CAAQ;IAC/B,OAAO,CAAC,iBAAiB,CAAQ;IACjC,OAAO,CAAC,aAAa,CAAS;gBAElB,OAAO,GAAE,cAAmB;IA4BxC;;OAEG;IACH,SAAS,IAAI,cAAc;IAI3B;;;OAGG;YACW,cAAc;IA8C5B;;;OAGG;YACW,yBAAyB;IAmKvC;;OAEG;IACH,cAAc,IAAI,WAAW,GAAG,IAAI;IAIpC;;OAEG;IACG,KAAK,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAsVxD;;;OAGG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IA4DjC;;OAEG;IACH,OAAO,CAAC,YAAY;IAYpB;;OAEG;IACG,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBhC;;;OAGG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAyB5B;;OAEG;IACH,OAAO,CAAC,gBAAgB;IAsBxB;;OAEG;YACW,iBAAiB;IA0G/B;;;OAGG;YACW,kBAAkB;IAsDhC;;;OAGG;YACW,sBAAsB;IA8FpC;;;;OAIG;YACW,kBAAkB;IAsBhC;;OAEG;IACH,cAAc,IAAI,OAAO;IAIzB;;;OAGG;IACG,MAAM,CACX,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;QACR,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,cAAc,CAAC,EAAE,OAAO,CAAA;QACxB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;QACzB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,YAAY,CAAC,EAAE,MAAM,EAAE,CAAA;QAEvB,YAAY,CAAC,EAAE,MAAM,CAAA;QACrB,eAAe,CAAC,EAAE,MAAM,CAAA;QACxB,gBAAgB,CAAC,EAAE,MAAM,CAAA;KACpB,GACJ,OAAO,CAAC,YAAY,EAAE,CAAC;IAgG1B;;;OAGG;YACW,YAAY;IAiH1B;;;;;OAKG;IACH,OAAO,CAAC,cAAc;IAgHtB;;OAEG;IACG,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAK9D;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,MAAM,CAAC;IAIxC;;OAEG;IACH,gBAAgB,IAAI,aAAa,GAAG,SAAS;IAI7C;;OAEG;IACH,oBAAoB,IAAI,iBAAiB,GAAG,SAAS;IAIrD;;;OAGG;YACW,4BAA4B;IA2F1C;;;OAGG;YACW,gBAAgB;IA8C9B;;OAEG;YACW,gBAAgB;CAiB9B;AAED,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,EAAE,CAAA;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB"}