@hanhnd/agent-kit 1.0.28 → 1.0.30

package/dist/memory/__tests__/chunker.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/memory/__tests__/chunker.test.js

@@ -0,0 +1,105 @@
+import * as assert from 'node:assert/strict';
+import { describe, test } from 'node:test';
+import { chunkMarkdown, cleanContentForEmbedding, computeChunkId } from '../chunker.js';
+const CFG = { chunkSize: 200, overlapLines: 2 };
+const SRC = 'test.md';
+describe('chunkMarkdown', () => {
+    test('returns [] for empty string', () => {
+        assert.deepEqual(chunkMarkdown('', SRC, CFG), []);
+    });
+    test('returns [] for whitespace-only string', () => {
+        assert.deepEqual(chunkMarkdown('   \n\n  ', SRC, CFG), []);
+    });
+    test('single heading + short body produces one chunk with correct metadata', () => {
+        const text = '# My Heading\nThis is the body text.';
+        const chunks = chunkMarkdown(text, SRC, CFG);
+        assert.equal(chunks.length, 1);
+        const [c] = chunks;
+        assert.equal(c.heading, 'My Heading');
+        assert.equal(c.headingLevel, 1);
+        assert.equal(c.source, SRC);
+        assert.ok(c.lineStart >= 1);
+        assert.ok(c.lineEnd >= c.lineStart);
+    });
+    test('chunk id is deterministic — same content produces same id', () => {
+        const text = '# Section\nHello world.';
+        const [a] = chunkMarkdown(text, SRC, CFG);
+        const [b] = chunkMarkdown(text, 'other-source.md', CFG);
+        assert.equal(a.id, b.id, 'id must depend on content only, not source');
+    });
+    test('chunk id changes when content changes', () => {
+        const [a] = chunkMarkdown('# H\nVersion A', SRC, CFG);
+        const [b] = chunkMarkdown('# H\nVersion B', SRC, CFG);
+        assert.notEqual(a.id, b.id);
+    });
+    test('body exceeding chunkSize produces multiple chunks', () => {
+        const word = 'a'.repeat(50);
+        const body = Array.from({ length: 10 }, (_, i) => `Paragraph ${i}: ${word}`).join('\n\n');
+        const text = `# Big Section\n${body}`;
+        const chunks = chunkMarkdown(text, SRC, { chunkSize: 100, overlapLines: 0 });
+        assert.ok(chunks.length > 1, `Expected >1 chunks, got ${chunks.length}`);
+        for (const c of chunks) {
+            assert.equal(c.heading, 'Big Section');
+        }
+    });
+    test('multiple headings produce separate chunks with correct headingLevel', () => {
+        const text = [
+            '# Top Level',
+            'Top content.',
+            '## Sub Level',
+            'Sub content.',
+            '### Deep Level',
+            'Deep content.',
+        ].join('\n');
+        const chunks = chunkMarkdown(text, SRC, CFG);
+        const levels = chunks.map((c) => c.headingLevel);
+        assert.ok(levels.includes(1), 'Expected headingLevel 1');
+        assert.ok(levels.includes(2), 'Expected headingLevel 2');
+        assert.ok(levels.includes(3), 'Expected headingLevel 3');
+    });
+    test('HTML comment is stripped from chunk content', () => {
+        const text = '# Section\nVisible text <!-- hidden comment --> more visible text.';
+        const [chunk] = chunkMarkdown(text, SRC, CFG);
+        assert.ok(!chunk.content.includes('hidden comment'), 'HTML comment must be stripped');
+        assert.ok(chunk.content.includes('Visible text'), 'Visible text must remain');
+    });
+    test('multiline HTML comment is stripped', () => {
+        const text = '# Section\nBefore.\n<!-- multi\nline\ncomment -->\nAfter.';
+        const [chunk] = chunkMarkdown(text, SRC, CFG);
+        assert.ok(!chunk.content.includes('multi'), 'Multiline comment must be stripped');
+        assert.ok(chunk.content.includes('Before.'), 'Text before comment must remain');
+    });
+});
+describe('computeChunkId', () => {
+    test('returns 16-character hex string', () => {
+        const id = computeChunkId('hello world');
+        assert.equal(id.length, 16);
+        assert.match(id, /^[0-9a-f]{16}$/);
+    });
+    test('same content always produces same id', () => {
+        assert.equal(computeChunkId('abc'), computeChunkId('abc'));
+    });
+    test('different content produces different id', () => {
+        assert.notEqual(computeChunkId('abc'), computeChunkId('xyz'));
+    });
+});
+describe('cleanContentForEmbedding', () => {
+    test('strips HTML comment', () => {
+        assert.equal(cleanContentForEmbedding('hello <!-- comment --> world'), 'hello  world');
+    });
+    test('strips multiline HTML comment', () => {
+        const input = 'before <!-- line1\nline2 --> after';
+        const result = cleanContentForEmbedding(input);
+        assert.ok(!result.includes('line1'));
+        assert.ok(result.includes('before'));
+        assert.ok(result.includes('after'));
+    });
+    test('returns original string when no comments present', () => {
+        const text = 'plain text without comments';
+        assert.equal(cleanContentForEmbedding(text), text);
+    });
+    test('preserves fenced code block content', () => {
+        const text = '```js\nconst x = 1;\n```';
+        assert.equal(cleanContentForEmbedding(text), text);
+    });
+});

package/dist/memory/__tests__/indexer.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/memory/__tests__/indexer.test.js

@@ -0,0 +1,392 @@
+import * as assert from 'node:assert/strict';
+import fsDefault from 'node:fs';
+import * as fs from 'node:fs';
+import { syncBuiltinESMExports } from 'node:module';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { after, before, describe, test } from 'node:test';
+import { MemoryIndexer } from '../indexer.js';
+import { MemoryStore } from '../store.js';
+// Stub embedder — returns deterministic non-zero vectors without loading any model
+class StubEmbedder {
+    async embed(texts) {
+        return texts.map(() => new Float32Array(384).fill(0.05));
+    }
+    get dimension() {
+        return 384;
+    }
+    isReady() {
+        return true;
+    }
+}
+function makeConfig(wikiDir) {
+    return {
+        enabled: true,
+        wikiDir,
+        topK: 5,
+        chunkSize: 1500,
+        overlapLines: 2,
+        embeddingModel: 'Xenova/bge-small-en-v1.5',
+        vectorDimension: 384,
+    };
+}
+describe('MemoryIndexer', () => {
+    let tmpDir;
+    let store;
+    let indexer;
+    let config;
+    before(() => {
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-indexer-test-'));
+        config = makeConfig(tmpDir);
+        store = new MemoryStore(path.join(config.wikiDir, 'index.db'), config);
+        indexer = new MemoryIndexer(store, new StubEmbedder(), config);
+    });
+    after(() => {
+        store.close();
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+    });
+    test('indexFile on new file — indexed > 0, skipped === 0', async () => {
+        const filePath = path.join(tmpDir, 'new-file.md');
+        fs.writeFileSync(filePath, '# New File\nThis file has some content for indexing.', 'utf8');
+        const stats = await indexer.indexFile(filePath);
+        assert.ok(stats.indexed > 0, `Expected indexed > 0, got ${stats.indexed}`);
+        assert.equal(stats.skipped, 0);
+    });
+    test('indexFile on unchanged file — indexed === 0, skipped > 0', async () => {
+        const filePath = path.join(tmpDir, 'stable-file.md');
+        fs.writeFileSync(filePath, '# Stable\nThis content does not change between runs.', 'utf8');
+        // First run indexes it
+        await indexer.indexFile(filePath);
+        // Second run — same content
+        const stats = await indexer.indexFile(filePath);
+        assert.equal(stats.indexed, 0, `Expected indexed === 0, got ${stats.indexed}`);
+        assert.ok(stats.skipped > 0, `Expected skipped > 0, got ${stats.skipped}`);
+    });
+    test('indexFile after modification — only changed chunks re-indexed', async () => {
+        const filePath = path.join(tmpDir, 'modified-file.md');
+        fs.writeFileSync(filePath, '# Modified\nOriginal content.', 'utf8');
+        await indexer.indexFile(filePath);
+        fs.writeFileSync(filePath, '# Modified\nUpdated content that changed completely.', 'utf8');
+        const stats = await indexer.indexFile(filePath);
+        assert.ok(stats.indexed > 0, `Expected re-indexed chunks after modification`);
+    });
+    test('indexDirectory removes stale source when file is deleted', async () => {
+        const staleFile = path.join(tmpDir, 'stale-file.md');
+        fs.writeFileSync(staleFile, '# Stale\nThis file will be deleted.', 'utf8');
+        await indexer.indexFile(staleFile);
+        const staleSource = path.relative(tmpDir, staleFile);
+        const before = store.hashesBySource(staleSource);
+        assert.ok(before.size > 0, 'Stale file must be indexed first');
+        // Delete the file and re-index the directory
+        fs.unlinkSync(staleFile);
+        await indexer.indexDirectory(tmpDir);
+        const afterDeletion = store.hashesBySource(staleSource);
+        assert.equal(afterDeletion.size, 0, 'Stale source must be removed from store after directory scan');
+    });
+    test('search returns result with correct source for indexed content', async () => {
+        const filePath = path.join(config.wikiDir, 'compiled', 'searchable.md');
+        const fileContent = '# Searchable\nspecialUniqueTermForSearch is in this document.';
+        fs.mkdirSync(path.dirname(filePath), { recursive: true });
+        fs.writeFileSync(filePath, fileContent, 'utf8');
+        await indexer.indexDirectory(path.join(config.wikiDir, 'compiled'), {
+            relativeBase: config.wikiDir,
+        });
+        const results = await indexer.search('specialUniqueTermForSearch', 5);
+        assert.ok(results.length > 0, 'Expected at least one search result');
+        const expectedSource = path.relative(config.wikiDir, filePath);
+        const match = results.find((r) => r.chunk.source === expectedSource);
+        assert.ok(match, `Expected result with source=${expectedSource}, got: ${results.map((r) => r.chunk.source).join(', ')}`);
+        assert.equal(match.chunk.content, fileContent);
+        assert.equal(match.contentSource, 'file');
+    });
+    test('indexDirectory walks nested markdown files and excludes configured basenames', async () => {
+        const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'recursive-index-'));
+        const testCfg = makeConfig(path.join(testDir, 'wiki'));
+        const testStore = new MemoryStore(path.join(testCfg.wikiDir, 'index.db'), testCfg);
+        const testIndexer = new MemoryIndexer(testStore, new StubEmbedder(), testCfg);
+        const compiledDir = path.join(testCfg.wikiDir, 'compiled');
+        try {
+            fs.mkdirSync(path.join(compiledDir, 'entities'), { recursive: true });
+            fs.writeFileSync(path.join(compiledDir, 'entities', 'foo.md'), '# Foo\nrecursiveUniqueTerm', 'utf8');
+            fs.writeFileSync(path.join(compiledDir, 'entities', 'index.md'), '# Index\nskip me', 'utf8');
+            fs.writeFileSync(path.join(compiledDir, 'log.md'), '# Log\nskip me', 'utf8');
+            fs.writeFileSync(path.join(compiledDir, 'entities', 'notes.txt'), 'skip me', 'utf8');
+            const stats = await testIndexer.indexDirectory(compiledDir, {
+                relativeBase: testCfg.wikiDir,
+                excludeFiles: ['index.md', 'log.md'],
+            });
+            assert.ok(stats.indexed > 0, `Expected indexed > 0, got ${stats.indexed}`);
+            assert.ok(testStore.hashesBySource('compiled/entities/foo.md').size > 0);
+            assert.equal(testStore.hashesBySource('compiled/entities/index.md').size, 0);
+            assert.equal(testStore.hashesBySource('compiled/log.md').size, 0);
+            assert.equal(testStore.hashesBySource('compiled/entities/notes.txt').size, 0);
+        }
+        finally {
+            testStore.close();
+            fs.rmSync(testDir, { recursive: true, force: true });
+        }
+    });
+    test('indexDirectory returns zero stats when root directory is missing', async () => {
+        const stats = await indexer.indexDirectory(path.join(config.wikiDir, 'missing'), {
+            relativeBase: config.wikiDir,
+        });
+        assert.deepEqual(stats, { indexed: 0, deleted: 0, skipped: 0 });
+    });
+    test('indexDirectory removes stale pre-migration daily-file sources', async () => {
+        const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'recursive-stale-'));
+        const testCfg = makeConfig(path.join(testDir, 'wiki'));
+        const testStore = new MemoryStore(path.join(testCfg.wikiDir, 'index.db'), testCfg);
+        const testIndexer = new MemoryIndexer(testStore, new StubEmbedder(), testCfg);
+        const compiledDir = path.join(testCfg.wikiDir, 'compiled');
+        try {
+            fs.mkdirSync(compiledDir, { recursive: true });
+            testStore.upsert([{
+                    id: 'stale-daily-file-0001',
+                    source: '2026-05-18.md',
+                    heading: 'Stale',
+                    headingLevel: 1,
+                    content: 'pre migration content',
+                    lineStart: 1,
+                    lineEnd: 2,
+                }], [new Float32Array(384)]);
+            assert.ok(testStore.hashesBySource('2026-05-18.md').size > 0);
+            const stats = await testIndexer.indexDirectory(compiledDir, {
+                relativeBase: testCfg.wikiDir,
+            });
+            assert.equal(stats.deleted, 1);
+            assert.equal(testStore.hashesBySource('2026-05-18.md').size, 0);
+        }
+        finally {
+            testStore.close();
+            fs.rmSync(testDir, { recursive: true, force: true });
+        }
+    });
+    test('search deduplicates sources and reads each matched source once', async () => {
+        const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'search-dedup-'));
+        const testCfg = {
+            ...makeConfig(path.join(testDir, 'wiki')),
+            chunkSize: 40,
+            overlapLines: 0,
+        };
+        const testStore = new MemoryStore(path.join(testCfg.wikiDir, 'index.db'), testCfg);
+        const testIndexer = new MemoryIndexer(testStore, new StubEmbedder(), testCfg);
+        const filePath = path.join(testCfg.wikiDir, 'compiled', 'entities', 'dedup.md');
+        const fileContent = [
+            '# Dedup',
+            'dedupUniqueTerm first chunk text',
+            'dedupUniqueTerm second chunk text',
+            'dedupUniqueTerm third chunk text',
+        ].join('\n');
+        const originalReadFileSync = fsDefault.readFileSync;
+        let readCount = 0;
+        try {
+            fs.mkdirSync(path.dirname(filePath), { recursive: true });
+            fs.writeFileSync(filePath, fileContent, 'utf8');
+            await testIndexer.indexDirectory(path.join(testCfg.wikiDir, 'compiled'), {
+                relativeBase: testCfg.wikiDir,
+            });
+            const expectedPath = path.join(testCfg.wikiDir, 'compiled/entities/dedup.md');
+            fsDefault.readFileSync = ((targetPath, options) => {
+                if (targetPath === expectedPath)
+                    readCount += 1;
+                return originalReadFileSync(targetPath, options);
+            });
+            syncBuiltinESMExports();
+            const results = await testIndexer.search('dedupUniqueTerm', 5);
+            assert.equal(results.filter((r) => r.chunk.source === 'compiled/entities/dedup.md').length, 1);
+            assert.equal(readCount, 1);
+            assert.equal(results[0].chunk.content, fileContent);
+            assert.equal(results[0].contentSource, 'file');
+        }
+        finally {
+            fsDefault.readFileSync = originalReadFileSync;
+            syncBuiltinESMExports();
+            testStore.close();
+            fs.rmSync(testDir, { recursive: true, force: true });
+        }
+    });
+    test('search continues past duplicate sources until topK unique sources are returned', async () => {
+        const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'search-unique-topk-'));
+        const testCfg = makeConfig(path.join(testDir, 'wiki'));
+        const firstPath = path.join(testCfg.wikiDir, 'compiled', 'entities', 'first.md');
+        const secondPath = path.join(testCfg.wikiDir, 'compiled', 'entities', 'second.md');
+        const firstContent = '# First\nfirst source full content';
+        const secondContent = '# Second\nsecond source full content';
+        const chunks = [
+            {
+                id: 'first-1',
+                source: 'compiled/entities/first.md',
+                heading: 'First',
+                headingLevel: 1,
+                content: 'first matching chunk one',
+                lineStart: 1,
+                lineEnd: 2,
+            },
+            {
+                id: 'first-2',
+                source: 'compiled/entities/first.md',
+                heading: 'First',
+                headingLevel: 1,
+                content: 'first matching chunk two',
+                lineStart: 3,
+                lineEnd: 4,
+            },
+            {
+                id: 'second-1',
+                source: 'compiled/entities/second.md',
+                heading: 'Second',
+                headingLevel: 1,
+                content: 'second matching chunk',
+                lineStart: 1,
+                lineEnd: 2,
+            },
+        ];
+        const fakeStore = {
+            vecAvailable: false,
+            searchBm25: () => [
+                { id: 'first-1', score: 1 },
+                { id: 'first-2', score: 0.9 },
+                { id: 'second-1', score: 0.8 },
+            ],
+            getChunksByIds: (ids) => chunks.filter((chunk) => ids.includes(chunk.id)),
+        };
+        const testIndexer = new MemoryIndexer(fakeStore, new StubEmbedder(), testCfg);
+        try {
+            fs.mkdirSync(path.dirname(firstPath), { recursive: true });
+            fs.writeFileSync(firstPath, firstContent, 'utf8');
+            fs.writeFileSync(secondPath, secondContent, 'utf8');
+            const results = await testIndexer.search('duplicate source query', 2);
+            assert.equal(results.length, 2);
+            assert.deepEqual(results.map((result) => result.chunk.source), [
+                'compiled/entities/first.md',
+                'compiled/entities/second.md',
+            ]);
+            assert.equal(results[0].chunk.content, firstContent);
+            assert.equal(results[1].chunk.content, secondContent);
+        }
+        finally {
+            fs.rmSync(testDir, { recursive: true, force: true });
+        }
+    });
+    test('search does not include dense-only results when BM25 has matches', async () => {
+        const testCfg = makeConfig('/tmp/search-dense-filter');
+        const chunks = [
+            {
+                id: 'preference-1',
+                source: 'compiled/preferences.md',
+                heading: '',
+                headingLevel: 0,
+                content: 'I like fish',
+                lineStart: 1,
+                lineEnd: 1,
+            },
+            {
+                id: 'dense-only-1',
+                source: 'compiled/entities/worktree.md',
+                heading: 'Worktree',
+                headingLevel: 1,
+                content: 'Unrelated worktree lifecycle content',
+                lineStart: 1,
+                lineEnd: 2,
+            },
+        ];
+        const fakeStore = {
+            vecAvailable: true,
+            searchDense: () => [
+                { id: 'dense-only-1', score: 0.99 },
+                { id: 'preference-1', score: 0.98 },
+            ],
+            searchBm25: () => [{ id: 'preference-1', score: 1 }],
+            getChunksByIds: (ids) => chunks.filter((chunk) => ids.includes(chunk.id)),
+        };
+        const testIndexer = new MemoryIndexer(fakeStore, new StubEmbedder(), testCfg);
+        const results = await testIndexer.search('personal likes and preferences of the user', 5);
+        assert.deepEqual(results.map((result) => result.chunk.id), ['preference-1']);
+        assert.equal(results[0].retriever, 'both');
+    });
+    test('search still returns dense-only results when BM25 has no matches', async () => {
+        const testCfg = makeConfig('/tmp/search-dense-fallback');
+        const chunks = [
+            {
+                id: 'semantic-1',
+                source: 'compiled/preferences.md',
+                heading: '',
+                headingLevel: 0,
+                content: 'I like fish',
+                lineStart: 1,
+                lineEnd: 1,
+            },
+        ];
+        const fakeStore = {
+            vecAvailable: true,
+            searchDense: () => [{ id: 'semantic-1', score: 0.99 }],
+            searchBm25: () => [],
+            getChunksByIds: (ids) => chunks.filter((chunk) => ids.includes(chunk.id)),
+        };
+        const testIndexer = new MemoryIndexer(fakeStore, new StubEmbedder(), testCfg);
+        const results = await testIndexer.search('favorite dish', 5);
+        assert.deepEqual(results.map((result) => result.chunk.id), ['semantic-1']);
+        assert.equal(results[0].retriever, 'dense');
+    });
+    test('search falls back to stored chunk content when source file is missing', async () => {
+        const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'search-fallback-'));
+        const testCfg = makeConfig(path.join(testDir, 'wiki'));
+        const testStore = new MemoryStore(path.join(testCfg.wikiDir, 'index.db'), testCfg);
+        const testIndexer = new MemoryIndexer(testStore, new StubEmbedder(), testCfg);
+        const filePath = path.join(testCfg.wikiDir, 'compiled', 'entities', 'missing.md');
+        const fileContent = '# Missing\nfallbackUniqueTerm stored chunk text';
+        try {
+            fs.mkdirSync(path.dirname(filePath), { recursive: true });
+            fs.writeFileSync(filePath, fileContent, 'utf8');
+            await testIndexer.indexDirectory(path.join(testCfg.wikiDir, 'compiled'), {
+                relativeBase: testCfg.wikiDir,
+            });
+            fs.unlinkSync(filePath);
+            const results = await testIndexer.search('fallbackUniqueTerm', 5);
+            assert.equal(results.length, 1);
+            assert.equal(results[0].contentSource, 'fallback');
+            assert.equal(results[0].chunk.content, fileContent);
+        }
+        finally {
+            testStore.close();
+            fs.rmSync(testDir, { recursive: true, force: true });
+        }
+    });
+    test('search on empty store returns []', async () => {
+        const emptyDir = fs.mkdtempSync(path.join(os.tmpdir(), 'empty-store-'));
+        const emptyCfg = makeConfig(emptyDir);
+        const emptyStore = new MemoryStore(path.join(emptyCfg.wikiDir, 'index.db'), emptyCfg);
+        const emptyIndexer = new MemoryIndexer(emptyStore, new StubEmbedder(), emptyCfg);
+        try {
+            const results = await emptyIndexer.search('anything', 5);
+            assert.deepEqual(results, []);
+        }
+        finally {
+            emptyStore.close();
+            fs.rmSync(emptyDir, { recursive: true, force: true });
+        }
+    });
+    test('save appends manual content to daily wiki raw save file without indexing it', async () => {
+        const testDir = fs.mkdtempSync(path.join(os.tmpdir(), 'save-daily-'));
+        const testCfg = makeConfig(path.join(testDir, 'wiki'));
+        const testStore = new MemoryStore(path.join(testCfg.wikiDir, 'index.db'), testCfg);
+        const testIndexer = new MemoryIndexer(testStore, new StubEmbedder(), testCfg);
+        const datePart = new Date().toISOString().slice(0, 10);
+        const savePath = path.join(testCfg.wikiDir, 'raw', `conv_save_${datePart}.md`);
+        try {
+            const firstStats = await testIndexer.save('first manual save');
+            const secondStats = await testIndexer.save('second manual save');
+            assert.deepEqual(firstStats, { indexed: 0, deleted: 0, skipped: 0 });
+            assert.deepEqual(secondStats, { indexed: 0, deleted: 0, skipped: 0 });
+            assert.equal(fs.existsSync(savePath), true);
+            const content = fs.readFileSync(savePath, 'utf8');
+            assert.match(content, /first manual save/);
+            assert.match(content, /second manual save/);
+            assert.equal(fs.readdirSync(path.dirname(savePath)).filter((name) => /^conv_save_.*\.md$/.test(name)).length, 1);
+        }
+        finally {
+            testStore.close();
+            fs.rmSync(testDir, { recursive: true, force: true });
+        }
+    });
+});

package/dist/memory/__tests__/store.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/memory/__tests__/store.test.js

@@ -0,0 +1,152 @@
+import * as assert from 'node:assert/strict';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
+import { after, before, describe, test } from 'node:test';
+import { MemoryStore } from '../store.js';
+const TEST_CONFIG = {
+    enabled: true,
+    wikiDir: '',
+    topK: 5,
+    chunkSize: 1500,
+    overlapLines: 2,
+    embeddingModel: 'Xenova/bge-small-en-v1.5',
+    vectorDimension: 384,
+};
+function makeChunk(overrides = {}) {
+    return {
+        id: 'test-id-0000001',
+        source: 'test.md',
+        heading: 'Test Heading',
+        headingLevel: 1,
+        content: 'This is test content for BM25 search.',
+        lineStart: 1,
+        lineEnd: 5,
+        ...overrides,
+    };
+}
+describe('MemoryStore', () => {
+    let tmpDir;
+    let dbPath;
+    let store;
+    before(() => {
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-store-test-'));
+        dbPath = path.join(tmpDir, 'index.db');
+        store = new MemoryStore(dbPath, TEST_CONFIG);
+    });
+    after(() => {
+        store.close();
+        fs.rmSync(tmpDir, { recursive: true, force: true });
+    });
+    test('vecAvailable is a boolean', () => {
+        assert.equal(typeof store.vecAvailable, 'boolean');
+    });
+    test('hashesBySource returns empty set for unknown source', () => {
+        const hashes = store.hashesBySource('nonexistent.md');
+        assert.ok(hashes instanceof Set);
+        assert.equal(hashes.size, 0);
+    });
+    test('upsert stores chunks and hashesBySource returns their ids', () => {
+        const chunk = makeChunk({ id: 'upsert-test-0001', source: 'upsert.md' });
+        const embedding = new Float32Array(384).fill(0.1);
+        store.upsert([chunk], [embedding]);
+        const hashes = store.hashesBySource('upsert.md');
+        assert.ok(hashes.has('upsert-test-0001'));
+    });
+    test('deleteBySource removes all chunks for that source', () => {
+        const c1 = makeChunk({ id: 'del-src-0001', source: 'deleteme.md', content: 'delete source chunk 1' });
+        const c2 = makeChunk({ id: 'del-src-0002', source: 'deleteme.md', content: 'delete source chunk 2' });
+        store.upsert([c1, c2], [new Float32Array(384), new Float32Array(384)]);
+        store.deleteBySource('deleteme.md');
+        const hashes = store.hashesBySource('deleteme.md');
+        assert.equal(hashes.size, 0);
+    });
+    test('searchBm25 returns matching result after upsert', () => {
+        const chunk = makeChunk({
+            id: 'bm25-search-0001',
+            source: 'bm25.md',
+            content: 'uniqueKeywordXYZ for BM25 testing',
+        });
+        store.upsert([chunk], [new Float32Array(384)]);
+        const results = store.searchBm25('uniqueKeywordXYZ', 5);
+        assert.ok(results.length > 0, 'Expected at least one BM25 result');
+        assert.ok(results.some((r) => r.id === 'bm25-search-0001'), `Expected chunk id in results, got: ${results.map((r) => r.id).join(', ')}`);
+    });
+    test('searchBm25 ignores filler words and matches preference source/content terms', () => {
+        const preference = makeChunk({
+            id: 'preference-search-0001',
+            source: 'compiled/preferences.md',
+            heading: '',
+            headingLevel: 0,
+            content: 'I like fish',
+        });
+        const unrelated = makeChunk({
+            id: 'preference-search-0002',
+            source: 'compiled/entities/worktree.md',
+            heading: 'Open Questions',
+            content: 'How should git manage worktree lifecycle decisions?',
+        });
+        store.upsert([preference, unrelated], [new Float32Array(384), new Float32Array(384)]);
+        const results = store.searchBm25('personal likes and preferences of the user', 5);
+        assert.ok(results.length > 0, 'Expected preference query to return results');
+        assert.equal(results[0].id, 'preference-search-0001');
+    });
+    test('searchBm25 returns empty array for empty query', () => {
+        const results = store.searchBm25('   ', 5);
+        assert.deepEqual(results, []);
+    });
+    test('getChunksByIds returns correct metadata for stored chunk', () => {
+        const chunk = makeChunk({
+            id: 'get-by-ids-0001',
+            source: 'metadata.md',
+            heading: 'Metadata Section',
+            headingLevel: 2,
+            content: 'Content for metadata test',
+            lineStart: 10,
+            lineEnd: 20,
+        });
+        store.upsert([chunk], [new Float32Array(384)]);
+        const results = store.getChunksByIds(['get-by-ids-0001']);
+        assert.equal(results.length, 1);
+        const [r] = results;
+        assert.equal(r.id, 'get-by-ids-0001');
+        assert.equal(r.source, 'metadata.md');
+        assert.equal(r.heading, 'Metadata Section');
+        assert.equal(r.headingLevel, 2);
+        assert.equal(r.lineStart, 10);
+        assert.equal(r.lineEnd, 20);
+    });
+    test('getChunksByIds returns only found rows for mixed ids', () => {
+        const chunk = makeChunk({ id: 'partial-found-0001', source: 'partial.md', content: 'partial' });
+        store.upsert([chunk], [new Float32Array(384)]);
+        const results = store.getChunksByIds(['partial-found-0001', 'does-not-exist-999']);
+        assert.equal(results.length, 1);
+        assert.equal(results[0].id, 'partial-found-0001');
+    });
+    test('indexedSources includes source after upsert', () => {
+        const chunk = makeChunk({ id: 'indexed-src-0001', source: 'indexed-source.md', content: 'indexed' });
+        store.upsert([chunk], [new Float32Array(384)]);
+        const sources = store.indexedSources();
+        assert.ok(sources.includes('indexed-source.md'), `Expected 'indexed-source.md' in ${sources.join(', ')}`);
+    });
+    test('deleteByIds removes specific chunks', () => {
+        const c1 = makeChunk({ id: 'del-ids-0001', source: 'del-ids.md', content: 'delete by id 1' });
+        const c2 = makeChunk({ id: 'del-ids-0002', source: 'del-ids.md', content: 'delete by id 2' });
+        store.upsert([c1, c2], [new Float32Array(384), new Float32Array(384)]);
+        store.deleteByIds(['del-ids-0001']);
+        const hashes = store.hashesBySource('del-ids.md');
+        assert.ok(!hashes.has('del-ids-0001'), 'Deleted chunk id must be gone');
+        assert.ok(hashes.has('del-ids-0002'), 'Non-deleted chunk id must remain');
+    });
+    test('searchBm25 still works regardless of vecAvailable', () => {
+        // This verifies FTS5 degraded mode is always functional
+        const chunk = makeChunk({
+            id: 'fts5-degraded-0001',
+            source: 'fts5.md',
+            content: 'degradedModeTest keyword',
+        });
+        store.upsert([chunk], [new Float32Array(384)]);
+        const results = store.searchBm25('degradedModeTest', 5);
+        assert.ok(results.length > 0, 'FTS5 search must work regardless of vecAvailable');
+    });
+});

package/dist/memory/chunker.d.ts

@@ -0,0 +1,4 @@
+import type { MemoryChunk, MemoryConfig } from './types.js';
+export declare function computeChunkId(content: string): string;
+export declare function cleanContentForEmbedding(text: string): string;
+export declare function chunkMarkdown(text: string, source: string, config: Pick<MemoryConfig, 'chunkSize' | 'overlapLines'>): MemoryChunk[];