@hanhnd/agent-kit 1.0.35 → 1.0.37

package/dist/services/memory/store.test.js

@@ -1,8 +1,8 @@
+import Database from 'libsql';
 import * as assert from 'node:assert/strict';
 import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
-import Database from 'better-sqlite3';
 import { after, before, describe, test } from 'node:test';
 import { MemoryStore, SCHEMA_VERSION } from './store.js';
 import { EmbeddingModelName } from './types.js';
@@ -29,6 +29,25 @@ function makeChunk(overrides = {}) {
         ...overrides,
     };
 }
+function makeEmbedding(activeIndex, dimension = TEST_CONFIG.vectorDimension) {
+    const embedding = new Float32Array(dimension);
+    embedding[activeIndex] = 1;
+    return embedding;
+}
+function readUserVersion(db) {
+    const value = db.pragma('user_version', { simple: true });
+    if (typeof value === 'number')
+        return value;
+    if (Array.isArray(value)) {
+        const [row] = value;
+        return typeof row?.user_version === 'number' ? row.user_version : 0;
+    }
+    if (value && typeof value === 'object' && 'user_version' in value) {
+        const row = value;
+        return typeof row.user_version === 'number' ? row.user_version : 0;
+    }
+    return 0;
+}
 describe('MemoryStore', () => {
     let tmpDir;
     let dbPath;
@@ -42,8 +61,50 @@ describe('MemoryStore', () => {
         store.close();
         fs.rmSync(tmpDir, { recursive: true, force: true });
     });
-    test('vecAvailable is a boolean', () => {
-        assert.equal(typeof store.vecAvailable, 'boolean');
+    test('libsql supports required local driver and vector features', () => {
+        const libsqlDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-libsql-compat-'));
+        const libsqlDb = new Database(path.join(libsqlDir, 'index.db'));
+        try {
+            assert.equal(readUserVersion(libsqlDb), 0);
+            libsqlDb.pragma('journal_mode = WAL');
+            libsqlDb.pragma('busy_timeout = 5000');
+            libsqlDb.pragma(`user_version = ${SCHEMA_VERSION}`);
+            assert.equal(readUserVersion(libsqlDb), SCHEMA_VERSION);
+            libsqlDb.exec(`
+        CREATE TABLE vector_items (
+          rowid INTEGER PRIMARY KEY AUTOINCREMENT,
+          name TEXT NOT NULL,
+          embedding F32_BLOB(3) NOT NULL
+        );
+        CREATE INDEX idx_vector_items_embedding
+          ON vector_items (libsql_vector_idx(embedding));
+      `);
+            const insert = libsqlDb.prepare(`INSERT INTO vector_items (name, embedding) VALUES (?, vector32(?))`);
+            const info = insert.run('alpha', '[1,0,0]');
+            assert.equal(info.lastInsertRowid, 1);
+            insert.run('beta', '[0,1,0]');
+            const insertGamma = libsqlDb.transaction(() => insert.run('gamma', '[0,0,1]'));
+            insertGamma();
+            const distance = libsqlDb
+                .prepare(`SELECT vector_distance_cos(embedding, vector32(?)) AS distance FROM vector_items WHERE name = ?`)
+                .get('[1,0,0]', 'alpha');
+            assert.equal(distance.distance, 0);
+            const rows = libsqlDb
+                .prepare(`SELECT vector_items.name
+           FROM vector_top_k('idx_vector_items_embedding', vector32(?), ?) AS vector_matches
+           JOIN vector_items ON vector_items.rowid = vector_matches.id
+           ORDER BY vector_distance_cos(vector_items.embedding, vector32(?))`)
+                .all('[1,0,0]', 2, '[1,0,0]');
+            assert.equal(rows[0].name, 'alpha');
+            assert.equal(rows.length, 2);
+        }
+        finally {
+            libsqlDb.close();
+            fs.rmSync(libsqlDir, { recursive: true, force: true });
+        }
+    });
+    test('vecAvailable is true after schema creation', () => {
+        assert.equal(store.vecAvailable, true);
     });
     test('hashesBySource returns empty set for unknown source', () => {
         const hashes = store.hashesBySource('nonexistent.md');
@@ -52,7 +113,7 @@ describe('MemoryStore', () => {
     });
     test('upsert stores chunks and hashesBySource returns their ids', () => {
         const chunk = makeChunk({ id: 'upsert-test-0001', source: 'upsert.md', sourceType: 'entity', fileMtimeAt: 2000 });
-        const embedding = new Float32Array(384).fill(0.1);
+        const embedding = makeEmbedding(0);
         store.upsert([chunk], [embedding]);
         const hashes = store.hashesBySource('upsert.md');
         assert.ok(hashes.has('upsert-test-0001'));
@@ -71,7 +132,7 @@ describe('MemoryStore', () => {
             source: 'deleteme.md',
             content: 'delete source chunk 2',
         });
-        store.upsert([c1, c2], [new Float32Array(384), new Float32Array(384)]);
+        store.upsert([c1, c2], [makeEmbedding(1), makeEmbedding(2)]);
         store.deleteBySource('deleteme.md');
         const hashes = store.hashesBySource('deleteme.md');
         assert.equal(hashes.size, 0);
@@ -82,7 +143,7 @@ describe('MemoryStore', () => {
             source: 'bm25.md',
             content: 'uniqueKeywordXYZ for BM25 testing',
         });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(3)]);
         const results = store.searchBm25('uniqueKeywordXYZ', 5);
         assert.ok(results.length > 0, 'Expected at least one BM25 result');
         assert.ok(results.some((r) => r.id === 'bm25-search-0001'), `Expected chunk id in results, got: ${results.map((r) => r.id).join(', ')}`);
@@ -101,7 +162,7 @@ describe('MemoryStore', () => {
             heading: 'Open Questions',
             content: 'How should git manage worktree lifecycle decisions?',
         });
-        store.upsert([preference, unrelated], [new Float32Array(384), new Float32Array(384)]);
+        store.upsert([preference, unrelated], [makeEmbedding(4), makeEmbedding(5)]);
         const results = store.searchBm25('personal likes and preferences of the user', 5);
         assert.ok(results.length > 0, 'Expected preference query to return results');
         assert.equal(results[0].id, 'preference-search-0001');
@@ -122,7 +183,7 @@ describe('MemoryStore', () => {
             lineEnd: 20,
             fileMtimeAt: 3000,
         });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(6)]);
         const results = store.getChunksByIds(['get-by-ids-0001']);
         assert.equal(results.length, 1);
         const [r] = results;
@@ -137,7 +198,7 @@ describe('MemoryStore', () => {
     });
     test('getChunksByIds returns only found rows for mixed ids', () => {
         const chunk = makeChunk({ id: 'partial-found-0001', source: 'partial.md', content: 'partial' });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(7)]);
         const results = store.getChunksByIds(['partial-found-0001', 'does-not-exist-999']);
         assert.equal(results.length, 1);
         assert.equal(results[0].id, 'partial-found-0001');
@@ -148,14 +209,14 @@ describe('MemoryStore', () => {
             source: 'indexed-source.md',
             content: 'indexed',
         });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(8)]);
         const sources = store.indexedSources();
         assert.ok(sources.includes('indexed-source.md'), `Expected 'indexed-source.md' in ${sources.join(', ')}`);
     });
     test('deleteByIds removes specific chunks', () => {
         const c1 = makeChunk({ id: 'del-ids-0001', source: 'del-ids.md', content: 'delete by id 1' });
         const c2 = makeChunk({ id: 'del-ids-0002', source: 'del-ids.md', content: 'delete by id 2' });
-        store.upsert([c1, c2], [new Float32Array(384), new Float32Array(384)]);
+        store.upsert([c1, c2], [makeEmbedding(9), makeEmbedding(10)]);
         store.deleteByIds(['del-ids-0001']);
         const hashes = store.hashesBySource('del-ids.md');
         assert.ok(!hashes.has('del-ids-0001'), 'Deleted chunk id must be gone');
@@ -168,7 +229,7 @@ describe('MemoryStore', () => {
             source: 'fts5.md',
             content: 'degradedModeTest keyword',
         });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(11)]);
         const results = store.searchBm25('degradedModeTest', 5);
         assert.ok(results.length > 0, 'FTS5 search must work regardless of vecAvailable');
     });
@@ -205,7 +266,7 @@ describe('MemoryStore', () => {
                     fileMtimeAt: 500,
                     content: 'entity',
                 }),
-            ], [new Float32Array(384), new Float32Array(384), new Float32Array(384), new Float32Array(384)]);
+            ], [makeEmbedding(12), makeEmbedding(13), makeEmbedding(14), makeEmbedding(15)]);
             const allRows = recentStore.getRecentSources({ limit: 10 });
             assert.deepEqual(allRows.map((row) => row.source), [
                 'compiled/entities/entity.md',
@@ -221,7 +282,41 @@ describe('MemoryStore', () => {
             fs.rmSync(recentDir, { recursive: true, force: true });
         }
     });
-    test('schema migration recreates versioned schema with new columns and recency index', () => {
+    test('searchDense returns nearest vector with bounded scores', () => {
+        const denseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-dense-store-'));
+        const denseStore = new MemoryStore(path.join(denseDir, 'index.db'), { ...TEST_CONFIG, wikiDir: denseDir });
+        try {
+            denseStore.upsert([
+                makeChunk({ id: 'dense-alpha', source: 'dense.md', content: 'alpha vector' }),
+                makeChunk({ id: 'dense-beta', source: 'dense.md', content: 'beta vector' }),
+                makeChunk({ id: 'dense-gamma', source: 'dense.md', content: 'gamma vector' }),
+            ], [makeEmbedding(21), makeEmbedding(22), makeEmbedding(23)]);
+            const results = denseStore.searchDense(makeEmbedding(22), 3);
+            assert.ok(results.length > 0, 'Expected dense search results');
+            assert.equal(results[0].id, 'dense-beta');
+            assert.ok(results.every((result) => result.score >= 0 && result.score <= 1));
+        }
+        finally {
+            denseStore.close();
+            fs.rmSync(denseDir, { recursive: true, force: true });
+        }
+    });
+    test('searchDense returns empty array for invalid query embedding', () => {
+        assert.deepEqual(store.searchDense(new Float32Array(3), 5), []);
+    });
+    test('upsert skips invalid embeddings while persisting valid chunks', () => {
+        const validChunk = makeChunk({ id: 'valid-embedding-0001', source: 'embedding-validation.md' });
+        const invalidChunk = makeChunk({ id: 'invalid-embedding-0001', source: 'embedding-validation.md' });
+        const nonFiniteChunk = makeChunk({ id: 'non-finite-embedding-0001', source: 'embedding-validation.md' });
+        const nonFiniteEmbedding = makeEmbedding(24);
+        nonFiniteEmbedding[25] = Number.NaN;
+        store.upsert([validChunk, invalidChunk, nonFiniteChunk], [makeEmbedding(24), new Float32Array(3), nonFiniteEmbedding]);
+        const hashes = store.hashesBySource('embedding-validation.md');
+        assert.ok(hashes.has('valid-embedding-0001'));
+        assert.ok(!hashes.has('invalid-embedding-0001'));
+        assert.ok(!hashes.has('non-finite-embedding-0001'));
+    });
+    test('schema migration recreates versioned schema with new columns, vector index, and working BM25', () => {
         const migrationDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-migration-'));
         const migrationDbPath = path.join(migrationDir, 'index.db');
         const db = new Database(migrationDbPath);
@@ -264,22 +359,75 @@ describe('MemoryStore', () => {
         try {
             const verifyDb = new Database(migrationDbPath, { readonly: true });
             try {
-                assert.equal(verifyDb.pragma('user_version', { simple: true }), SCHEMA_VERSION);
+                assert.equal(readUserVersion(verifyDb), SCHEMA_VERSION);
                 const columns = verifyDb.prepare(`PRAGMA table_info(memory_chunks)`).all();
                 assert.ok(columns.some((column) => column.name === 'source_type'));
                 assert.ok(columns.some((column) => column.name === 'file_mtime_at'));
+                assert.ok(columns.some((column) => column.name === 'embedding'));
                 const indexes = verifyDb.prepare(`PRAGMA index_list(memory_chunks)`).all();
                 assert.ok(indexes.some((index) => index.name === 'idx_memory_chunks_type_mtime'));
+                assert.ok(indexes.some((index) => index.name === 'idx_memory_chunks_embedding'));
                 const rowCount = verifyDb.prepare(`SELECT COUNT(*) AS count FROM memory_chunks`).get();
                 assert.equal(rowCount.count, 0);
             }
             finally {
                 verifyDb.close();
             }
+            const migratedChunk = makeChunk({
+                id: 'migrated-bm25-0001',
+                source: 'migrated.md',
+                content: 'postMigrationKeyword is searchable after migration',
+            });
+            migratedStore.upsert([migratedChunk], [makeEmbedding(16)]);
+            const results = migratedStore.searchBm25('postMigrationKeyword', 5);
+            assert.ok(results.some((result) => result.id === 'migrated-bm25-0001'));
         }
         finally {
             migratedStore.close();
             fs.rmSync(migrationDir, { recursive: true, force: true });
         }
     });
+    test('recreates schema when existing vector dimension differs from config', () => {
+        const dimensionDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-dimension-migration-'));
+        const dimensionDbPath = path.join(dimensionDir, 'index.db');
+        const baseStore = new MemoryStore(dimensionDbPath, { ...TEST_CONFIG, wikiDir: dimensionDir });
+        try {
+            baseStore.upsert([makeChunk({ id: 'old-dimension-row', source: 'dimension.md' })], [makeEmbedding(17)]);
+        }
+        finally {
+            baseStore.close();
+        }
+        const largeConfig = {
+            ...TEST_CONFIG,
+            wikiDir: dimensionDir,
+            embeddingModel: EmbeddingModelName.LARGE,
+            vectorDimension: 768,
+        };
+        const largeStore = new MemoryStore(dimensionDbPath, largeConfig);
+        try {
+            const verifyDb = new Database(dimensionDbPath, { readonly: true });
+            try {
+                const columns = verifyDb.prepare(`PRAGMA table_info(memory_chunks)`).all();
+                const embeddingColumn = columns.find((column) => column.name === 'embedding');
+                assert.equal(embeddingColumn?.type, 'F32_BLOB(768)');
+                const rowCount = verifyDb.prepare(`SELECT COUNT(*) AS count FROM memory_chunks`).get();
+                assert.equal(rowCount.count, 0);
+            }
+            finally {
+                verifyDb.close();
+            }
+            const chunk = makeChunk({
+                id: 'new-dimension-row',
+                source: 'dimension.md',
+                content: 'large dimension vector can be indexed',
+            });
+            largeStore.upsert([chunk], [makeEmbedding(18, 768)]);
+            const results = largeStore.searchBm25('large dimension vector', 5);
+            assert.ok(results.some((result) => result.id === 'new-dimension-row'));
+        }
+        finally {
+            largeStore.close();
+            fs.rmSync(dimensionDir, { recursive: true, force: true });
+        }
+    });
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hanhnd/agent-kit",
-  "version": "1.0.35",
+  "version": "1.0.37",
   "description": "Agent Kit MCP server for software development agent workflows",
   "type": "module",
   "main": "dist/server.js",
@@ -22,7 +22,7 @@
   },
   "scripts": {
     "build": "rm -rf dist && tsc && npm run bundle",
-    "bundle": "esbuild src/entrypoints/server.ts src/entrypoints/cli.ts --bundle --platform=node --format=esm --outdir=dist --external:node:* --external:better-sqlite3 --external:sqlite-vec --external:fastembed --external:node-llama-cpp",
+    "bundle": "esbuild src/entrypoints/server.ts src/entrypoints/cli.ts --bundle --platform=node --format=esm --outdir=dist --external:node:* --external:libsql --external:fastembed --external:node-llama-cpp",
     "dev": "tsc --watch",
     "prepublishOnly": "npm run build",
     "test": "tsc && node --test $(find dist -name '*.test.js' | tr '\\n' ' ')"
@@ -30,15 +30,13 @@
   "dependencies": {
     "node-llama-cpp": "^3.0.0",
     "@modelcontextprotocol/sdk": "^1.13.1",
-    "better-sqlite3": "^11.0.0",
     "dedent": "^1.5.3",
     "fastembed": "^1.14.0",
-    "sqlite-vec": "^0.1.0",
+    "libsql": "^0.5.29",
     "stopword": "^3.1.5",
     "zod": "^3.24.4"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.0",
     "@types/node": "^22.15.21",
     "esbuild": "^0.25.4",
     "typescript": "^5.8.3"