@hanhnd/agent-kit 1.0.34 → 1.0.36

package/dist/services/memory/store.test.js

@@ -1,9 +1,10 @@
+import Database from 'libsql';
 import * as assert from 'node:assert/strict';
 import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import { after, before, describe, test } from 'node:test';
-import { MemoryStore } from './store.js';
+import { MemoryStore, SCHEMA_VERSION } from './store.js';
 import { EmbeddingModelName } from './types.js';
 const TEST_CONFIG = {
     enabled: true,
@@ -11,21 +12,42 @@ const TEST_CONFIG = {
     topK: 5,
     chunkSize: 1500,
     overlapLines: 2,
-    embeddingModel: EmbeddingModelName.SMALL,
+    embeddingModel: EmbeddingModelName.BASE,
     vectorDimension: 384,
 };
 function makeChunk(overrides = {}) {
     return {
         id: 'test-id-0000001',
         source: 'test.md',
+        sourceType: 'wiki',
         heading: 'Test Heading',
         headingLevel: 1,
         content: 'This is test content for BM25 search.',
         lineStart: 1,
         lineEnd: 5,
+        fileMtimeAt: 1000,
         ...overrides,
     };
 }
+function makeEmbedding(activeIndex, dimension = TEST_CONFIG.vectorDimension) {
+    const embedding = new Float32Array(dimension);
+    embedding[activeIndex] = 1;
+    return embedding;
+}
+function readUserVersion(db) {
+    const value = db.pragma('user_version', { simple: true });
+    if (typeof value === 'number')
+        return value;
+    if (Array.isArray(value)) {
+        const [row] = value;
+        return typeof row?.user_version === 'number' ? row.user_version : 0;
+    }
+    if (value && typeof value === 'object' && 'user_version' in value) {
+        const row = value;
+        return typeof row.user_version === 'number' ? row.user_version : 0;
+    }
+    return 0;
+}
 describe('MemoryStore', () => {
     let tmpDir;
     let dbPath;
@@ -39,8 +61,50 @@ describe('MemoryStore', () => {
         store.close();
         fs.rmSync(tmpDir, { recursive: true, force: true });
     });
-    test('vecAvailable is a boolean', () => {
-        assert.equal(typeof store.vecAvailable, 'boolean');
+    test('libsql supports required local driver and vector features', () => {
+        const libsqlDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-libsql-compat-'));
+        const libsqlDb = new Database(path.join(libsqlDir, 'index.db'));
+        try {
+            assert.equal(readUserVersion(libsqlDb), 0);
+            libsqlDb.pragma('journal_mode = WAL');
+            libsqlDb.pragma('busy_timeout = 5000');
+            libsqlDb.pragma(`user_version = ${SCHEMA_VERSION}`);
+            assert.equal(readUserVersion(libsqlDb), SCHEMA_VERSION);
+            libsqlDb.exec(`
+        CREATE TABLE vector_items (
+          rowid INTEGER PRIMARY KEY AUTOINCREMENT,
+          name TEXT NOT NULL,
+          embedding F32_BLOB(3) NOT NULL
+        );
+        CREATE INDEX idx_vector_items_embedding
+          ON vector_items (libsql_vector_idx(embedding));
+      `);
+            const insert = libsqlDb.prepare(`INSERT INTO vector_items (name, embedding) VALUES (?, vector32(?))`);
+            const info = insert.run('alpha', '[1,0,0]');
+            assert.equal(info.lastInsertRowid, 1);
+            insert.run('beta', '[0,1,0]');
+            const insertGamma = libsqlDb.transaction(() => insert.run('gamma', '[0,0,1]'));
+            insertGamma();
+            const distance = libsqlDb
+                .prepare(`SELECT vector_distance_cos(embedding, vector32(?)) AS distance FROM vector_items WHERE name = ?`)
+                .get('[1,0,0]', 'alpha');
+            assert.equal(distance.distance, 0);
+            const rows = libsqlDb
+                .prepare(`SELECT vector_items.name
+           FROM vector_top_k('idx_vector_items_embedding', vector32(?), ?) AS vector_matches
+           JOIN vector_items ON vector_items.rowid = vector_matches.id
+           ORDER BY vector_distance_cos(vector_items.embedding, vector32(?))`)
+                .all('[1,0,0]', 2, '[1,0,0]');
+            assert.equal(rows[0].name, 'alpha');
+            assert.equal(rows.length, 2);
+        }
+        finally {
+            libsqlDb.close();
+            fs.rmSync(libsqlDir, { recursive: true, force: true });
+        }
+    });
+    test('vecAvailable is true after schema creation', () => {
+        assert.equal(store.vecAvailable, true);
     });
     test('hashesBySource returns empty set for unknown source', () => {
         const hashes = store.hashesBySource('nonexistent.md');
@@ -48,11 +112,14 @@ describe('MemoryStore', () => {
         assert.equal(hashes.size, 0);
     });
     test('upsert stores chunks and hashesBySource returns their ids', () => {
-        const chunk = makeChunk({ id: 'upsert-test-0001', source: 'upsert.md' });
-        const embedding = new Float32Array(384).fill(0.1);
+        const chunk = makeChunk({ id: 'upsert-test-0001', source: 'upsert.md', sourceType: 'entity', fileMtimeAt: 2000 });
+        const embedding = makeEmbedding(0);
         store.upsert([chunk], [embedding]);
         const hashes = store.hashesBySource('upsert.md');
         assert.ok(hashes.has('upsert-test-0001'));
+        const [stored] = store.getChunksByIds(['upsert-test-0001']);
+        assert.equal(stored.sourceType, 'entity');
+        assert.equal(stored.fileMtimeAt, 2000);
     });
     test('deleteBySource removes all chunks for that source', () => {
         const c1 = makeChunk({
@@ -65,7 +132,7 @@ describe('MemoryStore', () => {
             source: 'deleteme.md',
             content: 'delete source chunk 2',
         });
-        store.upsert([c1, c2], [new Float32Array(384), new Float32Array(384)]);
+        store.upsert([c1, c2], [makeEmbedding(1), makeEmbedding(2)]);
         store.deleteBySource('deleteme.md');
         const hashes = store.hashesBySource('deleteme.md');
         assert.equal(hashes.size, 0);
@@ -76,7 +143,7 @@ describe('MemoryStore', () => {
             source: 'bm25.md',
             content: 'uniqueKeywordXYZ for BM25 testing',
         });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(3)]);
         const results = store.searchBm25('uniqueKeywordXYZ', 5);
         assert.ok(results.length > 0, 'Expected at least one BM25 result');
         assert.ok(results.some((r) => r.id === 'bm25-search-0001'), `Expected chunk id in results, got: ${results.map((r) => r.id).join(', ')}`);
@@ -95,7 +162,7 @@ describe('MemoryStore', () => {
             heading: 'Open Questions',
             content: 'How should git manage worktree lifecycle decisions?',
         });
-        store.upsert([preference, unrelated], [new Float32Array(384), new Float32Array(384)]);
+        store.upsert([preference, unrelated], [makeEmbedding(4), makeEmbedding(5)]);
         const results = store.searchBm25('personal likes and preferences of the user', 5);
         assert.ok(results.length > 0, 'Expected preference query to return results');
         assert.equal(results[0].id, 'preference-search-0001');
@@ -108,26 +175,30 @@ describe('MemoryStore', () => {
         const chunk = makeChunk({
             id: 'get-by-ids-0001',
             source: 'metadata.md',
+            sourceType: 'concept',
             heading: 'Metadata Section',
             headingLevel: 2,
             content: 'Content for metadata test',
             lineStart: 10,
             lineEnd: 20,
+            fileMtimeAt: 3000,
         });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(6)]);
         const results = store.getChunksByIds(['get-by-ids-0001']);
         assert.equal(results.length, 1);
         const [r] = results;
         assert.equal(r.id, 'get-by-ids-0001');
         assert.equal(r.source, 'metadata.md');
+        assert.equal(r.sourceType, 'concept');
         assert.equal(r.heading, 'Metadata Section');
         assert.equal(r.headingLevel, 2);
         assert.equal(r.lineStart, 10);
         assert.equal(r.lineEnd, 20);
+        assert.equal(r.fileMtimeAt, 3000);
     });
     test('getChunksByIds returns only found rows for mixed ids', () => {
         const chunk = makeChunk({ id: 'partial-found-0001', source: 'partial.md', content: 'partial' });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(7)]);
         const results = store.getChunksByIds(['partial-found-0001', 'does-not-exist-999']);
         assert.equal(results.length, 1);
         assert.equal(results[0].id, 'partial-found-0001');
@@ -138,14 +209,14 @@ describe('MemoryStore', () => {
             source: 'indexed-source.md',
             content: 'indexed',
         });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(8)]);
         const sources = store.indexedSources();
         assert.ok(sources.includes('indexed-source.md'), `Expected 'indexed-source.md' in ${sources.join(', ')}`);
     });
     test('deleteByIds removes specific chunks', () => {
         const c1 = makeChunk({ id: 'del-ids-0001', source: 'del-ids.md', content: 'delete by id 1' });
         const c2 = makeChunk({ id: 'del-ids-0002', source: 'del-ids.md', content: 'delete by id 2' });
-        store.upsert([c1, c2], [new Float32Array(384), new Float32Array(384)]);
+        store.upsert([c1, c2], [makeEmbedding(9), makeEmbedding(10)]);
         store.deleteByIds(['del-ids-0001']);
         const hashes = store.hashesBySource('del-ids.md');
         assert.ok(!hashes.has('del-ids-0001'), 'Deleted chunk id must be gone');
@@ -158,8 +229,162 @@ describe('MemoryStore', () => {
             source: 'fts5.md',
             content: 'degradedModeTest keyword',
         });
-        store.upsert([chunk], [new Float32Array(384)]);
+        store.upsert([chunk], [makeEmbedding(11)]);
         const results = store.searchBm25('degradedModeTest', 5);
         assert.ok(results.length > 0, 'FTS5 search must work regardless of vecAvailable');
     });
+    test('getRecentSources returns distinct sources ordered by newest chunk mtime', () => {
+        const recentDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-recent-store-'));
+        const recentStore = new MemoryStore(path.join(recentDir, 'index.db'), { ...TEST_CONFIG, wikiDir: recentDir });
+        try {
+            recentStore.upsert([
+                makeChunk({
+                    id: 'recent-digest-old',
+                    source: 'compiled/provisional/conversation-digests/old.md',
+                    sourceType: 'digest',
+                    fileMtimeAt: 100,
+                    content: 'old digest',
+                }),
+                makeChunk({
+                    id: 'recent-digest-new',
+                    source: 'compiled/provisional/conversation-digests/new.md',
+                    sourceType: 'digest',
+                    fileMtimeAt: 300,
+                    content: 'new digest',
+                }),
+                makeChunk({
+                    id: 'recent-digest-newer-same-source',
+                    source: 'compiled/provisional/conversation-digests/old.md',
+                    sourceType: 'digest',
+                    fileMtimeAt: 400,
+                    content: 'old digest update',
+                }),
+                makeChunk({
+                    id: 'recent-entity',
+                    source: 'compiled/entities/entity.md',
+                    sourceType: 'entity',
+                    fileMtimeAt: 500,
+                    content: 'entity',
+                }),
+            ], [makeEmbedding(12), makeEmbedding(13), makeEmbedding(14), makeEmbedding(15)]);
+            const allRows = recentStore.getRecentSources({ limit: 10 });
+            assert.deepEqual(allRows.map((row) => row.source), [
+                'compiled/entities/entity.md',
+                'compiled/provisional/conversation-digests/old.md',
+                'compiled/provisional/conversation-digests/new.md',
+            ]);
+            const digestRows = recentStore.getRecentSources({ limit: 10, sourceType: 'digest' });
+            assert.deepEqual(digestRows.map((row) => row.source), ['compiled/provisional/conversation-digests/old.md', 'compiled/provisional/conversation-digests/new.md']);
+            assert.equal(digestRows[0].fileMtimeAt, 400);
+        }
+        finally {
+            recentStore.close();
+            fs.rmSync(recentDir, { recursive: true, force: true });
+        }
+    });
+    test('searchDense returns nearest vector with bounded scores', () => {
+        const denseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-dense-store-'));
+        const denseStore = new MemoryStore(path.join(denseDir, 'index.db'), { ...TEST_CONFIG, wikiDir: denseDir });
+        try {
+            denseStore.upsert([
+                makeChunk({ id: 'dense-alpha', source: 'dense.md', content: 'alpha vector' }),
+                makeChunk({ id: 'dense-beta', source: 'dense.md', content: 'beta vector' }),
+                makeChunk({ id: 'dense-gamma', source: 'dense.md', content: 'gamma vector' }),
+            ], [makeEmbedding(21), makeEmbedding(22), makeEmbedding(23)]);
+            const results = denseStore.searchDense(makeEmbedding(22), 3);
+            assert.ok(results.length > 0, 'Expected dense search results');
+            assert.equal(results[0].id, 'dense-beta');
+            assert.ok(results.every((result) => result.score >= 0 && result.score <= 1));
+        }
+        finally {
+            denseStore.close();
+            fs.rmSync(denseDir, { recursive: true, force: true });
+        }
+    });
+    test('searchDense returns empty array for invalid query embedding', () => {
+        assert.deepEqual(store.searchDense(new Float32Array(3), 5), []);
+    });
+    test('upsert skips invalid embeddings while persisting valid chunks', () => {
+        const validChunk = makeChunk({ id: 'valid-embedding-0001', source: 'embedding-validation.md' });
+        const invalidChunk = makeChunk({ id: 'invalid-embedding-0001', source: 'embedding-validation.md' });
+        const nonFiniteChunk = makeChunk({ id: 'non-finite-embedding-0001', source: 'embedding-validation.md' });
+        const nonFiniteEmbedding = makeEmbedding(24);
+        nonFiniteEmbedding[25] = Number.NaN;
+        store.upsert([validChunk, invalidChunk, nonFiniteChunk], [makeEmbedding(24), new Float32Array(3), nonFiniteEmbedding]);
+        const hashes = store.hashesBySource('embedding-validation.md');
+        assert.ok(hashes.has('valid-embedding-0001'));
+        assert.ok(!hashes.has('invalid-embedding-0001'));
+        assert.ok(!hashes.has('non-finite-embedding-0001'));
+    });
+    test('schema migration recreates versioned schema with new columns, vector index, and working BM25', () => {
+        const migrationDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-migration-'));
+        const migrationDbPath = path.join(migrationDir, 'index.db');
+        const db = new Database(migrationDbPath);
+        db.exec(`
+      PRAGMA user_version = 1;
+      CREATE TABLE memory_chunks (
+        rowid INTEGER PRIMARY KEY AUTOINCREMENT,
+        id TEXT NOT NULL UNIQUE,
+        source TEXT NOT NULL,
+        heading TEXT NOT NULL DEFAULT '',
+        heading_level INTEGER NOT NULL DEFAULT 0,
+        content TEXT NOT NULL,
+        line_start INTEGER NOT NULL,
+        line_end INTEGER NOT NULL,
+        indexed_at INTEGER NOT NULL
+      );
+      CREATE VIRTUAL TABLE memory_fts USING fts5(
+        source,
+        heading,
+        content,
+        content='memory_chunks',
+        content_rowid='rowid'
+      );
+      CREATE TRIGGER memory_chunks_ai
+        AFTER INSERT ON memory_chunks BEGIN
+          INSERT INTO memory_fts(rowid, source, heading, content)
+            VALUES (new.rowid, new.source, new.heading, new.content);
+        END;
+      CREATE TRIGGER memory_chunks_ad
+        AFTER DELETE ON memory_chunks BEGIN
+          INSERT INTO memory_fts(memory_fts, rowid, source, heading, content)
+            VALUES ('delete', old.rowid, old.source, old.heading, old.content);
+        END;
+    `);
+        db.prepare(`INSERT INTO memory_chunks
+        (id, source, heading, heading_level, content, line_start, line_end, indexed_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`).run('old-row', 'old.md', '', 0, 'old content', 1, 1, 1);
+        db.close();
+        const migratedStore = new MemoryStore(migrationDbPath, { ...TEST_CONFIG, wikiDir: migrationDir });
+        try {
+            const verifyDb = new Database(migrationDbPath, { readonly: true });
+            try {
+                assert.equal(readUserVersion(verifyDb), SCHEMA_VERSION);
+                const columns = verifyDb.prepare(`PRAGMA table_info(memory_chunks)`).all();
+                assert.ok(columns.some((column) => column.name === 'source_type'));
+                assert.ok(columns.some((column) => column.name === 'file_mtime_at'));
+                assert.ok(columns.some((column) => column.name === 'embedding'));
+                const indexes = verifyDb.prepare(`PRAGMA index_list(memory_chunks)`).all();
+                assert.ok(indexes.some((index) => index.name === 'idx_memory_chunks_type_mtime'));
+                assert.ok(indexes.some((index) => index.name === 'idx_memory_chunks_embedding'));
+                const rowCount = verifyDb.prepare(`SELECT COUNT(*) AS count FROM memory_chunks`).get();
+                assert.equal(rowCount.count, 0);
+            }
+            finally {
+                verifyDb.close();
+            }
+            const migratedChunk = makeChunk({
+                id: 'migrated-bm25-0001',
+                source: 'migrated.md',
+                content: 'postMigrationKeyword is searchable after migration',
+            });
+            migratedStore.upsert([migratedChunk], [makeEmbedding(16)]);
+            const results = migratedStore.searchBm25('postMigrationKeyword', 5);
+            assert.ok(results.some((result) => result.id === 'migrated-bm25-0001'));
+        }
+        finally {
+            migratedStore.close();
+            fs.rmSync(migrationDir, { recursive: true, force: true });
+        }
+    });
 });

package/dist/services/memory/types.d.ts

@@ -1,17 +1,21 @@
 import type { ConversationDigestSettings } from '../digest/types.js';
 export declare enum EmbeddingModelName {
     TINY = "tiny",
-    SMALL = "small",
+    BASE = "base",
     LARGE = "large"
 }
+export declare const SOURCE_TYPES: readonly ["digest", "concept", "entity", "preference", "wiki"];
+export type SourceType = (typeof SOURCE_TYPES)[number];
 export interface MemoryChunk {
     id: string;
     source: string;
+    sourceType: SourceType;
     heading: string;
     headingLevel: number;
     content: string;
     lineStart: number;
     lineEnd: number;
+    fileMtimeAt: number;
 }
 export interface SearchResult {
     chunk: MemoryChunk;
@@ -34,3 +38,7 @@ export interface IndexStats {
     deleted: number;
     skipped: number;
 }
+export interface RecentSource {
+    source: string;
+    fileMtimeAt: number;
+}

package/dist/services/memory/types.js

@@ -1,6 +1,7 @@
 export var EmbeddingModelName;
 (function (EmbeddingModelName) {
     EmbeddingModelName["TINY"] = "tiny";
-    EmbeddingModelName["SMALL"] = "small";
+    EmbeddingModelName["BASE"] = "base";
     EmbeddingModelName["LARGE"] = "large";
 })(EmbeddingModelName || (EmbeddingModelName = {}));
+export const SOURCE_TYPES = ['digest', 'concept', 'entity', 'preference', 'wiki'];

package/dist/utils/paths.d.ts

@@ -1,3 +1,5 @@
+export declare const AGENT_KIT_HOME: string;
 export declare const CREDENTIALS_FILE: string;
+export declare const GLOBAL_SETTINGS_FILE: string;
 export declare const MODEL_CACHE_DIR: string;
 export declare const FASTEMBED_CACHE_DIR: string;

package/dist/utils/paths.js

@@ -1,6 +1,7 @@
 import * as os from 'os';
 import * as path from 'path';
-const AGENT_KIT_HOME = path.join(os.homedir(), '.agent-kit');
+export const AGENT_KIT_HOME = path.join(os.homedir(), '.agent-kit');
 export const CREDENTIALS_FILE = path.join(AGENT_KIT_HOME, 'credentials');
+export const GLOBAL_SETTINGS_FILE = path.join(AGENT_KIT_HOME, 'settings.json');
 export const MODEL_CACHE_DIR = path.join(AGENT_KIT_HOME, 'cache', 'models');
 export const FASTEMBED_CACHE_DIR = path.join(MODEL_CACHE_DIR, 'fastembed');

package/dist/utils/utils.d.ts

@@ -7,4 +7,5 @@ type McpTextResult = {
 };
 export declare function mcpText(text: string): McpTextResult;
 export declare function mcpJson(obj: unknown): McpTextResult;
+export declare function formatError(err: unknown): string;
 export {};

package/dist/utils/utils.js

@@ -3,9 +3,6 @@ export function getWorkspaceRoot() {
         process.env.CLAUDE_PROJECT_DIR ||
         process.env.CODEX_PROJECT_DIR ||
         process.env.GEMINI_PROJECT_DIR ||
-        process.env.CLAUDE_WORKSPACE ||
-        process.env.CODEX_WORKSPACE ||
-        process.env.GEMINI_WORKSPACE ||
         process.env.INIT_CWD ||
         process.env.PWD ||
         process.cwd());
@@ -16,3 +13,6 @@ export function mcpText(text) {
 export function mcpJson(obj) {
     return mcpText(JSON.stringify(obj, null, 2));
 }
+export function formatError(err) {
+    return err instanceof Error ? err.message : String(err);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hanhnd/agent-kit",
-  "version": "1.0.34",
+  "version": "1.0.36",
   "description": "Agent Kit MCP server for software development agent workflows",
   "type": "module",
   "main": "dist/server.js",
@@ -22,7 +22,7 @@
   },
   "scripts": {
     "build": "rm -rf dist && tsc && npm run bundle",
-    "bundle": "esbuild src/entrypoints/server.ts src/entrypoints/cli.ts --bundle --platform=node --format=esm --outdir=dist --external:node:* --external:better-sqlite3 --external:sqlite-vec --external:fastembed --external:node-llama-cpp --banner:js=\"import { createRequire } from 'module'; const require = createRequire(import.meta.url);\"",
+    "bundle": "esbuild src/entrypoints/server.ts src/entrypoints/cli.ts --bundle --platform=node --format=esm --outdir=dist --external:node:* --external:libsql --external:fastembed --external:node-llama-cpp",
     "dev": "tsc --watch",
     "prepublishOnly": "npm run build",
     "test": "tsc && node --test $(find dist -name '*.test.js' | tr '\\n' ' ')"
@@ -30,15 +30,13 @@
   "dependencies": {
     "node-llama-cpp": "^3.0.0",
     "@modelcontextprotocol/sdk": "^1.13.1",
-    "better-sqlite3": "^11.0.0",
     "dedent": "^1.5.3",
     "fastembed": "^1.14.0",
-    "sqlite-vec": "^0.1.0",
+    "libsql": "^0.5.29",
     "stopword": "^3.1.5",
     "zod": "^3.24.4"
   },
   "devDependencies": {
-    "@types/better-sqlite3": "^7.6.0",
     "@types/node": "^22.15.21",
     "esbuild": "^0.25.4",
     "typescript": "^5.8.3"