@hanhnd/agent-kit 1.0.35 → 1.0.37

package/dist/services/digest/digest-processor.test.js

@@ -1,8 +1,6 @@
 import * as assert from 'node:assert/strict';
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import Database from 'better-sqlite3';
-import { load as loadSqliteVec } from 'sqlite-vec';
 import { afterEach, describe, test } from 'node:test';
 import { createTempDirTracker } from '../../utils/temp-dir.test.js';
 import { DEFAULT_DIGEST_MODEL_ID } from './constants.js';
@@ -53,7 +51,7 @@ describe('digestConversationFile', () => {
         assert.equal(result.contentHash, input.contentHash);
         assert.match(path.basename(result.markdown), /^[a-f0-9]{16}-conv\.md$/);
     });
-    test('indexes existing provisional digest with real store vector rows when sqlite-vec is available', async () => {
+    test('indexes existing provisional digest with searchable store vector data', async () => {
         const workspace = tempDirs.makeTempDir('digest-processor-index-');
         const inputPath = path.join(workspace, '.agent-kit', 'wiki', 'archive', 'conversations', 'conv.md');
         fs.mkdirSync(path.dirname(inputPath), { recursive: true });
@@ -74,22 +72,10 @@ describe('digestConversationFile', () => {
             });
             assert.equal(result.indexed, true);
             assert.equal(result.markdown, markdownPath);
-            if (store.vecAvailable) {
-                const db = new Database(dbPath, { readonly: true });
-                try {
-                    loadSqliteVec(db);
-                    const row = db
-                        .prepare(`SELECT COUNT(*) AS count
-               FROM memory_vec mv
-               JOIN memory_chunks mc ON mc.rowid = mv.rowid
-               WHERE mc.source = ?`)
-                        .get(path.relative(config.wikiDir, markdownPath));
-                    assert.ok(row.count > 0, `Expected vector rows for ${markdownPath}`);
-                }
-                finally {
-                    db.close();
-                }
-            }
+            const denseResults = store.searchDense(new Float32Array(384).fill(0.05), 5);
+            const indexedSource = path.relative(config.wikiDir, markdownPath);
+            const indexedChunks = store.getChunksByIds(denseResults.map((denseResult) => denseResult.id));
+            assert.ok(indexedChunks.some((chunk) => chunk.source === indexedSource), `Expected dense search to return a chunk for ${indexedSource}`);
         }
         finally {
             store.close();

package/dist/services/memory/constants.d.ts

@@ -1,4 +1,5 @@
-import { type MemoryConfig } from './types.js';
+import { type MemoryConfig, EmbeddingModelName } from './types.js';
+export declare const EMBEDDING_MODEL_DIMENSIONS: Record<EmbeddingModelName, number>;
 export declare const DEFAULT_MEMORY_CONFIG: MemoryConfig;
 export declare const LOCK_RETRY_MS = 50;
 export declare const LOCK_TIMEOUT_MS = 500;

package/dist/services/memory/constants.js

@@ -1,4 +1,9 @@
 import { EmbeddingModelName } from './types.js';
+export const EMBEDDING_MODEL_DIMENSIONS = {
+    [EmbeddingModelName.TINY]: 384,
+    [EmbeddingModelName.BASE]: 384,
+    [EmbeddingModelName.LARGE]: 768,
+};
 export const DEFAULT_MEMORY_CONFIG = {
     enabled: false,
     wikiDir: '',
@@ -6,7 +11,7 @@ export const DEFAULT_MEMORY_CONFIG = {
     chunkSize: 1500,
     overlapLines: 2,
     embeddingModel: EmbeddingModelName.BASE,
-    vectorDimension: 384,
+    vectorDimension: EMBEDDING_MODEL_DIMENSIONS[EmbeddingModelName.BASE],
 };
 export const LOCK_RETRY_MS = 50;
 export const LOCK_TIMEOUT_MS = 500;

package/dist/services/memory/memory.test.js

@@ -402,4 +402,14 @@ describe('resolveMemoryConfig', () => {
         const config = resolveMemoryConfig({ memory: { wikiDir: '/custom/wiki' } }, '/repo');
         assert.equal(config.wikiDir, '/custom/wiki');
     });
+    test('derives vector dimension from embedding model', () => {
+        const config = resolveMemoryConfig({ memory: { embeddingModel: EmbeddingModelName.LARGE } }, '/repo');
+        assert.equal(config.embeddingModel, EmbeddingModelName.LARGE);
+        assert.equal(config.vectorDimension, 768);
+    });
+    test('preserves explicit vector dimension override', () => {
+        const config = resolveMemoryConfig({ memory: { embeddingModel: EmbeddingModelName.LARGE, vectorDimension: 1024 } }, '/repo');
+        assert.equal(config.embeddingModel, EmbeddingModelName.LARGE);
+        assert.equal(config.vectorDimension, 1024);
+    });
 });

package/dist/services/memory/store.d.ts

@@ -1,17 +1,19 @@
 import type { MemoryChunk, MemoryConfig, RecentSource, SourceType } from './types.js';
-export declare const SCHEMA_VERSION = 2;
+export declare const SCHEMA_VERSION = 4;
 export declare class StoreError extends Error {
     constructor(message: string, cause?: unknown);
 }
 export declare class MemoryStore {
     private readonly config;
     private db;
-    private _vecAvailable;
     constructor(dbPath: string, config: MemoryConfig);
     private openDatabase;
-    private loadVecExtension;
     private createSchema;
+    private readEmbeddingColumnDimension;
+    private readUserVersion;
     upsert(chunks: MemoryChunk[], embeddings: Float32Array[]): void;
+    private toVectorBinding;
+    private normalizeVectorDistance;
     hashesBySource(source: string): Set<string>;
     indexedSources(): string[];
     getRecentSources(options: {

package/dist/services/memory/store.js

@@ -1,10 +1,9 @@
-import Database from 'better-sqlite3';
+import Database from 'libsql';
 import * as fs from 'fs';
-import { createRequire } from 'module';
 import * as path from 'path';
 import { eng, removeStopwords } from 'stopword';
-const require = createRequire(import.meta.url);
-export const SCHEMA_VERSION = 2;
+export const SCHEMA_VERSION = 4;
+const LibsqlDatabase = Database;
 export class StoreError extends Error {
     constructor(message, cause) {
         super(message);
@@ -16,7 +15,6 @@ export class StoreError extends Error {
 export class MemoryStore {
     config;
     db;
-    _vecAvailable = false;
     constructor(dbPath, config) {
         this.config = config;
         fs.mkdirSync(path.dirname(dbPath), { recursive: true });
@@ -24,10 +22,9 @@ export class MemoryStore {
     }
     openDatabase(dbPath) {
         try {
-            const db = new Database(dbPath);
+            const db = new LibsqlDatabase(dbPath);
             db.pragma('journal_mode = WAL');
             db.pragma('busy_timeout = 5000');
-            this.loadVecExtension(db);
             this.createSchema(db);
             return db;
         }
@@ -39,34 +36,30 @@ export class MemoryStore {
             catch {
                 // ignore
             }
-            const db = new Database(dbPath);
-            db.pragma('journal_mode = WAL');
-            db.pragma('busy_timeout = 5000');
-            this.loadVecExtension(db);
-            this.createSchema(db);
-            return db;
-        }
-    }
-    loadVecExtension(db) {
-        try {
-            const sqliteVec = require('sqlite-vec');
-            sqliteVec.load(db);
-            this._vecAvailable = true;
-        }
-        catch (err) {
-            console.warn('[memory-store] sqlite-vec unavailable, degrading to FTS5-only:', err);
-            this._vecAvailable = false;
+            try {
+                const db = new LibsqlDatabase(dbPath);
+                db.pragma('journal_mode = WAL');
+                db.pragma('busy_timeout = 5000');
+                this.createSchema(db);
+                return db;
+            }
+            catch (retryErr) {
+                throw new StoreError('database open failed', retryErr);
+            }
         }
     }
     createSchema(db) {
         try {
-            const current = Number(db.pragma('user_version', { simple: true }) ?? 0);
-            if (current < SCHEMA_VERSION) {
+            const current = this.readUserVersion(db);
+            const currentVectorDimension = current >= SCHEMA_VERSION ? this.readEmbeddingColumnDimension(db) : undefined;
+            if (current < SCHEMA_VERSION ||
+                (currentVectorDimension !== undefined && currentVectorDimension !== this.config.vectorDimension)) {
                 db.exec(`
           DROP TRIGGER IF EXISTS memory_chunks_ai;
           DROP TRIGGER IF EXISTS memory_chunks_ad;
           DROP TABLE IF EXISTS memory_fts;
           DROP TABLE IF EXISTS memory_vec;
+          DROP INDEX IF EXISTS idx_memory_chunks_embedding;
           DROP TABLE IF EXISTS memory_chunks;
         `);
                 db.pragma(`user_version = ${SCHEMA_VERSION}`);
@@ -83,7 +76,8 @@ export class MemoryStore {
           line_start    INTEGER NOT NULL,
           line_end      INTEGER NOT NULL,
           indexed_at    INTEGER NOT NULL,
-          file_mtime_at INTEGER NOT NULL DEFAULT 0
+          file_mtime_at INTEGER NOT NULL DEFAULT 0,
+          embedding     F32_BLOB(${this.config.vectorDimension}) NOT NULL
         );
 
         CREATE INDEX IF NOT EXISTS idx_memory_chunks_type_mtime
@@ -108,48 +102,63 @@ export class MemoryStore {
             INSERT INTO memory_fts(memory_fts, rowid, source, heading, content)
               VALUES ('delete', old.rowid, old.source, old.heading, old.content);
           END;
+
+        CREATE INDEX IF NOT EXISTS idx_memory_chunks_embedding
+          ON memory_chunks (libsql_vector_idx(embedding));
       `);
-            if (this._vecAvailable) {
-                try {
-                    db.exec(`
-            CREATE VIRTUAL TABLE IF NOT EXISTS memory_vec USING vec0(
-              embedding FLOAT[${this.config.vectorDimension}]
-            );
-          `);
-                }
-                catch (err) {
-                    console.warn('[memory-store] Failed to create memory_vec, disabling vector search:', err);
-                    this._vecAvailable = false;
-                }
-            }
         }
         catch (err) {
             throw new StoreError('schema migration failed', err);
         }
     }
+    readEmbeddingColumnDimension(db) {
+        try {
+            const rows = db.pragma('table_info(memory_chunks)');
+            if (!Array.isArray(rows))
+                return undefined;
+            const embeddingColumn = rows.find((row) => {
+                if (!row || typeof row !== 'object')
+                    return false;
+                const candidate = row;
+                return candidate.name === 'embedding' && typeof candidate.type === 'string';
+            });
+            const match = /^F32_BLOB\((\d+)\)$/i.exec(embeddingColumn?.type.trim() ?? '');
+            if (!match)
+                return undefined;
+            return Number.parseInt(match[1], 10);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    readUserVersion(db) {
+        const value = db.pragma('user_version', { simple: true });
+        if (typeof value === 'number')
+            return value;
+        if (Array.isArray(value)) {
+            const [row] = value;
+            return typeof row?.user_version === 'number' ? row.user_version : 0;
+        }
+        if (value && typeof value === 'object' && 'user_version' in value) {
+            const row = value;
+            return typeof row.user_version === 'number' ? row.user_version : 0;
+        }
+        return 0;
+    }
     upsert(chunks, embeddings) {
         const insertChunk = this.db.prepare(`
       INSERT OR REPLACE INTO memory_chunks
-        (id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        (id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at, embedding)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector32(?))
     `);
-        let insertVec = null;
-        if (this._vecAvailable) {
-            insertVec = this.db.prepare(`INSERT OR REPLACE INTO memory_vec (rowid, embedding) VALUES (?, ?)`);
-        }
         const upsertAll = this.db.transaction(() => {
             for (let i = 0; i < chunks.length; i++) {
                 const chunk = chunks[i];
                 try {
+                    const vectorBinding = this.toVectorBinding(embeddings[i]);
                     // Delete existing row first to get correct rowid
                     this.db.prepare(`DELETE FROM memory_chunks WHERE id = ?`).run(chunk.id);
-                    const info = insertChunk.run(chunk.id, chunk.source, chunk.sourceType, chunk.heading, chunk.headingLevel, chunk.content, chunk.lineStart, chunk.lineEnd, Date.now(), chunk.fileMtimeAt);
-                    if (insertVec && embeddings[i] && info.lastInsertRowid) {
-                        const rowid = typeof info.lastInsertRowid === 'bigint' ? info.lastInsertRowid : BigInt(info.lastInsertRowid);
-                        const emb = embeddings[i];
-                        const buf = Buffer.from(emb.buffer, emb.byteOffset, emb.byteLength);
-                        insertVec.run(rowid, buf);
-                    }
+                    insertChunk.run(chunk.id, chunk.source, chunk.sourceType, chunk.heading, chunk.headingLevel, chunk.content, chunk.lineStart, chunk.lineEnd, Date.now(), chunk.fileMtimeAt, vectorBinding);
                 }
                 catch (err) {
                     console.warn(`[memory-store] Failed to upsert chunk ${chunk.id}:`, err);
@@ -163,6 +172,25 @@ export class MemoryStore {
             throw new StoreError('Full batch upsert failed', err);
         }
     }
+    toVectorBinding(embedding) {
+        if (!embedding)
+            throw new Error('missing embedding');
+        if (!(embedding instanceof Float32Array))
+            throw new Error('missing embedding');
+        if (embedding.length !== this.config.vectorDimension)
+            throw new Error('embedding dimension mismatch');
+        const values = Array.from(embedding, (value) => {
+            if (!Number.isFinite(value))
+                throw new Error('embedding contains non-finite value');
+            return value;
+        });
+        return JSON.stringify(values);
+    }
+    normalizeVectorDistance(distance) {
+        if (distance <= 0)
+            return 1;
+        return Math.max(0, Math.min(1, 1 - distance));
+    }
     hashesBySource(source) {
         const rows = this.db.prepare(`SELECT id FROM memory_chunks WHERE source = ?`).all(source);
         return new Set(rows.map((r) => r.id));
@@ -187,12 +215,7 @@ export class MemoryStore {
         }
     }
     deleteBySource(source) {
-        const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE source = ?`).all(source);
         const deleteAll = this.db.transaction(() => {
-            if (this._vecAvailable && rows.length > 0) {
-                const placeholders = rows.map(() => '?').join(', ');
-                this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${placeholders})`).run(...rows.map((r) => r.rowid));
-            }
             this.db.prepare(`DELETE FROM memory_chunks WHERE source = ?`).run(source);
         });
         deleteAll();
@@ -201,31 +224,24 @@ export class MemoryStore {
         if (ids.length === 0)
             return;
         const placeholders = ids.map(() => '?').join(', ');
-        const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE id IN (${placeholders})`).all(...ids);
         const deleteAll = this.db.transaction(() => {
-            if (this._vecAvailable && rows.length > 0) {
-                const rPlaceholders = rows.map(() => '?').join(', ');
-                this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${rPlaceholders})`).run(...rows.map((r) => r.rowid));
-            }
             this.db.prepare(`DELETE FROM memory_chunks WHERE id IN (${placeholders})`).run(...ids);
         });
         deleteAll();
     }
     searchDense(embedding, limit) {
-        if (!this._vecAvailable)
-            return [];
         try {
-            const buf = Buffer.from(embedding.buffer);
+            const queryVector = this.toVectorBinding(embedding);
             const rows = this.db
-                .prepare(`SELECT mc.id, mv.distance
-           FROM memory_vec mv
-           JOIN memory_chunks mc ON mc.rowid = mv.rowid
-           WHERE mv.embedding MATCH ? AND k = ?
-           ORDER BY mv.distance`)
-                .all(buf, limit);
+                .prepare(`SELECT mc.id, vector_distance_cos(mc.embedding, vector32(?)) AS distance
+           FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vector_matches
+           JOIN memory_chunks mc ON mc.rowid = vector_matches.id
+           ORDER BY distance
+           LIMIT ?`)
+                .all(queryVector, queryVector, limit, limit);
             return rows.map((r) => ({
                 id: r.id,
-                score: Math.max(0, 1 - r.distance),
+                score: this.normalizeVectorDistance(r.distance),
             }));
         }
         catch (err) {
@@ -293,7 +309,7 @@ export class MemoryStore {
         }));
     }
     get vecAvailable() {
-        return this._vecAvailable;
+        return true;
     }
     close() {
         this.db.close();