@hanhnd/agent-kit 1.0.34 → 1.0.36

package/dist/services/memory/store.js

@@ -1,7 +1,9 @@
-import Database from 'better-sqlite3';
+import Database from 'libsql';
 import * as fs from 'fs';
 import * as path from 'path';
 import { eng, removeStopwords } from 'stopword';
+export const SCHEMA_VERSION = 3;
+const LibsqlDatabase = Database;
 export class StoreError extends Error {
     constructor(message, cause) {
         super(message);
@@ -13,7 +15,6 @@ export class StoreError extends Error {
 export class MemoryStore {
     config;
     db;
-    _vecAvailable = false;
     constructor(dbPath, config) {
         this.config = config;
         fs.mkdirSync(path.dirname(dbPath), { recursive: true });
@@ -21,10 +22,9 @@ export class MemoryStore {
     }
     openDatabase(dbPath) {
         try {
-            const db = new Database(dbPath);
+            const db = new LibsqlDatabase(dbPath);
             db.pragma('journal_mode = WAL');
             db.pragma('busy_timeout = 5000');
-            this.loadVecExtension(db);
             this.createSchema(db);
             return db;
         }
@@ -36,97 +36,107 @@ export class MemoryStore {
             catch {
                 // ignore
             }
-            const db = new Database(dbPath);
-            db.pragma('journal_mode = WAL');
-            db.pragma('busy_timeout = 5000');
-            this.loadVecExtension(db);
-            this.createSchema(db);
-            return db;
+            try {
+                const db = new LibsqlDatabase(dbPath);
+                db.pragma('journal_mode = WAL');
+                db.pragma('busy_timeout = 5000');
+                this.createSchema(db);
+                return db;
+            }
+            catch (retryErr) {
+                throw new StoreError('database open failed', retryErr);
+            }
         }
     }
-    loadVecExtension(db) {
+    createSchema(db) {
         try {
-            // eslint-disable-next-line @typescript-eslint/no-require-imports
-            const sqliteVec = require('sqlite-vec');
-            sqliteVec.load(db);
-            this._vecAvailable = true;
+            const current = this.readUserVersion(db);
+            if (current < SCHEMA_VERSION) {
+                db.exec(`
+          DROP TRIGGER IF EXISTS memory_chunks_ai;
+          DROP TRIGGER IF EXISTS memory_chunks_ad;
+          DROP TABLE IF EXISTS memory_fts;
+          DROP TABLE IF EXISTS memory_vec;
+          DROP INDEX IF EXISTS idx_memory_chunks_embedding;
+          DROP TABLE IF EXISTS memory_chunks;
+        `);
+                db.pragma(`user_version = ${SCHEMA_VERSION}`);
+            }
+            db.exec(`
+        CREATE TABLE IF NOT EXISTS memory_chunks (
+          rowid         INTEGER PRIMARY KEY AUTOINCREMENT,
+          id            TEXT NOT NULL UNIQUE,
+          source        TEXT NOT NULL,
+          source_type   TEXT NOT NULL DEFAULT 'wiki',
+          heading       TEXT NOT NULL DEFAULT '',
+          heading_level INTEGER NOT NULL DEFAULT 0,
+          content       TEXT NOT NULL,
+          line_start    INTEGER NOT NULL,
+          line_end      INTEGER NOT NULL,
+          indexed_at    INTEGER NOT NULL,
+          file_mtime_at INTEGER NOT NULL DEFAULT 0,
+          embedding     F32_BLOB(${this.config.vectorDimension}) NOT NULL
+        );
+
+        CREATE INDEX IF NOT EXISTS idx_memory_chunks_type_mtime
+          ON memory_chunks (source_type, file_mtime_at DESC);
+
+        CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
+          source,
+          heading,
+          content,
+          content='memory_chunks',
+          content_rowid='rowid'
+        );
+
+        CREATE TRIGGER IF NOT EXISTS memory_chunks_ai
+          AFTER INSERT ON memory_chunks BEGIN
+            INSERT INTO memory_fts(rowid, source, heading, content)
+              VALUES (new.rowid, new.source, new.heading, new.content);
+          END;
+
+        CREATE TRIGGER IF NOT EXISTS memory_chunks_ad
+          AFTER DELETE ON memory_chunks BEGIN
+            INSERT INTO memory_fts(memory_fts, rowid, source, heading, content)
+              VALUES ('delete', old.rowid, old.source, old.heading, old.content);
+          END;
+
+        CREATE INDEX IF NOT EXISTS idx_memory_chunks_embedding
+          ON memory_chunks (libsql_vector_idx(embedding));
+      `);
         }
         catch (err) {
-            console.warn('[memory-store] sqlite-vec unavailable, degrading to FTS5-only:', err);
-            this._vecAvailable = false;
+            throw new StoreError('schema migration failed', err);
         }
     }
-    createSchema(db) {
-        db.exec(`
-      CREATE TABLE IF NOT EXISTS memory_chunks (
-        rowid         INTEGER PRIMARY KEY AUTOINCREMENT,
-        id            TEXT NOT NULL UNIQUE,
-        source        TEXT NOT NULL,
-        heading       TEXT NOT NULL DEFAULT '',
-        heading_level INTEGER NOT NULL DEFAULT 0,
-        content       TEXT NOT NULL,
-        line_start    INTEGER NOT NULL,
-        line_end      INTEGER NOT NULL,
-        indexed_at    INTEGER NOT NULL
-      );
-
-      CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5(
-        source,
-        heading,
-        content,
-        content='memory_chunks',
-        content_rowid='rowid'
-      );
-
-      CREATE TRIGGER IF NOT EXISTS memory_chunks_ai
-        AFTER INSERT ON memory_chunks BEGIN
-          INSERT INTO memory_fts(rowid, source, heading, content)
-            VALUES (new.rowid, new.source, new.heading, new.content);
-        END;
-
-      CREATE TRIGGER IF NOT EXISTS memory_chunks_ad
-        AFTER DELETE ON memory_chunks BEGIN
-          INSERT INTO memory_fts(memory_fts, rowid, source, heading, content)
-            VALUES ('delete', old.rowid, old.source, old.heading, old.content);
-        END;
-    `);
-        if (this._vecAvailable) {
-            try {
-                db.exec(`
-          CREATE VIRTUAL TABLE IF NOT EXISTS memory_vec USING vec0(
-            embedding FLOAT[${this.config.vectorDimension}]
-          );
-        `);
-            }
-            catch (err) {
-                console.warn('[memory-store] Failed to create memory_vec, disabling vector search:', err);
-                this._vecAvailable = false;
-            }
+    readUserVersion(db) {
+        const value = db.pragma('user_version', { simple: true });
+        if (typeof value === 'number')
+            return value;
+        if (Array.isArray(value)) {
+            const [row] = value;
+            return typeof row?.user_version === 'number' ? row.user_version : 0;
+        }
+        if (value && typeof value === 'object' && 'user_version' in value) {
+            const row = value;
+            return typeof row.user_version === 'number' ? row.user_version : 0;
         }
+        return 0;
     }
     upsert(chunks, embeddings) {
         const insertChunk = this.db.prepare(`
       INSERT OR REPLACE INTO memory_chunks
-        (id, source, heading, heading_level, content, line_start, line_end, indexed_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+        (id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at, embedding)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector32(?))
     `);
-        let insertVec = null;
-        if (this._vecAvailable) {
-            insertVec = this.db.prepare(`INSERT OR REPLACE INTO memory_vec (rowid, embedding) VALUES (?, ?)`);
-        }
         const upsertAll = this.db.transaction(() => {
             for (let i = 0; i < chunks.length; i++) {
                 const chunk = chunks[i];
                 try {
+                    const vectorBinding = this.toVectorBinding(embeddings[i]);
                     // Delete existing row first to get correct rowid
                     this.db.prepare(`DELETE FROM memory_chunks WHERE id = ?`).run(chunk.id);
-                    const info = insertChunk.run(chunk.id, chunk.source, chunk.heading, chunk.headingLevel, chunk.content, chunk.lineStart, chunk.lineEnd, Date.now());
-                    if (insertVec && embeddings[i] && info.lastInsertRowid) {
-                        const rowid = Number(info.lastInsertRowid);
-                        const emb = embeddings[i];
-                        const buf = Buffer.from(emb.buffer, emb.byteOffset, emb.byteLength);
-                        insertVec.run(rowid, buf);
-                    }
+                    insertChunk.run(chunk.id, chunk.source, chunk.sourceType, chunk.heading, chunk.headingLevel, chunk.content, chunk.lineStart, chunk.lineEnd, Date.now(), chunk.fileMtimeAt, vectorBinding);
                 }
                 catch (err) {
                     console.warn(`[memory-store] Failed to upsert chunk ${chunk.id}:`, err);
@@ -140,6 +150,25 @@ export class MemoryStore {
             throw new StoreError('Full batch upsert failed', err);
         }
     }
+    toVectorBinding(embedding) {
+        if (!embedding)
+            throw new Error('missing embedding');
+        if (!(embedding instanceof Float32Array))
+            throw new Error('missing embedding');
+        if (embedding.length !== this.config.vectorDimension)
+            throw new Error('embedding dimension mismatch');
+        const values = Array.from(embedding, (value) => {
+            if (!Number.isFinite(value))
+                throw new Error('embedding contains non-finite value');
+            return value;
+        });
+        return JSON.stringify(values);
+    }
+    normalizeVectorDistance(distance) {
+        if (distance <= 0)
+            return 1;
+        return Math.max(0, Math.min(1, 1 - distance));
+    }
     hashesBySource(source) {
         const rows = this.db.prepare(`SELECT id FROM memory_chunks WHERE source = ?`).all(source);
         return new Set(rows.map((r) => r.id));
@@ -148,13 +177,23 @@ export class MemoryStore {
         const rows = this.db.prepare(`SELECT DISTINCT source FROM memory_chunks`).all();
         return rows.map((r) => r.source);
     }
+    getRecentSources(options) {
+        const limit = Math.min(Math.max(1, options.limit || 5), 50);
+        const whereClause = options.sourceType ? ' WHERE source_type = ?' : '';
+        const params = options.sourceType ? [options.sourceType, limit] : [limit];
+        try {
+            const rows = this.db
+                .prepare(`SELECT source, MAX(file_mtime_at) AS t FROM memory_chunks${whereClause} GROUP BY source ORDER BY t DESC LIMIT ?`)
+                .all(...params);
+            return rows.map((row) => ({ source: row.source, fileMtimeAt: row.t }));
+        }
+        catch (err) {
+            console.warn('[memory-store] Recent source lookup failed:', err);
+            return [];
+        }
+    }
     deleteBySource(source) {
-        const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE source = ?`).all(source);
         const deleteAll = this.db.transaction(() => {
-            if (this._vecAvailable && rows.length > 0) {
-                const placeholders = rows.map(() => '?').join(', ');
-                this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${placeholders})`).run(...rows.map((r) => r.rowid));
-            }
             this.db.prepare(`DELETE FROM memory_chunks WHERE source = ?`).run(source);
         });
         deleteAll();
@@ -163,31 +202,24 @@ export class MemoryStore {
         if (ids.length === 0)
             return;
         const placeholders = ids.map(() => '?').join(', ');
-        const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE id IN (${placeholders})`).all(...ids);
         const deleteAll = this.db.transaction(() => {
-            if (this._vecAvailable && rows.length > 0) {
-                const rPlaceholders = rows.map(() => '?').join(', ');
-                this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${rPlaceholders})`).run(...rows.map((r) => r.rowid));
-            }
             this.db.prepare(`DELETE FROM memory_chunks WHERE id IN (${placeholders})`).run(...ids);
         });
         deleteAll();
     }
     searchDense(embedding, limit) {
-        if (!this._vecAvailable)
-            return [];
         try {
-            const buf = Buffer.from(embedding.buffer);
+            const queryVector = this.toVectorBinding(embedding);
             const rows = this.db
-                .prepare(`SELECT mc.id, mv.distance
-           FROM memory_vec mv
-           JOIN memory_chunks mc ON mc.rowid = mv.rowid
-           WHERE mv.embedding MATCH ? AND k = ?
-           ORDER BY mv.distance`)
-                .all(buf, limit);
+                .prepare(`SELECT mc.id, vector_distance_cos(mc.embedding, vector32(?)) AS distance
+           FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vector_matches
+           JOIN memory_chunks mc ON mc.rowid = vector_matches.id
+           ORDER BY distance
+           LIMIT ?`)
+                .all(queryVector, queryVector, limit, limit);
             return rows.map((r) => ({
                 id: r.id,
-                score: Math.max(0, 1 - r.distance),
+                score: this.normalizeVectorDistance(r.distance),
             }));
         }
         catch (err) {
@@ -239,21 +271,23 @@ export class MemoryStore {
             return [];
         const placeholders = ids.map(() => '?').join(', ');
         const rows = this.db
-            .prepare(`SELECT id, source, heading, heading_level, content, line_start, line_end
+            .prepare(`SELECT id, source, source_type, heading, heading_level, content, line_start, line_end, file_mtime_at
          FROM memory_chunks WHERE id IN (${placeholders})`)
             .all(...ids);
         return rows.map((r) => ({
             id: r.id,
             source: r.source,
+            sourceType: r.source_type,
             heading: r.heading,
             headingLevel: r.heading_level,
             content: r.content,
             lineStart: r.line_start,
             lineEnd: r.line_end,
+            fileMtimeAt: r.file_mtime_at,
         }));
     }
     get vecAvailable() {
-        return this._vecAvailable;
+        return true;
     }
     close() {
         this.db.close();