@hanhnd/agent-kit 1.0.35 → 1.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +62 -71
- package/dist/server.js +62 -71
- package/dist/services/digest/digest-processor.test.js +5 -19
- package/dist/services/memory/store.d.ts +4 -3
- package/dist/services/memory/store.js +67 -73
- package/dist/services/memory/store.test.js +120 -15
- package/package.json +3 -5
package/dist/cli.js
CHANGED
|
@@ -15310,12 +15310,11 @@ ${sanitized}`;
|
|
|
15310
15310
|
|
|
15311
15311
|
// src/services/memory/store.ts
|
|
15312
15312
|
var import_stopword = __toESM(require_stopword_cjs(), 1);
|
|
15313
|
-
import Database from "
|
|
15313
|
+
import Database from "libsql";
|
|
15314
15314
|
import * as fs5 from "fs";
|
|
15315
|
-
import { createRequire } from "module";
|
|
15316
15315
|
import * as path7 from "path";
|
|
15317
|
-
var
|
|
15318
|
-
var
|
|
15316
|
+
var SCHEMA_VERSION = 3;
|
|
15317
|
+
var LibsqlDatabase = Database;
|
|
15319
15318
|
var StoreError = class extends Error {
|
|
15320
15319
|
constructor(message, cause) {
|
|
15321
15320
|
super(message);
|
|
@@ -15330,13 +15329,11 @@ var MemoryStore = class {
|
|
|
15330
15329
|
this.db = this.openDatabase(dbPath);
|
|
15331
15330
|
}
|
|
15332
15331
|
db;
|
|
15333
|
-
_vecAvailable = false;
|
|
15334
15332
|
openDatabase(dbPath) {
|
|
15335
15333
|
try {
|
|
15336
|
-
const db = new
|
|
15334
|
+
const db = new LibsqlDatabase(dbPath);
|
|
15337
15335
|
db.pragma("journal_mode = WAL");
|
|
15338
15336
|
db.pragma("busy_timeout = 5000");
|
|
15339
|
-
this.loadVecExtension(db);
|
|
15340
15337
|
this.createSchema(db);
|
|
15341
15338
|
return db;
|
|
15342
15339
|
} catch (err) {
|
|
@@ -15345,33 +15342,27 @@ var MemoryStore = class {
|
|
|
15345
15342
|
fs5.unlinkSync(dbPath);
|
|
15346
15343
|
} catch {
|
|
15347
15344
|
}
|
|
15348
|
-
|
|
15349
|
-
|
|
15350
|
-
|
|
15351
|
-
|
|
15352
|
-
|
|
15353
|
-
|
|
15354
|
-
|
|
15355
|
-
|
|
15356
|
-
|
|
15357
|
-
try {
|
|
15358
|
-
const sqliteVec = require2("sqlite-vec");
|
|
15359
|
-
sqliteVec.load(db);
|
|
15360
|
-
this._vecAvailable = true;
|
|
15361
|
-
} catch (err) {
|
|
15362
|
-
console.warn("[memory-store] sqlite-vec unavailable, degrading to FTS5-only:", err);
|
|
15363
|
-
this._vecAvailable = false;
|
|
15345
|
+
try {
|
|
15346
|
+
const db = new LibsqlDatabase(dbPath);
|
|
15347
|
+
db.pragma("journal_mode = WAL");
|
|
15348
|
+
db.pragma("busy_timeout = 5000");
|
|
15349
|
+
this.createSchema(db);
|
|
15350
|
+
return db;
|
|
15351
|
+
} catch (retryErr) {
|
|
15352
|
+
throw new StoreError("database open failed", retryErr);
|
|
15353
|
+
}
|
|
15364
15354
|
}
|
|
15365
15355
|
}
|
|
15366
15356
|
createSchema(db) {
|
|
15367
15357
|
try {
|
|
15368
|
-
const current =
|
|
15358
|
+
const current = this.readUserVersion(db);
|
|
15369
15359
|
if (current < SCHEMA_VERSION) {
|
|
15370
15360
|
db.exec(`
|
|
15371
15361
|
DROP TRIGGER IF EXISTS memory_chunks_ai;
|
|
15372
15362
|
DROP TRIGGER IF EXISTS memory_chunks_ad;
|
|
15373
15363
|
DROP TABLE IF EXISTS memory_fts;
|
|
15374
15364
|
DROP TABLE IF EXISTS memory_vec;
|
|
15365
|
+
DROP INDEX IF EXISTS idx_memory_chunks_embedding;
|
|
15375
15366
|
DROP TABLE IF EXISTS memory_chunks;
|
|
15376
15367
|
`);
|
|
15377
15368
|
db.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
@@ -15388,7 +15379,8 @@ var MemoryStore = class {
|
|
|
15388
15379
|
line_start INTEGER NOT NULL,
|
|
15389
15380
|
line_end INTEGER NOT NULL,
|
|
15390
15381
|
indexed_at INTEGER NOT NULL,
|
|
15391
|
-
file_mtime_at INTEGER NOT NULL DEFAULT 0
|
|
15382
|
+
file_mtime_at INTEGER NOT NULL DEFAULT 0,
|
|
15383
|
+
embedding F32_BLOB(${this.config.vectorDimension}) NOT NULL
|
|
15392
15384
|
);
|
|
15393
15385
|
|
|
15394
15386
|
CREATE INDEX IF NOT EXISTS idx_memory_chunks_type_mtime
|
|
@@ -15413,39 +15405,40 @@ var MemoryStore = class {
|
|
|
15413
15405
|
INSERT INTO memory_fts(memory_fts, rowid, source, heading, content)
|
|
15414
15406
|
VALUES ('delete', old.rowid, old.source, old.heading, old.content);
|
|
15415
15407
|
END;
|
|
15408
|
+
|
|
15409
|
+
CREATE INDEX IF NOT EXISTS idx_memory_chunks_embedding
|
|
15410
|
+
ON memory_chunks (libsql_vector_idx(embedding));
|
|
15416
15411
|
`);
|
|
15417
|
-
if (this._vecAvailable) {
|
|
15418
|
-
try {
|
|
15419
|
-
db.exec(`
|
|
15420
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS memory_vec USING vec0(
|
|
15421
|
-
embedding FLOAT[${this.config.vectorDimension}]
|
|
15422
|
-
);
|
|
15423
|
-
`);
|
|
15424
|
-
} catch (err) {
|
|
15425
|
-
console.warn("[memory-store] Failed to create memory_vec, disabling vector search:", err);
|
|
15426
|
-
this._vecAvailable = false;
|
|
15427
|
-
}
|
|
15428
|
-
}
|
|
15429
15412
|
} catch (err) {
|
|
15430
15413
|
throw new StoreError("schema migration failed", err);
|
|
15431
15414
|
}
|
|
15432
15415
|
}
|
|
15416
|
+
readUserVersion(db) {
|
|
15417
|
+
const value = db.pragma("user_version", { simple: true });
|
|
15418
|
+
if (typeof value === "number") return value;
|
|
15419
|
+
if (Array.isArray(value)) {
|
|
15420
|
+
const [row] = value;
|
|
15421
|
+
return typeof row?.user_version === "number" ? row.user_version : 0;
|
|
15422
|
+
}
|
|
15423
|
+
if (value && typeof value === "object" && "user_version" in value) {
|
|
15424
|
+
const row = value;
|
|
15425
|
+
return typeof row.user_version === "number" ? row.user_version : 0;
|
|
15426
|
+
}
|
|
15427
|
+
return 0;
|
|
15428
|
+
}
|
|
15433
15429
|
upsert(chunks, embeddings) {
|
|
15434
15430
|
const insertChunk = this.db.prepare(`
|
|
15435
15431
|
INSERT OR REPLACE INTO memory_chunks
|
|
15436
|
-
(id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at)
|
|
15437
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
15432
|
+
(id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at, embedding)
|
|
15433
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector32(?))
|
|
15438
15434
|
`);
|
|
15439
|
-
let insertVec = null;
|
|
15440
|
-
if (this._vecAvailable) {
|
|
15441
|
-
insertVec = this.db.prepare(`INSERT OR REPLACE INTO memory_vec (rowid, embedding) VALUES (?, ?)`);
|
|
15442
|
-
}
|
|
15443
15435
|
const upsertAll = this.db.transaction(() => {
|
|
15444
15436
|
for (let i = 0; i < chunks.length; i++) {
|
|
15445
15437
|
const chunk = chunks[i];
|
|
15446
15438
|
try {
|
|
15439
|
+
const vectorBinding = this.toVectorBinding(embeddings[i]);
|
|
15447
15440
|
this.db.prepare(`DELETE FROM memory_chunks WHERE id = ?`).run(chunk.id);
|
|
15448
|
-
|
|
15441
|
+
insertChunk.run(
|
|
15449
15442
|
chunk.id,
|
|
15450
15443
|
chunk.source,
|
|
15451
15444
|
chunk.sourceType,
|
|
@@ -15455,14 +15448,9 @@ var MemoryStore = class {
|
|
|
15455
15448
|
chunk.lineStart,
|
|
15456
15449
|
chunk.lineEnd,
|
|
15457
15450
|
Date.now(),
|
|
15458
|
-
chunk.fileMtimeAt
|
|
15451
|
+
chunk.fileMtimeAt,
|
|
15452
|
+
vectorBinding
|
|
15459
15453
|
);
|
|
15460
|
-
if (insertVec && embeddings[i] && info.lastInsertRowid) {
|
|
15461
|
-
const rowid = typeof info.lastInsertRowid === "bigint" ? info.lastInsertRowid : BigInt(info.lastInsertRowid);
|
|
15462
|
-
const emb = embeddings[i];
|
|
15463
|
-
const buf = Buffer.from(emb.buffer, emb.byteOffset, emb.byteLength);
|
|
15464
|
-
insertVec.run(rowid, buf);
|
|
15465
|
-
}
|
|
15466
15454
|
} catch (err) {
|
|
15467
15455
|
console.warn(`[memory-store] Failed to upsert chunk ${chunk.id}:`, err);
|
|
15468
15456
|
}
|
|
@@ -15474,6 +15462,20 @@ var MemoryStore = class {
|
|
|
15474
15462
|
throw new StoreError("Full batch upsert failed", err);
|
|
15475
15463
|
}
|
|
15476
15464
|
}
|
|
15465
|
+
toVectorBinding(embedding) {
|
|
15466
|
+
if (!embedding) throw new Error("missing embedding");
|
|
15467
|
+
if (!(embedding instanceof Float32Array)) throw new Error("missing embedding");
|
|
15468
|
+
if (embedding.length !== this.config.vectorDimension) throw new Error("embedding dimension mismatch");
|
|
15469
|
+
const values = Array.from(embedding, (value) => {
|
|
15470
|
+
if (!Number.isFinite(value)) throw new Error("embedding contains non-finite value");
|
|
15471
|
+
return value;
|
|
15472
|
+
});
|
|
15473
|
+
return JSON.stringify(values);
|
|
15474
|
+
}
|
|
15475
|
+
normalizeVectorDistance(distance) {
|
|
15476
|
+
if (distance <= 0) return 1;
|
|
15477
|
+
return Math.max(0, Math.min(1, 1 - distance));
|
|
15478
|
+
}
|
|
15477
15479
|
hashesBySource(source) {
|
|
15478
15480
|
const rows = this.db.prepare(`SELECT id FROM memory_chunks WHERE source = ?`).all(source);
|
|
15479
15481
|
return new Set(rows.map((r) => r.id));
|
|
@@ -15497,12 +15499,7 @@ var MemoryStore = class {
|
|
|
15497
15499
|
}
|
|
15498
15500
|
}
|
|
15499
15501
|
deleteBySource(source) {
|
|
15500
|
-
const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE source = ?`).all(source);
|
|
15501
15502
|
const deleteAll = this.db.transaction(() => {
|
|
15502
|
-
if (this._vecAvailable && rows.length > 0) {
|
|
15503
|
-
const placeholders = rows.map(() => "?").join(", ");
|
|
15504
|
-
this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${placeholders})`).run(...rows.map((r) => r.rowid));
|
|
15505
|
-
}
|
|
15506
15503
|
this.db.prepare(`DELETE FROM memory_chunks WHERE source = ?`).run(source);
|
|
15507
15504
|
});
|
|
15508
15505
|
deleteAll();
|
|
@@ -15510,30 +15507,24 @@ var MemoryStore = class {
|
|
|
15510
15507
|
deleteByIds(ids) {
|
|
15511
15508
|
if (ids.length === 0) return;
|
|
15512
15509
|
const placeholders = ids.map(() => "?").join(", ");
|
|
15513
|
-
const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE id IN (${placeholders})`).all(...ids);
|
|
15514
15510
|
const deleteAll = this.db.transaction(() => {
|
|
15515
|
-
if (this._vecAvailable && rows.length > 0) {
|
|
15516
|
-
const rPlaceholders = rows.map(() => "?").join(", ");
|
|
15517
|
-
this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${rPlaceholders})`).run(...rows.map((r) => r.rowid));
|
|
15518
|
-
}
|
|
15519
15511
|
this.db.prepare(`DELETE FROM memory_chunks WHERE id IN (${placeholders})`).run(...ids);
|
|
15520
15512
|
});
|
|
15521
15513
|
deleteAll();
|
|
15522
15514
|
}
|
|
15523
15515
|
searchDense(embedding, limit) {
|
|
15524
|
-
if (!this._vecAvailable) return [];
|
|
15525
15516
|
try {
|
|
15526
|
-
const
|
|
15517
|
+
const queryVector = this.toVectorBinding(embedding);
|
|
15527
15518
|
const rows = this.db.prepare(
|
|
15528
|
-
`SELECT mc.id,
|
|
15529
|
-
FROM
|
|
15530
|
-
JOIN memory_chunks mc ON mc.rowid =
|
|
15531
|
-
|
|
15532
|
-
|
|
15533
|
-
).all(
|
|
15519
|
+
`SELECT mc.id, vector_distance_cos(mc.embedding, vector32(?)) AS distance
|
|
15520
|
+
FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vector_matches
|
|
15521
|
+
JOIN memory_chunks mc ON mc.rowid = vector_matches.id
|
|
15522
|
+
ORDER BY distance
|
|
15523
|
+
LIMIT ?`
|
|
15524
|
+
).all(queryVector, queryVector, limit, limit);
|
|
15534
15525
|
return rows.map((r) => ({
|
|
15535
15526
|
id: r.id,
|
|
15536
|
-
score:
|
|
15527
|
+
score: this.normalizeVectorDistance(r.distance)
|
|
15537
15528
|
}));
|
|
15538
15529
|
} catch (err) {
|
|
15539
15530
|
console.warn("[memory-store] Dense search failed:", err);
|
|
@@ -15588,7 +15579,7 @@ var MemoryStore = class {
|
|
|
15588
15579
|
}));
|
|
15589
15580
|
}
|
|
15590
15581
|
get vecAvailable() {
|
|
15591
|
-
return
|
|
15582
|
+
return true;
|
|
15592
15583
|
}
|
|
15593
15584
|
close() {
|
|
15594
15585
|
this.db.close();
|
package/dist/server.js
CHANGED
|
@@ -37175,12 +37175,11 @@ ${content}
|
|
|
37175
37175
|
|
|
37176
37176
|
// src/services/memory/store.ts
|
|
37177
37177
|
var import_stopword = __toESM(require_stopword_cjs(), 1);
|
|
37178
|
-
import Database from "
|
|
37178
|
+
import Database from "libsql";
|
|
37179
37179
|
import * as fs7 from "fs";
|
|
37180
|
-
import { createRequire } from "module";
|
|
37181
37180
|
import * as path9 from "path";
|
|
37182
|
-
var
|
|
37183
|
-
var
|
|
37181
|
+
var SCHEMA_VERSION = 3;
|
|
37182
|
+
var LibsqlDatabase = Database;
|
|
37184
37183
|
var StoreError = class extends Error {
|
|
37185
37184
|
constructor(message, cause) {
|
|
37186
37185
|
super(message);
|
|
@@ -37195,13 +37194,11 @@ var MemoryStore = class {
|
|
|
37195
37194
|
this.db = this.openDatabase(dbPath);
|
|
37196
37195
|
}
|
|
37197
37196
|
db;
|
|
37198
|
-
_vecAvailable = false;
|
|
37199
37197
|
openDatabase(dbPath) {
|
|
37200
37198
|
try {
|
|
37201
|
-
const db = new
|
|
37199
|
+
const db = new LibsqlDatabase(dbPath);
|
|
37202
37200
|
db.pragma("journal_mode = WAL");
|
|
37203
37201
|
db.pragma("busy_timeout = 5000");
|
|
37204
|
-
this.loadVecExtension(db);
|
|
37205
37202
|
this.createSchema(db);
|
|
37206
37203
|
return db;
|
|
37207
37204
|
} catch (err) {
|
|
@@ -37210,33 +37207,27 @@ var MemoryStore = class {
|
|
|
37210
37207
|
fs7.unlinkSync(dbPath);
|
|
37211
37208
|
} catch {
|
|
37212
37209
|
}
|
|
37213
|
-
|
|
37214
|
-
|
|
37215
|
-
|
|
37216
|
-
|
|
37217
|
-
|
|
37218
|
-
|
|
37219
|
-
|
|
37220
|
-
|
|
37221
|
-
|
|
37222
|
-
try {
|
|
37223
|
-
const sqliteVec = require2("sqlite-vec");
|
|
37224
|
-
sqliteVec.load(db);
|
|
37225
|
-
this._vecAvailable = true;
|
|
37226
|
-
} catch (err) {
|
|
37227
|
-
console.warn("[memory-store] sqlite-vec unavailable, degrading to FTS5-only:", err);
|
|
37228
|
-
this._vecAvailable = false;
|
|
37210
|
+
try {
|
|
37211
|
+
const db = new LibsqlDatabase(dbPath);
|
|
37212
|
+
db.pragma("journal_mode = WAL");
|
|
37213
|
+
db.pragma("busy_timeout = 5000");
|
|
37214
|
+
this.createSchema(db);
|
|
37215
|
+
return db;
|
|
37216
|
+
} catch (retryErr) {
|
|
37217
|
+
throw new StoreError("database open failed", retryErr);
|
|
37218
|
+
}
|
|
37229
37219
|
}
|
|
37230
37220
|
}
|
|
37231
37221
|
createSchema(db) {
|
|
37232
37222
|
try {
|
|
37233
|
-
const current =
|
|
37223
|
+
const current = this.readUserVersion(db);
|
|
37234
37224
|
if (current < SCHEMA_VERSION) {
|
|
37235
37225
|
db.exec(`
|
|
37236
37226
|
DROP TRIGGER IF EXISTS memory_chunks_ai;
|
|
37237
37227
|
DROP TRIGGER IF EXISTS memory_chunks_ad;
|
|
37238
37228
|
DROP TABLE IF EXISTS memory_fts;
|
|
37239
37229
|
DROP TABLE IF EXISTS memory_vec;
|
|
37230
|
+
DROP INDEX IF EXISTS idx_memory_chunks_embedding;
|
|
37240
37231
|
DROP TABLE IF EXISTS memory_chunks;
|
|
37241
37232
|
`);
|
|
37242
37233
|
db.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
@@ -37253,7 +37244,8 @@ var MemoryStore = class {
|
|
|
37253
37244
|
line_start INTEGER NOT NULL,
|
|
37254
37245
|
line_end INTEGER NOT NULL,
|
|
37255
37246
|
indexed_at INTEGER NOT NULL,
|
|
37256
|
-
file_mtime_at INTEGER NOT NULL DEFAULT 0
|
|
37247
|
+
file_mtime_at INTEGER NOT NULL DEFAULT 0,
|
|
37248
|
+
embedding F32_BLOB(${this.config.vectorDimension}) NOT NULL
|
|
37257
37249
|
);
|
|
37258
37250
|
|
|
37259
37251
|
CREATE INDEX IF NOT EXISTS idx_memory_chunks_type_mtime
|
|
@@ -37278,39 +37270,40 @@ var MemoryStore = class {
|
|
|
37278
37270
|
INSERT INTO memory_fts(memory_fts, rowid, source, heading, content)
|
|
37279
37271
|
VALUES ('delete', old.rowid, old.source, old.heading, old.content);
|
|
37280
37272
|
END;
|
|
37273
|
+
|
|
37274
|
+
CREATE INDEX IF NOT EXISTS idx_memory_chunks_embedding
|
|
37275
|
+
ON memory_chunks (libsql_vector_idx(embedding));
|
|
37281
37276
|
`);
|
|
37282
|
-
if (this._vecAvailable) {
|
|
37283
|
-
try {
|
|
37284
|
-
db.exec(`
|
|
37285
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS memory_vec USING vec0(
|
|
37286
|
-
embedding FLOAT[${this.config.vectorDimension}]
|
|
37287
|
-
);
|
|
37288
|
-
`);
|
|
37289
|
-
} catch (err) {
|
|
37290
|
-
console.warn("[memory-store] Failed to create memory_vec, disabling vector search:", err);
|
|
37291
|
-
this._vecAvailable = false;
|
|
37292
|
-
}
|
|
37293
|
-
}
|
|
37294
37277
|
} catch (err) {
|
|
37295
37278
|
throw new StoreError("schema migration failed", err);
|
|
37296
37279
|
}
|
|
37297
37280
|
}
|
|
37281
|
+
readUserVersion(db) {
|
|
37282
|
+
const value = db.pragma("user_version", { simple: true });
|
|
37283
|
+
if (typeof value === "number") return value;
|
|
37284
|
+
if (Array.isArray(value)) {
|
|
37285
|
+
const [row] = value;
|
|
37286
|
+
return typeof row?.user_version === "number" ? row.user_version : 0;
|
|
37287
|
+
}
|
|
37288
|
+
if (value && typeof value === "object" && "user_version" in value) {
|
|
37289
|
+
const row = value;
|
|
37290
|
+
return typeof row.user_version === "number" ? row.user_version : 0;
|
|
37291
|
+
}
|
|
37292
|
+
return 0;
|
|
37293
|
+
}
|
|
37298
37294
|
upsert(chunks, embeddings) {
|
|
37299
37295
|
const insertChunk = this.db.prepare(`
|
|
37300
37296
|
INSERT OR REPLACE INTO memory_chunks
|
|
37301
|
-
(id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at)
|
|
37302
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
37297
|
+
(id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at, embedding)
|
|
37298
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector32(?))
|
|
37303
37299
|
`);
|
|
37304
|
-
let insertVec = null;
|
|
37305
|
-
if (this._vecAvailable) {
|
|
37306
|
-
insertVec = this.db.prepare(`INSERT OR REPLACE INTO memory_vec (rowid, embedding) VALUES (?, ?)`);
|
|
37307
|
-
}
|
|
37308
37300
|
const upsertAll = this.db.transaction(() => {
|
|
37309
37301
|
for (let i = 0; i < chunks.length; i++) {
|
|
37310
37302
|
const chunk = chunks[i];
|
|
37311
37303
|
try {
|
|
37304
|
+
const vectorBinding = this.toVectorBinding(embeddings[i]);
|
|
37312
37305
|
this.db.prepare(`DELETE FROM memory_chunks WHERE id = ?`).run(chunk.id);
|
|
37313
|
-
|
|
37306
|
+
insertChunk.run(
|
|
37314
37307
|
chunk.id,
|
|
37315
37308
|
chunk.source,
|
|
37316
37309
|
chunk.sourceType,
|
|
@@ -37320,14 +37313,9 @@ var MemoryStore = class {
|
|
|
37320
37313
|
chunk.lineStart,
|
|
37321
37314
|
chunk.lineEnd,
|
|
37322
37315
|
Date.now(),
|
|
37323
|
-
chunk.fileMtimeAt
|
|
37316
|
+
chunk.fileMtimeAt,
|
|
37317
|
+
vectorBinding
|
|
37324
37318
|
);
|
|
37325
|
-
if (insertVec && embeddings[i] && info.lastInsertRowid) {
|
|
37326
|
-
const rowid = typeof info.lastInsertRowid === "bigint" ? info.lastInsertRowid : BigInt(info.lastInsertRowid);
|
|
37327
|
-
const emb = embeddings[i];
|
|
37328
|
-
const buf = Buffer.from(emb.buffer, emb.byteOffset, emb.byteLength);
|
|
37329
|
-
insertVec.run(rowid, buf);
|
|
37330
|
-
}
|
|
37331
37319
|
} catch (err) {
|
|
37332
37320
|
console.warn(`[memory-store] Failed to upsert chunk ${chunk.id}:`, err);
|
|
37333
37321
|
}
|
|
@@ -37339,6 +37327,20 @@ var MemoryStore = class {
|
|
|
37339
37327
|
throw new StoreError("Full batch upsert failed", err);
|
|
37340
37328
|
}
|
|
37341
37329
|
}
|
|
37330
|
+
toVectorBinding(embedding) {
|
|
37331
|
+
if (!embedding) throw new Error("missing embedding");
|
|
37332
|
+
if (!(embedding instanceof Float32Array)) throw new Error("missing embedding");
|
|
37333
|
+
if (embedding.length !== this.config.vectorDimension) throw new Error("embedding dimension mismatch");
|
|
37334
|
+
const values = Array.from(embedding, (value) => {
|
|
37335
|
+
if (!Number.isFinite(value)) throw new Error("embedding contains non-finite value");
|
|
37336
|
+
return value;
|
|
37337
|
+
});
|
|
37338
|
+
return JSON.stringify(values);
|
|
37339
|
+
}
|
|
37340
|
+
normalizeVectorDistance(distance) {
|
|
37341
|
+
if (distance <= 0) return 1;
|
|
37342
|
+
return Math.max(0, Math.min(1, 1 - distance));
|
|
37343
|
+
}
|
|
37342
37344
|
hashesBySource(source) {
|
|
37343
37345
|
const rows = this.db.prepare(`SELECT id FROM memory_chunks WHERE source = ?`).all(source);
|
|
37344
37346
|
return new Set(rows.map((r) => r.id));
|
|
@@ -37362,12 +37364,7 @@ var MemoryStore = class {
|
|
|
37362
37364
|
}
|
|
37363
37365
|
}
|
|
37364
37366
|
deleteBySource(source) {
|
|
37365
|
-
const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE source = ?`).all(source);
|
|
37366
37367
|
const deleteAll = this.db.transaction(() => {
|
|
37367
|
-
if (this._vecAvailable && rows.length > 0) {
|
|
37368
|
-
const placeholders = rows.map(() => "?").join(", ");
|
|
37369
|
-
this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${placeholders})`).run(...rows.map((r) => r.rowid));
|
|
37370
|
-
}
|
|
37371
37368
|
this.db.prepare(`DELETE FROM memory_chunks WHERE source = ?`).run(source);
|
|
37372
37369
|
});
|
|
37373
37370
|
deleteAll();
|
|
@@ -37375,30 +37372,24 @@ var MemoryStore = class {
|
|
|
37375
37372
|
deleteByIds(ids) {
|
|
37376
37373
|
if (ids.length === 0) return;
|
|
37377
37374
|
const placeholders = ids.map(() => "?").join(", ");
|
|
37378
|
-
const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE id IN (${placeholders})`).all(...ids);
|
|
37379
37375
|
const deleteAll = this.db.transaction(() => {
|
|
37380
|
-
if (this._vecAvailable && rows.length > 0) {
|
|
37381
|
-
const rPlaceholders = rows.map(() => "?").join(", ");
|
|
37382
|
-
this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${rPlaceholders})`).run(...rows.map((r) => r.rowid));
|
|
37383
|
-
}
|
|
37384
37376
|
this.db.prepare(`DELETE FROM memory_chunks WHERE id IN (${placeholders})`).run(...ids);
|
|
37385
37377
|
});
|
|
37386
37378
|
deleteAll();
|
|
37387
37379
|
}
|
|
37388
37380
|
searchDense(embedding, limit) {
|
|
37389
|
-
if (!this._vecAvailable) return [];
|
|
37390
37381
|
try {
|
|
37391
|
-
const
|
|
37382
|
+
const queryVector = this.toVectorBinding(embedding);
|
|
37392
37383
|
const rows = this.db.prepare(
|
|
37393
|
-
`SELECT mc.id,
|
|
37394
|
-
FROM
|
|
37395
|
-
JOIN memory_chunks mc ON mc.rowid =
|
|
37396
|
-
|
|
37397
|
-
|
|
37398
|
-
).all(
|
|
37384
|
+
`SELECT mc.id, vector_distance_cos(mc.embedding, vector32(?)) AS distance
|
|
37385
|
+
FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vector_matches
|
|
37386
|
+
JOIN memory_chunks mc ON mc.rowid = vector_matches.id
|
|
37387
|
+
ORDER BY distance
|
|
37388
|
+
LIMIT ?`
|
|
37389
|
+
).all(queryVector, queryVector, limit, limit);
|
|
37399
37390
|
return rows.map((r) => ({
|
|
37400
37391
|
id: r.id,
|
|
37401
|
-
score:
|
|
37392
|
+
score: this.normalizeVectorDistance(r.distance)
|
|
37402
37393
|
}));
|
|
37403
37394
|
} catch (err) {
|
|
37404
37395
|
console.warn("[memory-store] Dense search failed:", err);
|
|
@@ -37453,7 +37444,7 @@ var MemoryStore = class {
|
|
|
37453
37444
|
}));
|
|
37454
37445
|
}
|
|
37455
37446
|
get vecAvailable() {
|
|
37456
|
-
return
|
|
37447
|
+
return true;
|
|
37457
37448
|
}
|
|
37458
37449
|
close() {
|
|
37459
37450
|
this.db.close();
|
|
@@ -1,8 +1,6 @@
|
|
|
1
1
|
import * as assert from 'node:assert/strict';
|
|
2
2
|
import * as fs from 'node:fs';
|
|
3
3
|
import * as path from 'node:path';
|
|
4
|
-
import Database from 'better-sqlite3';
|
|
5
|
-
import { load as loadSqliteVec } from 'sqlite-vec';
|
|
6
4
|
import { afterEach, describe, test } from 'node:test';
|
|
7
5
|
import { createTempDirTracker } from '../../utils/temp-dir.test.js';
|
|
8
6
|
import { DEFAULT_DIGEST_MODEL_ID } from './constants.js';
|
|
@@ -53,7 +51,7 @@ describe('digestConversationFile', () => {
|
|
|
53
51
|
assert.equal(result.contentHash, input.contentHash);
|
|
54
52
|
assert.match(path.basename(result.markdown), /^[a-f0-9]{16}-conv\.md$/);
|
|
55
53
|
});
|
|
56
|
-
test('indexes existing provisional digest with
|
|
54
|
+
test('indexes existing provisional digest with searchable store vector data', async () => {
|
|
57
55
|
const workspace = tempDirs.makeTempDir('digest-processor-index-');
|
|
58
56
|
const inputPath = path.join(workspace, '.agent-kit', 'wiki', 'archive', 'conversations', 'conv.md');
|
|
59
57
|
fs.mkdirSync(path.dirname(inputPath), { recursive: true });
|
|
@@ -74,22 +72,10 @@ describe('digestConversationFile', () => {
|
|
|
74
72
|
});
|
|
75
73
|
assert.equal(result.indexed, true);
|
|
76
74
|
assert.equal(result.markdown, markdownPath);
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
const row = db
|
|
82
|
-
.prepare(`SELECT COUNT(*) AS count
|
|
83
|
-
FROM memory_vec mv
|
|
84
|
-
JOIN memory_chunks mc ON mc.rowid = mv.rowid
|
|
85
|
-
WHERE mc.source = ?`)
|
|
86
|
-
.get(path.relative(config.wikiDir, markdownPath));
|
|
87
|
-
assert.ok(row.count > 0, `Expected vector rows for ${markdownPath}`);
|
|
88
|
-
}
|
|
89
|
-
finally {
|
|
90
|
-
db.close();
|
|
91
|
-
}
|
|
92
|
-
}
|
|
75
|
+
const denseResults = store.searchDense(new Float32Array(384).fill(0.05), 5);
|
|
76
|
+
const indexedSource = path.relative(config.wikiDir, markdownPath);
|
|
77
|
+
const indexedChunks = store.getChunksByIds(denseResults.map((denseResult) => denseResult.id));
|
|
78
|
+
assert.ok(indexedChunks.some((chunk) => chunk.source === indexedSource), `Expected dense search to return a chunk for ${indexedSource}`);
|
|
93
79
|
}
|
|
94
80
|
finally {
|
|
95
81
|
store.close();
|
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
import type { MemoryChunk, MemoryConfig, RecentSource, SourceType } from './types.js';
|
|
2
|
-
export declare const SCHEMA_VERSION =
|
|
2
|
+
export declare const SCHEMA_VERSION = 3;
|
|
3
3
|
export declare class StoreError extends Error {
|
|
4
4
|
constructor(message: string, cause?: unknown);
|
|
5
5
|
}
|
|
6
6
|
export declare class MemoryStore {
|
|
7
7
|
private readonly config;
|
|
8
8
|
private db;
|
|
9
|
-
private _vecAvailable;
|
|
10
9
|
constructor(dbPath: string, config: MemoryConfig);
|
|
11
10
|
private openDatabase;
|
|
12
|
-
private loadVecExtension;
|
|
13
11
|
private createSchema;
|
|
12
|
+
private readUserVersion;
|
|
14
13
|
upsert(chunks: MemoryChunk[], embeddings: Float32Array[]): void;
|
|
14
|
+
private toVectorBinding;
|
|
15
|
+
private normalizeVectorDistance;
|
|
15
16
|
hashesBySource(source: string): Set<string>;
|
|
16
17
|
indexedSources(): string[];
|
|
17
18
|
getRecentSources(options: {
|
|
@@ -1,10 +1,9 @@
|
|
|
1
|
-
import Database from '
|
|
1
|
+
import Database from 'libsql';
|
|
2
2
|
import * as fs from 'fs';
|
|
3
|
-
import { createRequire } from 'module';
|
|
4
3
|
import * as path from 'path';
|
|
5
4
|
import { eng, removeStopwords } from 'stopword';
|
|
6
|
-
const
|
|
7
|
-
|
|
5
|
+
export const SCHEMA_VERSION = 3;
|
|
6
|
+
const LibsqlDatabase = Database;
|
|
8
7
|
export class StoreError extends Error {
|
|
9
8
|
constructor(message, cause) {
|
|
10
9
|
super(message);
|
|
@@ -16,7 +15,6 @@ export class StoreError extends Error {
|
|
|
16
15
|
export class MemoryStore {
|
|
17
16
|
config;
|
|
18
17
|
db;
|
|
19
|
-
_vecAvailable = false;
|
|
20
18
|
constructor(dbPath, config) {
|
|
21
19
|
this.config = config;
|
|
22
20
|
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
@@ -24,10 +22,9 @@ export class MemoryStore {
|
|
|
24
22
|
}
|
|
25
23
|
openDatabase(dbPath) {
|
|
26
24
|
try {
|
|
27
|
-
const db = new
|
|
25
|
+
const db = new LibsqlDatabase(dbPath);
|
|
28
26
|
db.pragma('journal_mode = WAL');
|
|
29
27
|
db.pragma('busy_timeout = 5000');
|
|
30
|
-
this.loadVecExtension(db);
|
|
31
28
|
this.createSchema(db);
|
|
32
29
|
return db;
|
|
33
30
|
}
|
|
@@ -39,34 +36,28 @@ export class MemoryStore {
|
|
|
39
36
|
catch {
|
|
40
37
|
// ignore
|
|
41
38
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
const sqliteVec = require('sqlite-vec');
|
|
53
|
-
sqliteVec.load(db);
|
|
54
|
-
this._vecAvailable = true;
|
|
55
|
-
}
|
|
56
|
-
catch (err) {
|
|
57
|
-
console.warn('[memory-store] sqlite-vec unavailable, degrading to FTS5-only:', err);
|
|
58
|
-
this._vecAvailable = false;
|
|
39
|
+
try {
|
|
40
|
+
const db = new LibsqlDatabase(dbPath);
|
|
41
|
+
db.pragma('journal_mode = WAL');
|
|
42
|
+
db.pragma('busy_timeout = 5000');
|
|
43
|
+
this.createSchema(db);
|
|
44
|
+
return db;
|
|
45
|
+
}
|
|
46
|
+
catch (retryErr) {
|
|
47
|
+
throw new StoreError('database open failed', retryErr);
|
|
48
|
+
}
|
|
59
49
|
}
|
|
60
50
|
}
|
|
61
51
|
createSchema(db) {
|
|
62
52
|
try {
|
|
63
|
-
const current =
|
|
53
|
+
const current = this.readUserVersion(db);
|
|
64
54
|
if (current < SCHEMA_VERSION) {
|
|
65
55
|
db.exec(`
|
|
66
56
|
DROP TRIGGER IF EXISTS memory_chunks_ai;
|
|
67
57
|
DROP TRIGGER IF EXISTS memory_chunks_ad;
|
|
68
58
|
DROP TABLE IF EXISTS memory_fts;
|
|
69
59
|
DROP TABLE IF EXISTS memory_vec;
|
|
60
|
+
DROP INDEX IF EXISTS idx_memory_chunks_embedding;
|
|
70
61
|
DROP TABLE IF EXISTS memory_chunks;
|
|
71
62
|
`);
|
|
72
63
|
db.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
@@ -83,7 +74,8 @@ export class MemoryStore {
|
|
|
83
74
|
line_start INTEGER NOT NULL,
|
|
84
75
|
line_end INTEGER NOT NULL,
|
|
85
76
|
indexed_at INTEGER NOT NULL,
|
|
86
|
-
file_mtime_at INTEGER NOT NULL DEFAULT 0
|
|
77
|
+
file_mtime_at INTEGER NOT NULL DEFAULT 0,
|
|
78
|
+
embedding F32_BLOB(${this.config.vectorDimension}) NOT NULL
|
|
87
79
|
);
|
|
88
80
|
|
|
89
81
|
CREATE INDEX IF NOT EXISTS idx_memory_chunks_type_mtime
|
|
@@ -108,48 +100,43 @@ export class MemoryStore {
|
|
|
108
100
|
INSERT INTO memory_fts(memory_fts, rowid, source, heading, content)
|
|
109
101
|
VALUES ('delete', old.rowid, old.source, old.heading, old.content);
|
|
110
102
|
END;
|
|
103
|
+
|
|
104
|
+
CREATE INDEX IF NOT EXISTS idx_memory_chunks_embedding
|
|
105
|
+
ON memory_chunks (libsql_vector_idx(embedding));
|
|
111
106
|
`);
|
|
112
|
-
if (this._vecAvailable) {
|
|
113
|
-
try {
|
|
114
|
-
db.exec(`
|
|
115
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS memory_vec USING vec0(
|
|
116
|
-
embedding FLOAT[${this.config.vectorDimension}]
|
|
117
|
-
);
|
|
118
|
-
`);
|
|
119
|
-
}
|
|
120
|
-
catch (err) {
|
|
121
|
-
console.warn('[memory-store] Failed to create memory_vec, disabling vector search:', err);
|
|
122
|
-
this._vecAvailable = false;
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
107
|
}
|
|
126
108
|
catch (err) {
|
|
127
109
|
throw new StoreError('schema migration failed', err);
|
|
128
110
|
}
|
|
129
111
|
}
|
|
112
|
+
readUserVersion(db) {
|
|
113
|
+
const value = db.pragma('user_version', { simple: true });
|
|
114
|
+
if (typeof value === 'number')
|
|
115
|
+
return value;
|
|
116
|
+
if (Array.isArray(value)) {
|
|
117
|
+
const [row] = value;
|
|
118
|
+
return typeof row?.user_version === 'number' ? row.user_version : 0;
|
|
119
|
+
}
|
|
120
|
+
if (value && typeof value === 'object' && 'user_version' in value) {
|
|
121
|
+
const row = value;
|
|
122
|
+
return typeof row.user_version === 'number' ? row.user_version : 0;
|
|
123
|
+
}
|
|
124
|
+
return 0;
|
|
125
|
+
}
|
|
130
126
|
upsert(chunks, embeddings) {
|
|
131
127
|
const insertChunk = this.db.prepare(`
|
|
132
128
|
INSERT OR REPLACE INTO memory_chunks
|
|
133
|
-
(id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at)
|
|
134
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
129
|
+
(id, source, source_type, heading, heading_level, content, line_start, line_end, indexed_at, file_mtime_at, embedding)
|
|
130
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, vector32(?))
|
|
135
131
|
`);
|
|
136
|
-
let insertVec = null;
|
|
137
|
-
if (this._vecAvailable) {
|
|
138
|
-
insertVec = this.db.prepare(`INSERT OR REPLACE INTO memory_vec (rowid, embedding) VALUES (?, ?)`);
|
|
139
|
-
}
|
|
140
132
|
const upsertAll = this.db.transaction(() => {
|
|
141
133
|
for (let i = 0; i < chunks.length; i++) {
|
|
142
134
|
const chunk = chunks[i];
|
|
143
135
|
try {
|
|
136
|
+
const vectorBinding = this.toVectorBinding(embeddings[i]);
|
|
144
137
|
// Delete existing row first to get correct rowid
|
|
145
138
|
this.db.prepare(`DELETE FROM memory_chunks WHERE id = ?`).run(chunk.id);
|
|
146
|
-
|
|
147
|
-
if (insertVec && embeddings[i] && info.lastInsertRowid) {
|
|
148
|
-
const rowid = typeof info.lastInsertRowid === 'bigint' ? info.lastInsertRowid : BigInt(info.lastInsertRowid);
|
|
149
|
-
const emb = embeddings[i];
|
|
150
|
-
const buf = Buffer.from(emb.buffer, emb.byteOffset, emb.byteLength);
|
|
151
|
-
insertVec.run(rowid, buf);
|
|
152
|
-
}
|
|
139
|
+
insertChunk.run(chunk.id, chunk.source, chunk.sourceType, chunk.heading, chunk.headingLevel, chunk.content, chunk.lineStart, chunk.lineEnd, Date.now(), chunk.fileMtimeAt, vectorBinding);
|
|
153
140
|
}
|
|
154
141
|
catch (err) {
|
|
155
142
|
console.warn(`[memory-store] Failed to upsert chunk ${chunk.id}:`, err);
|
|
@@ -163,6 +150,25 @@ export class MemoryStore {
|
|
|
163
150
|
throw new StoreError('Full batch upsert failed', err);
|
|
164
151
|
}
|
|
165
152
|
}
|
|
153
|
+
toVectorBinding(embedding) {
|
|
154
|
+
if (!embedding)
|
|
155
|
+
throw new Error('missing embedding');
|
|
156
|
+
if (!(embedding instanceof Float32Array))
|
|
157
|
+
throw new Error('missing embedding');
|
|
158
|
+
if (embedding.length !== this.config.vectorDimension)
|
|
159
|
+
throw new Error('embedding dimension mismatch');
|
|
160
|
+
const values = Array.from(embedding, (value) => {
|
|
161
|
+
if (!Number.isFinite(value))
|
|
162
|
+
throw new Error('embedding contains non-finite value');
|
|
163
|
+
return value;
|
|
164
|
+
});
|
|
165
|
+
return JSON.stringify(values);
|
|
166
|
+
}
|
|
167
|
+
normalizeVectorDistance(distance) {
|
|
168
|
+
if (distance <= 0)
|
|
169
|
+
return 1;
|
|
170
|
+
return Math.max(0, Math.min(1, 1 - distance));
|
|
171
|
+
}
|
|
166
172
|
hashesBySource(source) {
|
|
167
173
|
const rows = this.db.prepare(`SELECT id FROM memory_chunks WHERE source = ?`).all(source);
|
|
168
174
|
return new Set(rows.map((r) => r.id));
|
|
@@ -187,12 +193,7 @@ export class MemoryStore {
|
|
|
187
193
|
}
|
|
188
194
|
}
|
|
189
195
|
deleteBySource(source) {
|
|
190
|
-
const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE source = ?`).all(source);
|
|
191
196
|
const deleteAll = this.db.transaction(() => {
|
|
192
|
-
if (this._vecAvailable && rows.length > 0) {
|
|
193
|
-
const placeholders = rows.map(() => '?').join(', ');
|
|
194
|
-
this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${placeholders})`).run(...rows.map((r) => r.rowid));
|
|
195
|
-
}
|
|
196
197
|
this.db.prepare(`DELETE FROM memory_chunks WHERE source = ?`).run(source);
|
|
197
198
|
});
|
|
198
199
|
deleteAll();
|
|
@@ -201,31 +202,24 @@ export class MemoryStore {
|
|
|
201
202
|
if (ids.length === 0)
|
|
202
203
|
return;
|
|
203
204
|
const placeholders = ids.map(() => '?').join(', ');
|
|
204
|
-
const rows = this.db.prepare(`SELECT rowid FROM memory_chunks WHERE id IN (${placeholders})`).all(...ids);
|
|
205
205
|
const deleteAll = this.db.transaction(() => {
|
|
206
|
-
if (this._vecAvailable && rows.length > 0) {
|
|
207
|
-
const rPlaceholders = rows.map(() => '?').join(', ');
|
|
208
|
-
this.db.prepare(`DELETE FROM memory_vec WHERE rowid IN (${rPlaceholders})`).run(...rows.map((r) => r.rowid));
|
|
209
|
-
}
|
|
210
206
|
this.db.prepare(`DELETE FROM memory_chunks WHERE id IN (${placeholders})`).run(...ids);
|
|
211
207
|
});
|
|
212
208
|
deleteAll();
|
|
213
209
|
}
|
|
214
210
|
searchDense(embedding, limit) {
|
|
215
|
-
if (!this._vecAvailable)
|
|
216
|
-
return [];
|
|
217
211
|
try {
|
|
218
|
-
const
|
|
212
|
+
const queryVector = this.toVectorBinding(embedding);
|
|
219
213
|
const rows = this.db
|
|
220
|
-
.prepare(`SELECT mc.id,
|
|
221
|
-
FROM
|
|
222
|
-
JOIN memory_chunks mc ON mc.rowid =
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
.all(
|
|
214
|
+
.prepare(`SELECT mc.id, vector_distance_cos(mc.embedding, vector32(?)) AS distance
|
|
215
|
+
FROM vector_top_k('idx_memory_chunks_embedding', vector32(?), ?) AS vector_matches
|
|
216
|
+
JOIN memory_chunks mc ON mc.rowid = vector_matches.id
|
|
217
|
+
ORDER BY distance
|
|
218
|
+
LIMIT ?`)
|
|
219
|
+
.all(queryVector, queryVector, limit, limit);
|
|
226
220
|
return rows.map((r) => ({
|
|
227
221
|
id: r.id,
|
|
228
|
-
score:
|
|
222
|
+
score: this.normalizeVectorDistance(r.distance),
|
|
229
223
|
}));
|
|
230
224
|
}
|
|
231
225
|
catch (err) {
|
|
@@ -293,7 +287,7 @@ export class MemoryStore {
|
|
|
293
287
|
}));
|
|
294
288
|
}
|
|
295
289
|
get vecAvailable() {
|
|
296
|
-
return
|
|
290
|
+
return true;
|
|
297
291
|
}
|
|
298
292
|
close() {
|
|
299
293
|
this.db.close();
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
+
import Database from 'libsql';
|
|
1
2
|
import * as assert from 'node:assert/strict';
|
|
2
3
|
import * as fs from 'node:fs';
|
|
3
4
|
import * as os from 'node:os';
|
|
4
5
|
import * as path from 'node:path';
|
|
5
|
-
import Database from 'better-sqlite3';
|
|
6
6
|
import { after, before, describe, test } from 'node:test';
|
|
7
7
|
import { MemoryStore, SCHEMA_VERSION } from './store.js';
|
|
8
8
|
import { EmbeddingModelName } from './types.js';
|
|
@@ -29,6 +29,25 @@ function makeChunk(overrides = {}) {
|
|
|
29
29
|
...overrides,
|
|
30
30
|
};
|
|
31
31
|
}
|
|
32
|
+
function makeEmbedding(activeIndex, dimension = TEST_CONFIG.vectorDimension) {
|
|
33
|
+
const embedding = new Float32Array(dimension);
|
|
34
|
+
embedding[activeIndex] = 1;
|
|
35
|
+
return embedding;
|
|
36
|
+
}
|
|
37
|
+
function readUserVersion(db) {
|
|
38
|
+
const value = db.pragma('user_version', { simple: true });
|
|
39
|
+
if (typeof value === 'number')
|
|
40
|
+
return value;
|
|
41
|
+
if (Array.isArray(value)) {
|
|
42
|
+
const [row] = value;
|
|
43
|
+
return typeof row?.user_version === 'number' ? row.user_version : 0;
|
|
44
|
+
}
|
|
45
|
+
if (value && typeof value === 'object' && 'user_version' in value) {
|
|
46
|
+
const row = value;
|
|
47
|
+
return typeof row.user_version === 'number' ? row.user_version : 0;
|
|
48
|
+
}
|
|
49
|
+
return 0;
|
|
50
|
+
}
|
|
32
51
|
describe('MemoryStore', () => {
|
|
33
52
|
let tmpDir;
|
|
34
53
|
let dbPath;
|
|
@@ -42,8 +61,50 @@ describe('MemoryStore', () => {
|
|
|
42
61
|
store.close();
|
|
43
62
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
44
63
|
});
|
|
45
|
-
test('
|
|
46
|
-
|
|
64
|
+
test('libsql supports required local driver and vector features', () => {
|
|
65
|
+
const libsqlDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-libsql-compat-'));
|
|
66
|
+
const libsqlDb = new Database(path.join(libsqlDir, 'index.db'));
|
|
67
|
+
try {
|
|
68
|
+
assert.equal(readUserVersion(libsqlDb), 0);
|
|
69
|
+
libsqlDb.pragma('journal_mode = WAL');
|
|
70
|
+
libsqlDb.pragma('busy_timeout = 5000');
|
|
71
|
+
libsqlDb.pragma(`user_version = ${SCHEMA_VERSION}`);
|
|
72
|
+
assert.equal(readUserVersion(libsqlDb), SCHEMA_VERSION);
|
|
73
|
+
libsqlDb.exec(`
|
|
74
|
+
CREATE TABLE vector_items (
|
|
75
|
+
rowid INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
76
|
+
name TEXT NOT NULL,
|
|
77
|
+
embedding F32_BLOB(3) NOT NULL
|
|
78
|
+
);
|
|
79
|
+
CREATE INDEX idx_vector_items_embedding
|
|
80
|
+
ON vector_items (libsql_vector_idx(embedding));
|
|
81
|
+
`);
|
|
82
|
+
const insert = libsqlDb.prepare(`INSERT INTO vector_items (name, embedding) VALUES (?, vector32(?))`);
|
|
83
|
+
const info = insert.run('alpha', '[1,0,0]');
|
|
84
|
+
assert.equal(info.lastInsertRowid, 1);
|
|
85
|
+
insert.run('beta', '[0,1,0]');
|
|
86
|
+
const insertGamma = libsqlDb.transaction(() => insert.run('gamma', '[0,0,1]'));
|
|
87
|
+
insertGamma();
|
|
88
|
+
const distance = libsqlDb
|
|
89
|
+
.prepare(`SELECT vector_distance_cos(embedding, vector32(?)) AS distance FROM vector_items WHERE name = ?`)
|
|
90
|
+
.get('[1,0,0]', 'alpha');
|
|
91
|
+
assert.equal(distance.distance, 0);
|
|
92
|
+
const rows = libsqlDb
|
|
93
|
+
.prepare(`SELECT vector_items.name
|
|
94
|
+
FROM vector_top_k('idx_vector_items_embedding', vector32(?), ?) AS vector_matches
|
|
95
|
+
JOIN vector_items ON vector_items.rowid = vector_matches.id
|
|
96
|
+
ORDER BY vector_distance_cos(vector_items.embedding, vector32(?))`)
|
|
97
|
+
.all('[1,0,0]', 2, '[1,0,0]');
|
|
98
|
+
assert.equal(rows[0].name, 'alpha');
|
|
99
|
+
assert.equal(rows.length, 2);
|
|
100
|
+
}
|
|
101
|
+
finally {
|
|
102
|
+
libsqlDb.close();
|
|
103
|
+
fs.rmSync(libsqlDir, { recursive: true, force: true });
|
|
104
|
+
}
|
|
105
|
+
});
|
|
106
|
+
test('vecAvailable is true after schema creation', () => {
|
|
107
|
+
assert.equal(store.vecAvailable, true);
|
|
47
108
|
});
|
|
48
109
|
test('hashesBySource returns empty set for unknown source', () => {
|
|
49
110
|
const hashes = store.hashesBySource('nonexistent.md');
|
|
@@ -52,7 +113,7 @@ describe('MemoryStore', () => {
|
|
|
52
113
|
});
|
|
53
114
|
test('upsert stores chunks and hashesBySource returns their ids', () => {
|
|
54
115
|
const chunk = makeChunk({ id: 'upsert-test-0001', source: 'upsert.md', sourceType: 'entity', fileMtimeAt: 2000 });
|
|
55
|
-
const embedding =
|
|
116
|
+
const embedding = makeEmbedding(0);
|
|
56
117
|
store.upsert([chunk], [embedding]);
|
|
57
118
|
const hashes = store.hashesBySource('upsert.md');
|
|
58
119
|
assert.ok(hashes.has('upsert-test-0001'));
|
|
@@ -71,7 +132,7 @@ describe('MemoryStore', () => {
|
|
|
71
132
|
source: 'deleteme.md',
|
|
72
133
|
content: 'delete source chunk 2',
|
|
73
134
|
});
|
|
74
|
-
store.upsert([c1, c2], [
|
|
135
|
+
store.upsert([c1, c2], [makeEmbedding(1), makeEmbedding(2)]);
|
|
75
136
|
store.deleteBySource('deleteme.md');
|
|
76
137
|
const hashes = store.hashesBySource('deleteme.md');
|
|
77
138
|
assert.equal(hashes.size, 0);
|
|
@@ -82,7 +143,7 @@ describe('MemoryStore', () => {
|
|
|
82
143
|
source: 'bm25.md',
|
|
83
144
|
content: 'uniqueKeywordXYZ for BM25 testing',
|
|
84
145
|
});
|
|
85
|
-
store.upsert([chunk], [
|
|
146
|
+
store.upsert([chunk], [makeEmbedding(3)]);
|
|
86
147
|
const results = store.searchBm25('uniqueKeywordXYZ', 5);
|
|
87
148
|
assert.ok(results.length > 0, 'Expected at least one BM25 result');
|
|
88
149
|
assert.ok(results.some((r) => r.id === 'bm25-search-0001'), `Expected chunk id in results, got: ${results.map((r) => r.id).join(', ')}`);
|
|
@@ -101,7 +162,7 @@ describe('MemoryStore', () => {
|
|
|
101
162
|
heading: 'Open Questions',
|
|
102
163
|
content: 'How should git manage worktree lifecycle decisions?',
|
|
103
164
|
});
|
|
104
|
-
store.upsert([preference, unrelated], [
|
|
165
|
+
store.upsert([preference, unrelated], [makeEmbedding(4), makeEmbedding(5)]);
|
|
105
166
|
const results = store.searchBm25('personal likes and preferences of the user', 5);
|
|
106
167
|
assert.ok(results.length > 0, 'Expected preference query to return results');
|
|
107
168
|
assert.equal(results[0].id, 'preference-search-0001');
|
|
@@ -122,7 +183,7 @@ describe('MemoryStore', () => {
|
|
|
122
183
|
lineEnd: 20,
|
|
123
184
|
fileMtimeAt: 3000,
|
|
124
185
|
});
|
|
125
|
-
store.upsert([chunk], [
|
|
186
|
+
store.upsert([chunk], [makeEmbedding(6)]);
|
|
126
187
|
const results = store.getChunksByIds(['get-by-ids-0001']);
|
|
127
188
|
assert.equal(results.length, 1);
|
|
128
189
|
const [r] = results;
|
|
@@ -137,7 +198,7 @@ describe('MemoryStore', () => {
|
|
|
137
198
|
});
|
|
138
199
|
test('getChunksByIds returns only found rows for mixed ids', () => {
|
|
139
200
|
const chunk = makeChunk({ id: 'partial-found-0001', source: 'partial.md', content: 'partial' });
|
|
140
|
-
store.upsert([chunk], [
|
|
201
|
+
store.upsert([chunk], [makeEmbedding(7)]);
|
|
141
202
|
const results = store.getChunksByIds(['partial-found-0001', 'does-not-exist-999']);
|
|
142
203
|
assert.equal(results.length, 1);
|
|
143
204
|
assert.equal(results[0].id, 'partial-found-0001');
|
|
@@ -148,14 +209,14 @@ describe('MemoryStore', () => {
|
|
|
148
209
|
source: 'indexed-source.md',
|
|
149
210
|
content: 'indexed',
|
|
150
211
|
});
|
|
151
|
-
store.upsert([chunk], [
|
|
212
|
+
store.upsert([chunk], [makeEmbedding(8)]);
|
|
152
213
|
const sources = store.indexedSources();
|
|
153
214
|
assert.ok(sources.includes('indexed-source.md'), `Expected 'indexed-source.md' in ${sources.join(', ')}`);
|
|
154
215
|
});
|
|
155
216
|
test('deleteByIds removes specific chunks', () => {
|
|
156
217
|
const c1 = makeChunk({ id: 'del-ids-0001', source: 'del-ids.md', content: 'delete by id 1' });
|
|
157
218
|
const c2 = makeChunk({ id: 'del-ids-0002', source: 'del-ids.md', content: 'delete by id 2' });
|
|
158
|
-
store.upsert([c1, c2], [
|
|
219
|
+
store.upsert([c1, c2], [makeEmbedding(9), makeEmbedding(10)]);
|
|
159
220
|
store.deleteByIds(['del-ids-0001']);
|
|
160
221
|
const hashes = store.hashesBySource('del-ids.md');
|
|
161
222
|
assert.ok(!hashes.has('del-ids-0001'), 'Deleted chunk id must be gone');
|
|
@@ -168,7 +229,7 @@ describe('MemoryStore', () => {
|
|
|
168
229
|
source: 'fts5.md',
|
|
169
230
|
content: 'degradedModeTest keyword',
|
|
170
231
|
});
|
|
171
|
-
store.upsert([chunk], [
|
|
232
|
+
store.upsert([chunk], [makeEmbedding(11)]);
|
|
172
233
|
const results = store.searchBm25('degradedModeTest', 5);
|
|
173
234
|
assert.ok(results.length > 0, 'FTS5 search must work regardless of vecAvailable');
|
|
174
235
|
});
|
|
@@ -205,7 +266,7 @@ describe('MemoryStore', () => {
|
|
|
205
266
|
fileMtimeAt: 500,
|
|
206
267
|
content: 'entity',
|
|
207
268
|
}),
|
|
208
|
-
], [
|
|
269
|
+
], [makeEmbedding(12), makeEmbedding(13), makeEmbedding(14), makeEmbedding(15)]);
|
|
209
270
|
const allRows = recentStore.getRecentSources({ limit: 10 });
|
|
210
271
|
assert.deepEqual(allRows.map((row) => row.source), [
|
|
211
272
|
'compiled/entities/entity.md',
|
|
@@ -221,7 +282,41 @@ describe('MemoryStore', () => {
|
|
|
221
282
|
fs.rmSync(recentDir, { recursive: true, force: true });
|
|
222
283
|
}
|
|
223
284
|
});
|
|
224
|
-
test('
|
|
285
|
+
test('searchDense returns nearest vector with bounded scores', () => {
|
|
286
|
+
const denseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-dense-store-'));
|
|
287
|
+
const denseStore = new MemoryStore(path.join(denseDir, 'index.db'), { ...TEST_CONFIG, wikiDir: denseDir });
|
|
288
|
+
try {
|
|
289
|
+
denseStore.upsert([
|
|
290
|
+
makeChunk({ id: 'dense-alpha', source: 'dense.md', content: 'alpha vector' }),
|
|
291
|
+
makeChunk({ id: 'dense-beta', source: 'dense.md', content: 'beta vector' }),
|
|
292
|
+
makeChunk({ id: 'dense-gamma', source: 'dense.md', content: 'gamma vector' }),
|
|
293
|
+
], [makeEmbedding(21), makeEmbedding(22), makeEmbedding(23)]);
|
|
294
|
+
const results = denseStore.searchDense(makeEmbedding(22), 3);
|
|
295
|
+
assert.ok(results.length > 0, 'Expected dense search results');
|
|
296
|
+
assert.equal(results[0].id, 'dense-beta');
|
|
297
|
+
assert.ok(results.every((result) => result.score >= 0 && result.score <= 1));
|
|
298
|
+
}
|
|
299
|
+
finally {
|
|
300
|
+
denseStore.close();
|
|
301
|
+
fs.rmSync(denseDir, { recursive: true, force: true });
|
|
302
|
+
}
|
|
303
|
+
});
|
|
304
|
+
test('searchDense returns empty array for invalid query embedding', () => {
|
|
305
|
+
assert.deepEqual(store.searchDense(new Float32Array(3), 5), []);
|
|
306
|
+
});
|
|
307
|
+
test('upsert skips invalid embeddings while persisting valid chunks', () => {
|
|
308
|
+
const validChunk = makeChunk({ id: 'valid-embedding-0001', source: 'embedding-validation.md' });
|
|
309
|
+
const invalidChunk = makeChunk({ id: 'invalid-embedding-0001', source: 'embedding-validation.md' });
|
|
310
|
+
const nonFiniteChunk = makeChunk({ id: 'non-finite-embedding-0001', source: 'embedding-validation.md' });
|
|
311
|
+
const nonFiniteEmbedding = makeEmbedding(24);
|
|
312
|
+
nonFiniteEmbedding[25] = Number.NaN;
|
|
313
|
+
store.upsert([validChunk, invalidChunk, nonFiniteChunk], [makeEmbedding(24), new Float32Array(3), nonFiniteEmbedding]);
|
|
314
|
+
const hashes = store.hashesBySource('embedding-validation.md');
|
|
315
|
+
assert.ok(hashes.has('valid-embedding-0001'));
|
|
316
|
+
assert.ok(!hashes.has('invalid-embedding-0001'));
|
|
317
|
+
assert.ok(!hashes.has('non-finite-embedding-0001'));
|
|
318
|
+
});
|
|
319
|
+
test('schema migration recreates versioned schema with new columns, vector index, and working BM25', () => {
|
|
225
320
|
const migrationDir = fs.mkdtempSync(path.join(os.tmpdir(), 'memory-migration-'));
|
|
226
321
|
const migrationDbPath = path.join(migrationDir, 'index.db');
|
|
227
322
|
const db = new Database(migrationDbPath);
|
|
@@ -264,18 +359,28 @@ describe('MemoryStore', () => {
|
|
|
264
359
|
try {
|
|
265
360
|
const verifyDb = new Database(migrationDbPath, { readonly: true });
|
|
266
361
|
try {
|
|
267
|
-
assert.equal(verifyDb
|
|
362
|
+
assert.equal(readUserVersion(verifyDb), SCHEMA_VERSION);
|
|
268
363
|
const columns = verifyDb.prepare(`PRAGMA table_info(memory_chunks)`).all();
|
|
269
364
|
assert.ok(columns.some((column) => column.name === 'source_type'));
|
|
270
365
|
assert.ok(columns.some((column) => column.name === 'file_mtime_at'));
|
|
366
|
+
assert.ok(columns.some((column) => column.name === 'embedding'));
|
|
271
367
|
const indexes = verifyDb.prepare(`PRAGMA index_list(memory_chunks)`).all();
|
|
272
368
|
assert.ok(indexes.some((index) => index.name === 'idx_memory_chunks_type_mtime'));
|
|
369
|
+
assert.ok(indexes.some((index) => index.name === 'idx_memory_chunks_embedding'));
|
|
273
370
|
const rowCount = verifyDb.prepare(`SELECT COUNT(*) AS count FROM memory_chunks`).get();
|
|
274
371
|
assert.equal(rowCount.count, 0);
|
|
275
372
|
}
|
|
276
373
|
finally {
|
|
277
374
|
verifyDb.close();
|
|
278
375
|
}
|
|
376
|
+
const migratedChunk = makeChunk({
|
|
377
|
+
id: 'migrated-bm25-0001',
|
|
378
|
+
source: 'migrated.md',
|
|
379
|
+
content: 'postMigrationKeyword is searchable after migration',
|
|
380
|
+
});
|
|
381
|
+
migratedStore.upsert([migratedChunk], [makeEmbedding(16)]);
|
|
382
|
+
const results = migratedStore.searchBm25('postMigrationKeyword', 5);
|
|
383
|
+
assert.ok(results.some((result) => result.id === 'migrated-bm25-0001'));
|
|
279
384
|
}
|
|
280
385
|
finally {
|
|
281
386
|
migratedStore.close();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@hanhnd/agent-kit",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.36",
|
|
4
4
|
"description": "Agent Kit MCP server for software development agent workflows",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/server.js",
|
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
},
|
|
23
23
|
"scripts": {
|
|
24
24
|
"build": "rm -rf dist && tsc && npm run bundle",
|
|
25
|
-
"bundle": "esbuild src/entrypoints/server.ts src/entrypoints/cli.ts --bundle --platform=node --format=esm --outdir=dist --external:node:* --external:
|
|
25
|
+
"bundle": "esbuild src/entrypoints/server.ts src/entrypoints/cli.ts --bundle --platform=node --format=esm --outdir=dist --external:node:* --external:libsql --external:fastembed --external:node-llama-cpp",
|
|
26
26
|
"dev": "tsc --watch",
|
|
27
27
|
"prepublishOnly": "npm run build",
|
|
28
28
|
"test": "tsc && node --test $(find dist -name '*.test.js' | tr '\\n' ' ')"
|
|
@@ -30,15 +30,13 @@
|
|
|
30
30
|
"dependencies": {
|
|
31
31
|
"node-llama-cpp": "^3.0.0",
|
|
32
32
|
"@modelcontextprotocol/sdk": "^1.13.1",
|
|
33
|
-
"better-sqlite3": "^11.0.0",
|
|
34
33
|
"dedent": "^1.5.3",
|
|
35
34
|
"fastembed": "^1.14.0",
|
|
36
|
-
"
|
|
35
|
+
"libsql": "^0.5.29",
|
|
37
36
|
"stopword": "^3.1.5",
|
|
38
37
|
"zod": "^3.24.4"
|
|
39
38
|
},
|
|
40
39
|
"devDependencies": {
|
|
41
|
-
"@types/better-sqlite3": "^7.6.0",
|
|
42
40
|
"@types/node": "^22.15.21",
|
|
43
41
|
"esbuild": "^0.25.4",
|
|
44
42
|
"typescript": "^5.8.3"
|