@memextend/claude-code 0.2.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hooks/pre-compact.cjs +98 -81
- package/dist/hooks/pre-compact.js +9 -9
- package/dist/hooks/pre-compact.js.map +1 -1
- package/dist/hooks/session-start.cjs +99 -82
- package/dist/hooks/session-start.js +5 -5
- package/dist/hooks/session-start.js.map +1 -1
- package/dist/hooks/stop.cjs +98 -81
- package/dist/hooks/stop.js +9 -9
- package/dist/hooks/stop.js.map +1 -1
- package/dist/mcp/server.cjs +107 -90
- package/dist/mcp/server.js +13 -13
- package/dist/mcp/server.js.map +1 -1
- package/package.json +2 -2
- package/scripts/build-hooks.js +9 -1
|
@@ -37,6 +37,7 @@ var SQLiteStorage = class {
|
|
|
37
37
|
constructor(dbPath) {
|
|
38
38
|
this.db = new import_better_sqlite3.default(dbPath);
|
|
39
39
|
this.db.pragma("journal_mode = WAL");
|
|
40
|
+
this.db.pragma("busy_timeout = 5000");
|
|
40
41
|
this.initialize();
|
|
41
42
|
}
|
|
42
43
|
initialize() {
|
|
@@ -390,122 +391,138 @@ var SQLiteStorage = class {
|
|
|
390
391
|
};
|
|
391
392
|
}
|
|
392
393
|
close() {
|
|
394
|
+
try {
|
|
395
|
+
this.db.pragma("wal_checkpoint(TRUNCATE)");
|
|
396
|
+
} catch {
|
|
397
|
+
}
|
|
393
398
|
this.db.close();
|
|
394
399
|
}
|
|
395
400
|
};
|
|
396
401
|
|
|
397
|
-
// ../../core/dist/storage/
|
|
398
|
-
var
|
|
399
|
-
var
|
|
402
|
+
// ../../core/dist/storage/sqlite-vec.js
|
|
403
|
+
var import_better_sqlite32 = __toESM(require("better-sqlite3"), 1);
|
|
404
|
+
var sqliteVec = __toESM(require("sqlite-vec"), 1);
|
|
405
|
+
var SQLiteVecStorage = class _SQLiteVecStorage {
|
|
400
406
|
db;
|
|
401
|
-
|
|
402
|
-
tableName = "memories";
|
|
407
|
+
tableName = "memory_vectors";
|
|
403
408
|
dimensions = 384;
|
|
404
409
|
constructor(db) {
|
|
405
410
|
this.db = db;
|
|
406
411
|
}
|
|
407
412
|
static async create(dbPath) {
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
413
|
+
let actualPath = dbPath;
|
|
414
|
+
if (dbPath.endsWith("vectors") || dbPath.endsWith("vectors/")) {
|
|
415
|
+
actualPath = dbPath.replace(/\/?$/, ".db");
|
|
416
|
+
}
|
|
417
|
+
const db = new import_better_sqlite32.default(actualPath);
|
|
418
|
+
sqliteVec.load(db);
|
|
419
|
+
db.pragma("journal_mode = WAL");
|
|
420
|
+
db.pragma("busy_timeout = 5000");
|
|
421
|
+
const storage = new _SQLiteVecStorage(db);
|
|
422
|
+
storage.initialize();
|
|
411
423
|
return storage;
|
|
412
424
|
}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
425
|
+
initialize() {
|
|
426
|
+
this.db.exec(`
|
|
427
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${this.tableName} USING vec0(
|
|
428
|
+
id TEXT PRIMARY KEY,
|
|
429
|
+
vector FLOAT[${this.dimensions}]
|
|
430
|
+
)
|
|
431
|
+
`);
|
|
418
432
|
}
|
|
419
433
|
async insertVector(id, vector) {
|
|
420
434
|
if (vector.length !== this.dimensions) {
|
|
421
435
|
throw new Error(`Vector must have ${this.dimensions} dimensions, got ${vector.length}`);
|
|
422
436
|
}
|
|
423
|
-
const
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
437
|
+
const float32 = new Float32Array(vector);
|
|
438
|
+
const vectorBuffer = Buffer.from(float32.buffer);
|
|
439
|
+
const stmt = this.db.prepare(`
|
|
440
|
+
INSERT OR REPLACE INTO ${this.tableName} (id, vector)
|
|
441
|
+
VALUES (?, ?)
|
|
442
|
+
`);
|
|
443
|
+
stmt.run(id, vectorBuffer);
|
|
430
444
|
}
|
|
431
445
|
async insertVectors(items) {
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
446
|
+
const stmt = this.db.prepare(`
|
|
447
|
+
INSERT OR REPLACE INTO ${this.tableName} (id, vector)
|
|
448
|
+
VALUES (?, ?)
|
|
449
|
+
`);
|
|
450
|
+
const insertMany = this.db.transaction((items2) => {
|
|
451
|
+
for (const item of items2) {
|
|
452
|
+
if (item.vector.length !== this.dimensions) {
|
|
453
|
+
throw new Error(`Vector must have ${this.dimensions} dimensions, got ${item.vector.length}`);
|
|
454
|
+
}
|
|
455
|
+
const float32 = new Float32Array(item.vector);
|
|
456
|
+
const vectorBuffer = Buffer.from(float32.buffer);
|
|
457
|
+
stmt.run(item.id, vectorBuffer);
|
|
435
458
|
}
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
this.table = await this.db.createTable(this.tableName, items);
|
|
439
|
-
} else {
|
|
440
|
-
await this.table.add(items);
|
|
441
|
-
}
|
|
442
|
-
await this.optimize();
|
|
459
|
+
});
|
|
460
|
+
insertMany(items);
|
|
443
461
|
}
|
|
444
462
|
async search(vector, limit = 10) {
|
|
445
|
-
if (
|
|
463
|
+
if (vector.length !== this.dimensions) {
|
|
464
|
+
throw new Error(`Query vector must have ${this.dimensions} dimensions, got ${vector.length}`);
|
|
465
|
+
}
|
|
466
|
+
const count = await this.getVectorCount();
|
|
467
|
+
if (count === 0) {
|
|
446
468
|
return [];
|
|
447
469
|
}
|
|
448
470
|
const effectiveLimit = limit > 0 ? limit : 100;
|
|
449
|
-
const
|
|
450
|
-
|
|
471
|
+
const float32 = new Float32Array(vector);
|
|
472
|
+
const vectorBuffer = Buffer.from(float32.buffer);
|
|
473
|
+
const stmt = this.db.prepare(`
|
|
474
|
+
SELECT id, distance
|
|
475
|
+
FROM ${this.tableName}
|
|
476
|
+
WHERE vector MATCH ?
|
|
477
|
+
AND k = ?
|
|
478
|
+
ORDER BY distance
|
|
479
|
+
`);
|
|
480
|
+
const rows = stmt.all(vectorBuffer, effectiveLimit);
|
|
481
|
+
return rows.map((row) => ({
|
|
451
482
|
id: row.id,
|
|
452
|
-
score: 1
|
|
453
|
-
// Convert distance to similarity
|
|
483
|
+
score: 1 / (1 + row.distance)
|
|
454
484
|
}));
|
|
455
485
|
}
|
|
456
486
|
async deleteVector(id) {
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
const sanitizedId = id.replace(/'/g, "''");
|
|
460
|
-
await this.table.delete(`id = '${sanitizedId}'`);
|
|
487
|
+
const stmt = this.db.prepare(`DELETE FROM ${this.tableName} WHERE id = ?`);
|
|
488
|
+
stmt.run(id);
|
|
461
489
|
}
|
|
462
490
|
async getVectorCount() {
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
return
|
|
491
|
+
const stmt = this.db.prepare(`SELECT COUNT(*) as count FROM ${this.tableName}`);
|
|
492
|
+
const result = stmt.get();
|
|
493
|
+
return result.count;
|
|
466
494
|
}
|
|
467
495
|
async getVectorsByIds(ids) {
|
|
468
496
|
const result = /* @__PURE__ */ new Map();
|
|
469
|
-
if (
|
|
497
|
+
if (ids.length === 0)
|
|
470
498
|
return result;
|
|
471
499
|
const BATCH_SIZE = 100;
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
500
|
+
for (let i = 0; i < ids.length; i += BATCH_SIZE) {
|
|
501
|
+
const batch = ids.slice(i, i + BATCH_SIZE);
|
|
502
|
+
const placeholders = batch.map(() => "?").join(",");
|
|
503
|
+
const stmt = this.db.prepare(`
|
|
504
|
+
SELECT id, vector FROM ${this.tableName}
|
|
505
|
+
WHERE id IN (${placeholders})
|
|
506
|
+
`);
|
|
507
|
+
const rows = stmt.all(...batch);
|
|
508
|
+
for (const row of rows) {
|
|
509
|
+
const vector = Array.from(new Float32Array(row.vector));
|
|
510
|
+
result.set(row.id, vector);
|
|
481
511
|
}
|
|
482
|
-
} catch {
|
|
483
512
|
}
|
|
484
513
|
return result;
|
|
485
514
|
}
|
|
486
515
|
async close() {
|
|
487
|
-
}
|
|
488
|
-
/**
|
|
489
|
-
* Optimize the LanceDB table to reduce storage.
|
|
490
|
-
* This compacts files, prunes old versions, and optimizes indices.
|
|
491
|
-
* Should be called periodically (e.g., after many inserts or on cleanup command).
|
|
492
|
-
*
|
|
493
|
-
* @param cleanupOlderThan - Date before which old versions should be pruned (default: now)
|
|
494
|
-
*/
|
|
495
|
-
async optimize(cleanupOlderThan) {
|
|
496
|
-
if (!this.table)
|
|
497
|
-
return null;
|
|
498
516
|
try {
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
return {
|
|
502
|
-
compacted: stats?.compaction?.filesRemoved ?? 0,
|
|
503
|
-
pruned: stats?.prune?.versionsRemoved ?? 0
|
|
504
|
-
};
|
|
505
|
-
} catch (error) {
|
|
506
|
-
console.error("[memextend] LanceDB optimize failed:", error);
|
|
507
|
-
return null;
|
|
517
|
+
this.db.pragma("wal_checkpoint(TRUNCATE)");
|
|
518
|
+
} catch {
|
|
508
519
|
}
|
|
520
|
+
this.db.close();
|
|
521
|
+
}
|
|
522
|
+
// No optimize needed - SQLite handles this automatically!
|
|
523
|
+
async optimize() {
|
|
524
|
+
this.db.exec("VACUUM");
|
|
525
|
+
return { compacted: 0, pruned: 0 };
|
|
509
526
|
}
|
|
510
527
|
};
|
|
511
528
|
|
|
@@ -651,12 +668,12 @@ async function createEmbedFunction(modelsDir) {
|
|
|
651
668
|
// ../../core/dist/memory/retrieve.js
|
|
652
669
|
var MemoryRetriever = class {
|
|
653
670
|
sqlite;
|
|
654
|
-
|
|
671
|
+
vectorStore;
|
|
655
672
|
embed;
|
|
656
673
|
options;
|
|
657
|
-
constructor(sqlite,
|
|
674
|
+
constructor(sqlite, vectorStore, embed, options = {}) {
|
|
658
675
|
this.sqlite = sqlite;
|
|
659
|
-
this.
|
|
676
|
+
this.vectorStore = vectorStore;
|
|
660
677
|
this.embed = embed;
|
|
661
678
|
this.options = {
|
|
662
679
|
defaultLimit: options.defaultLimit ?? 0,
|
|
@@ -674,12 +691,12 @@ var MemoryRetriever = class {
|
|
|
674
691
|
return this.sqlite.searchFTS(query, limit);
|
|
675
692
|
}
|
|
676
693
|
/**
|
|
677
|
-
* Vector similarity search
|
|
694
|
+
* Vector similarity search
|
|
678
695
|
*/
|
|
679
696
|
async vectorSearch(query, options = {}) {
|
|
680
697
|
const limit = options.limit ?? this.options.defaultLimit;
|
|
681
698
|
const queryVector = await this.embed(query);
|
|
682
|
-
const vectorResults = await this.
|
|
699
|
+
const vectorResults = await this.vectorStore.search(queryVector, limit * 2);
|
|
683
700
|
const results = [];
|
|
684
701
|
for (const vr of vectorResults) {
|
|
685
702
|
const memory = this.sqlite.getMemory(vr.id);
|
|
@@ -971,9 +988,9 @@ async function main() {
|
|
|
971
988
|
}
|
|
972
989
|
const projectId = getProjectId2(input.cwd);
|
|
973
990
|
const sqlite = new SQLiteStorage(DB_PATH);
|
|
974
|
-
const
|
|
991
|
+
const vectorStore = await SQLiteVecStorage.create(VECTORS_PATH);
|
|
975
992
|
const embedder = await createEmbedFunction(MODELS_PATH);
|
|
976
|
-
const retriever = new MemoryRetriever(sqlite,
|
|
993
|
+
const retriever = new MemoryRetriever(sqlite, vectorStore, embedder.embedQuery, {
|
|
977
994
|
defaultLimit: config.retrieval?.maxMemories ?? 0,
|
|
978
995
|
defaultRecentDays: config.retrieval?.recentDays ?? 0
|
|
979
996
|
});
|
|
@@ -1007,7 +1024,7 @@ async function main() {
|
|
|
1007
1024
|
let deduplicatedMemories = allMemories;
|
|
1008
1025
|
if (allMemories.length > 1) {
|
|
1009
1026
|
const memoryIds = allMemories.map((m) => m.id);
|
|
1010
|
-
const vectors = await
|
|
1027
|
+
const vectors = await vectorStore.getVectorsByIds(memoryIds);
|
|
1011
1028
|
if (vectors.size > 0) {
|
|
1012
1029
|
deduplicatedMemories = deduplicateMemories(allMemories, vectors, {
|
|
1013
1030
|
similarityThreshold: deduplicationThreshold
|
|
@@ -1024,7 +1041,7 @@ async function main() {
|
|
|
1024
1041
|
}
|
|
1025
1042
|
}
|
|
1026
1043
|
sqlite.close();
|
|
1027
|
-
await
|
|
1044
|
+
await vectorStore.close();
|
|
1028
1045
|
await embedder.close();
|
|
1029
1046
|
if (deduplicatedMemories.length === 0 && context.globalProfile.length === 0) {
|
|
1030
1047
|
log("SessionStart", "No memories to inject");
|
|
@@ -6,7 +6,7 @@ import { readFile } from 'fs/promises';
|
|
|
6
6
|
import { join, basename } from 'path';
|
|
7
7
|
import { homedir } from 'os';
|
|
8
8
|
import { execSync } from 'child_process';
|
|
9
|
-
import { SQLiteStorage,
|
|
9
|
+
import { SQLiteStorage, SQLiteVecStorage, MemoryRetriever, formatContextForInjection, createEmbedFunction, deduplicateMemories, getDeduplicationStats } from '@memextend/core';
|
|
10
10
|
import { log, logError } from './logger.js';
|
|
11
11
|
const MEMEXTEND_DIR = join(homedir(), '.memextend');
|
|
12
12
|
const CONFIG_PATH = join(MEMEXTEND_DIR, 'config.json');
|
|
@@ -43,10 +43,10 @@ async function main() {
|
|
|
43
43
|
const projectId = getProjectId(input.cwd);
|
|
44
44
|
// Initialize storage
|
|
45
45
|
const sqlite = new SQLiteStorage(DB_PATH);
|
|
46
|
-
const
|
|
46
|
+
const vectorStore = await SQLiteVecStorage.create(VECTORS_PATH);
|
|
47
47
|
// Create embedding function (uses real model if available, fallback otherwise)
|
|
48
48
|
const embedder = await createEmbedFunction(MODELS_PATH);
|
|
49
|
-
const retriever = new MemoryRetriever(sqlite,
|
|
49
|
+
const retriever = new MemoryRetriever(sqlite, vectorStore, embedder.embedQuery, {
|
|
50
50
|
defaultLimit: config.retrieval?.maxMemories ?? 0,
|
|
51
51
|
defaultRecentDays: config.retrieval?.recentDays ?? 0,
|
|
52
52
|
});
|
|
@@ -90,7 +90,7 @@ async function main() {
|
|
|
90
90
|
if (allMemories.length > 1) {
|
|
91
91
|
// Fetch vectors for all memories
|
|
92
92
|
const memoryIds = allMemories.map(m => m.id);
|
|
93
|
-
const vectors = await
|
|
93
|
+
const vectors = await vectorStore.getVectorsByIds(memoryIds);
|
|
94
94
|
if (vectors.size > 0) {
|
|
95
95
|
deduplicatedMemories = deduplicateMemories(allMemories, vectors, {
|
|
96
96
|
similarityThreshold: deduplicationThreshold
|
|
@@ -108,7 +108,7 @@ async function main() {
|
|
|
108
108
|
}
|
|
109
109
|
// Close storage and embedder
|
|
110
110
|
sqlite.close();
|
|
111
|
-
await
|
|
111
|
+
await vectorStore.close();
|
|
112
112
|
await embedder.close();
|
|
113
113
|
// Check if there's anything to inject
|
|
114
114
|
if (deduplicatedMemories.length === 0 && context.globalProfile.length === 0) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session-start.js","sourceRoot":"","sources":["../../src/hooks/session-start.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,8CAA8C;AAE9C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EACL,aAAa,EACb,
|
|
1
|
+
{"version":3,"file":"session-start.js","sourceRoot":"","sources":["../../src/hooks/session-start.ts"],"names":[],"mappings":"AAAA,2DAA2D;AAC3D,8CAA8C;AAE9C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;AAC7B,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,eAAe,EACf,yBAAyB,EACzB,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,EAEtB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAiB5C,MAAM,aAAa,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AACpD,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;AACvD,MAAM,OAAO,GAAG,IAAI,CAAC,aAAa,EAAE,cAAc,CAAC,CAAC;AACpD,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;AACpD,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;AAElD,KAAK,UAAU,IAAI;IACjB,wBAAwB;IACxB,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrB,CAAC;IACD,MAAM,KAAK,GAAc,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAEtE,GAAG,CAAC,cAAc,EAAE,YAAY,EAAE;QAChC,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,GAAG,EAAE,KAAK,CAAC,GAAG;KACf,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,oCAAoC;QACpC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACzB,GAAG,CAAC,cAAc,EAAE,wBAAwB,CAAC,CAAC;YAC9C,YAAY,CAAC,EAAE,CAAC,CAAC;YACjB,OAAO;QACT,CAAC;QAED,cAAc;QACd,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,UAAU,EAAE,CAAC;YAClC,GAAG,CAAC,cAAc,EAAE,gCAAgC,CAAC,CAAC;YACtD,YAAY,CAAC,EAAE,CAAC,CAAC;YACjB,OAAO;QACT,CAAC;QAED,iBAAiB;QACjB,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAE1C,qBAAqB;QACrB,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QAEhE,+EAA+E;QAC/E,MAAM,QAAQ,GAAG,MAAM,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAExD,MAAM,SAAS,GAAG,IAAI,eAAe,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,UAAU,EAAE;YAC9E,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC;YAChD,iBAAiB,EAAE,MAAM,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC;SACrD,CAAC,CAAC;QAEH,+BAA+B;QAC/B,MAAM,OAAO,GAAG,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,CAAC,aAAa,CAAC;gBACnB,EAAE,EAAE,SAAS;gBACb,IAAI,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC;gBACzB,IAAI,EAAE,KAAK,CAAC,GAAG;gBACf,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;QACL,CAAC;QAED,mDAAmD;QACnD,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;QAEjD,IAAI,aAAa,EAAE,CAAC;YAClB,GAAG,CAAC,cAAc,EAAE,6DAA6D,CAAC,CAAC;QACrF,CAAC;QAED,0BAA0B;QAC1B,2FAA2F;QAC3F,sDAAsD;QACtD,MAAM,WAAW,GAAG,MAAM,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC,CAAC;QACvD,MAAM,cAAc,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,oCAAoC;QACnG,MAAM,WAAW,GAAG,aAAa;YAC/B,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,EAAE,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,uBAAuB;YACtF,CAAC,CAAC,cAAc,CAAC;QAEnB,GAAG,CAAC,cAAc,EAAE,qBAAqB,EAAE,EAAE,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,CAAC,CAAC;QAExF,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,oBAAoB,CAAC,SAAS,EAAE;YAC9D,aAAa,EAAE,MAAM,CAAC,SAAS,EAAE,aAAa,IAAI,IAAI;YACtD,KAAK,EAAE,WAAW;YAClB,UAAU,EAAE,MAAM,CAAC,SAAS,EAAE,UAAU,IAAI,CAAC;SAC9C,CAAC,CAAC;QAEH,yCAAyC;QACzC,MAAM,WAAW,GAAa;YAC5B,GAAG,OAAO,CAAC,cAAc;YACzB,GAAG,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;SAC/C,CAAC;QAEF,yDAAyD;QACzD,MAAM,sBAAsB,GAAG,MAAM,CAAC,SAAS,EAAE,sBAAsB,IAAI,IAAI,CAAC;QAChF,IAAI,oBAAoB,GAAa,WAAW,CAAC;QAEjD,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,iCAAiC;YACjC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YAE7D,IAAI,OAAO,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBACrB,oBAAoB,GAAG,mBAAmB,CAAC,WAAW,EAAE,OAAO,EAAE;oBAC/D,mBAAmB,EAAE,sBAAsB;iBAC5C,CAAC,CAAC;gBAEH,MAAM,KAAK,GAAG,qBAAqB,CAAC,WAAW,CAAC,MAAM,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC;gBACrF,IAAI,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,CAAC;oBACtB,GAAG,CAAC,cAAc,EAAE,wCAAwC,EAAE;wBAC5D,QAAQ,EAAE,WAAW,CAAC,MAAM;wBAC5B,YAAY,EAAE,oBAAoB,CAAC,MAAM;wBACzC,OAAO,EAAE,KAAK,CAAC,OAAO;wBACtB,UAAU,EAAE,KAAK,CAAC,UAAU;qBAC7B,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAED,6BAA6B;QAC7B,MAAM,CAAC,KAAK,EAAE,CAAC;QACf,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC;QAC1B,MAAM,QAAQ,CAAC,KAAK,EAAE,CAAC;QAEvB,sCAAsC;QACtC,IAAI,oBAAoB,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5E,GAAG,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;YAC7C,YAAY,CAAC,EAAE,CAAC,CAAC;YACjB,OAAO;QACT,CAAC;QAED,6CAA6C;QAC7C,6FAA6F;QAC7F,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACjE,MAAM,mBAAmB,GAAG;YAC1B,cAAc,EAAE,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrE,aAAa,EAAE,OAAO,CAAC,aAAa;YACpC,gBAAgB,EAAE,oBAAoB;iBACnC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;iBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,QAAiB,EAAE,CAAC,CAAC;SAClE,CAAC;QAEF,GAAG,CAAC,cAAc,EAAE,oBAAoB,EAAE;YACxC,WAAW,EAAE,mBAAmB,CAAC,cAAc,CAAC,MAAM;YACtD,WAAW,EAAE,mBAAmB,CAAC,aAAa,CAAC,MAAM;YACrD,aAAa,EAAE,mBAAmB,CAAC,gBAAgB,CAAC,MAAM;YAC1D,aAAa,EAAE,WAAW,CAAC,MAAM;YACjC,iBAAiB,EAAE,oBAAoB,CAAC,MAAM;SAC/C,CAAC,CAAC;QAEH,8DAA8D;QAC9D,yFAAyF;QACzF,gGAAgG;QAChG,MAAM,QAAQ,GAAG,aAAa;YAC5B,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,eAAe,IAAI,IAAI,CAAC;YAC7C,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,EAAE,eAAe,IAAI,KAAK,CAAC,CAAC;QAEjD,MAAM,aAAa,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QAEhD,kDAAkD;QAClD,IAAI,gBAAgB,GAAG,yBAAyB,CAAC,mBAAmB,EAAE;YACpE,QAAQ;YACR,aAAa;SACd,CAAC,CAAC;QAEH,GAAG,CAAC,cAAc,EAAE,mBAAmB,EAAE;YACvC,aAAa;YACb,QAAQ;YACR,WAAW,EAAE,gBAAgB,CAAC,MAAM;SACrC,CAAC,CAAC;QAEH,kFAAkF;QAClF,IAAI,aAAa,EAAE,CAAC;YAClB,gBAAgB,GAAG,oGAAoG,gBAAgB,EAAE,CAAC;QAC5I,CAAC;QAED,YAAY,CAAC;YACX,kBAAkB,EAAE;gBAClB,aAAa,EAAE,cAAc;gBAC7B,iBAAiB,EAAE,gBAAgB;aACpC;SACF,CAAC,CAAC;IAEL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,oCAAoC;QACpC,QAAQ,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;QAChC,OAAO,CAAC,KAAK,CAAC,kCAAkC,EAAE,KAAK,CAAC,CAAC;QACzD,YAAY,CAAC,EAAE,CAAC,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAC/B,sBAAsB;IACtB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,QAAQ,CAAC,+BAA+B,EAAE;YACxD,GAAG;YACH,QAAQ,EAAE,OAAO;YACjB,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAC,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,0BAA0B;QAC1B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI,CAAC;QACH,IAAI,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,uBAAuB;IACzB,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,YAAY,CAAC,MAAkB;IACtC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;AACtC,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,0BAA0B,EAAE,KAAK,CAAC,CAAC;IACjD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,wCAAwC;AAC3D,CAAC,CAAC,CAAC"}
|
package/dist/hooks/stop.cjs
CHANGED
|
@@ -37,6 +37,7 @@ var SQLiteStorage = class {
|
|
|
37
37
|
constructor(dbPath) {
|
|
38
38
|
this.db = new import_better_sqlite3.default(dbPath);
|
|
39
39
|
this.db.pragma("journal_mode = WAL");
|
|
40
|
+
this.db.pragma("busy_timeout = 5000");
|
|
40
41
|
this.initialize();
|
|
41
42
|
}
|
|
42
43
|
initialize() {
|
|
@@ -390,122 +391,138 @@ var SQLiteStorage = class {
|
|
|
390
391
|
};
|
|
391
392
|
}
|
|
392
393
|
close() {
|
|
394
|
+
try {
|
|
395
|
+
this.db.pragma("wal_checkpoint(TRUNCATE)");
|
|
396
|
+
} catch {
|
|
397
|
+
}
|
|
393
398
|
this.db.close();
|
|
394
399
|
}
|
|
395
400
|
};
|
|
396
401
|
|
|
397
|
-
// ../../core/dist/storage/
|
|
398
|
-
var
|
|
399
|
-
var
|
|
402
|
+
// ../../core/dist/storage/sqlite-vec.js
|
|
403
|
+
var import_better_sqlite32 = __toESM(require("better-sqlite3"), 1);
|
|
404
|
+
var sqliteVec = __toESM(require("sqlite-vec"), 1);
|
|
405
|
+
var SQLiteVecStorage = class _SQLiteVecStorage {
|
|
400
406
|
db;
|
|
401
|
-
|
|
402
|
-
tableName = "memories";
|
|
407
|
+
tableName = "memory_vectors";
|
|
403
408
|
dimensions = 384;
|
|
404
409
|
constructor(db) {
|
|
405
410
|
this.db = db;
|
|
406
411
|
}
|
|
407
412
|
static async create(dbPath) {
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
413
|
+
let actualPath = dbPath;
|
|
414
|
+
if (dbPath.endsWith("vectors") || dbPath.endsWith("vectors/")) {
|
|
415
|
+
actualPath = dbPath.replace(/\/?$/, ".db");
|
|
416
|
+
}
|
|
417
|
+
const db = new import_better_sqlite32.default(actualPath);
|
|
418
|
+
sqliteVec.load(db);
|
|
419
|
+
db.pragma("journal_mode = WAL");
|
|
420
|
+
db.pragma("busy_timeout = 5000");
|
|
421
|
+
const storage = new _SQLiteVecStorage(db);
|
|
422
|
+
storage.initialize();
|
|
411
423
|
return storage;
|
|
412
424
|
}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
425
|
+
initialize() {
|
|
426
|
+
this.db.exec(`
|
|
427
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${this.tableName} USING vec0(
|
|
428
|
+
id TEXT PRIMARY KEY,
|
|
429
|
+
vector FLOAT[${this.dimensions}]
|
|
430
|
+
)
|
|
431
|
+
`);
|
|
418
432
|
}
|
|
419
433
|
async insertVector(id, vector) {
|
|
420
434
|
if (vector.length !== this.dimensions) {
|
|
421
435
|
throw new Error(`Vector must have ${this.dimensions} dimensions, got ${vector.length}`);
|
|
422
436
|
}
|
|
423
|
-
const
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
437
|
+
const float32 = new Float32Array(vector);
|
|
438
|
+
const vectorBuffer = Buffer.from(float32.buffer);
|
|
439
|
+
const stmt = this.db.prepare(`
|
|
440
|
+
INSERT OR REPLACE INTO ${this.tableName} (id, vector)
|
|
441
|
+
VALUES (?, ?)
|
|
442
|
+
`);
|
|
443
|
+
stmt.run(id, vectorBuffer);
|
|
430
444
|
}
|
|
431
445
|
async insertVectors(items) {
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
446
|
+
const stmt = this.db.prepare(`
|
|
447
|
+
INSERT OR REPLACE INTO ${this.tableName} (id, vector)
|
|
448
|
+
VALUES (?, ?)
|
|
449
|
+
`);
|
|
450
|
+
const insertMany = this.db.transaction((items2) => {
|
|
451
|
+
for (const item of items2) {
|
|
452
|
+
if (item.vector.length !== this.dimensions) {
|
|
453
|
+
throw new Error(`Vector must have ${this.dimensions} dimensions, got ${item.vector.length}`);
|
|
454
|
+
}
|
|
455
|
+
const float32 = new Float32Array(item.vector);
|
|
456
|
+
const vectorBuffer = Buffer.from(float32.buffer);
|
|
457
|
+
stmt.run(item.id, vectorBuffer);
|
|
435
458
|
}
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
this.table = await this.db.createTable(this.tableName, items);
|
|
439
|
-
} else {
|
|
440
|
-
await this.table.add(items);
|
|
441
|
-
}
|
|
442
|
-
await this.optimize();
|
|
459
|
+
});
|
|
460
|
+
insertMany(items);
|
|
443
461
|
}
|
|
444
462
|
async search(vector, limit = 10) {
|
|
445
|
-
if (
|
|
463
|
+
if (vector.length !== this.dimensions) {
|
|
464
|
+
throw new Error(`Query vector must have ${this.dimensions} dimensions, got ${vector.length}`);
|
|
465
|
+
}
|
|
466
|
+
const count = await this.getVectorCount();
|
|
467
|
+
if (count === 0) {
|
|
446
468
|
return [];
|
|
447
469
|
}
|
|
448
470
|
const effectiveLimit = limit > 0 ? limit : 100;
|
|
449
|
-
const
|
|
450
|
-
|
|
471
|
+
const float32 = new Float32Array(vector);
|
|
472
|
+
const vectorBuffer = Buffer.from(float32.buffer);
|
|
473
|
+
const stmt = this.db.prepare(`
|
|
474
|
+
SELECT id, distance
|
|
475
|
+
FROM ${this.tableName}
|
|
476
|
+
WHERE vector MATCH ?
|
|
477
|
+
AND k = ?
|
|
478
|
+
ORDER BY distance
|
|
479
|
+
`);
|
|
480
|
+
const rows = stmt.all(vectorBuffer, effectiveLimit);
|
|
481
|
+
return rows.map((row) => ({
|
|
451
482
|
id: row.id,
|
|
452
|
-
score: 1
|
|
453
|
-
// Convert distance to similarity
|
|
483
|
+
score: 1 / (1 + row.distance)
|
|
454
484
|
}));
|
|
455
485
|
}
|
|
456
486
|
async deleteVector(id) {
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
const sanitizedId = id.replace(/'/g, "''");
|
|
460
|
-
await this.table.delete(`id = '${sanitizedId}'`);
|
|
487
|
+
const stmt = this.db.prepare(`DELETE FROM ${this.tableName} WHERE id = ?`);
|
|
488
|
+
stmt.run(id);
|
|
461
489
|
}
|
|
462
490
|
async getVectorCount() {
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
return
|
|
491
|
+
const stmt = this.db.prepare(`SELECT COUNT(*) as count FROM ${this.tableName}`);
|
|
492
|
+
const result = stmt.get();
|
|
493
|
+
return result.count;
|
|
466
494
|
}
|
|
467
495
|
async getVectorsByIds(ids) {
|
|
468
496
|
const result = /* @__PURE__ */ new Map();
|
|
469
|
-
if (
|
|
497
|
+
if (ids.length === 0)
|
|
470
498
|
return result;
|
|
471
499
|
const BATCH_SIZE = 100;
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
500
|
+
for (let i = 0; i < ids.length; i += BATCH_SIZE) {
|
|
501
|
+
const batch = ids.slice(i, i + BATCH_SIZE);
|
|
502
|
+
const placeholders = batch.map(() => "?").join(",");
|
|
503
|
+
const stmt = this.db.prepare(`
|
|
504
|
+
SELECT id, vector FROM ${this.tableName}
|
|
505
|
+
WHERE id IN (${placeholders})
|
|
506
|
+
`);
|
|
507
|
+
const rows = stmt.all(...batch);
|
|
508
|
+
for (const row of rows) {
|
|
509
|
+
const vector = Array.from(new Float32Array(row.vector));
|
|
510
|
+
result.set(row.id, vector);
|
|
481
511
|
}
|
|
482
|
-
} catch {
|
|
483
512
|
}
|
|
484
513
|
return result;
|
|
485
514
|
}
|
|
486
515
|
async close() {
|
|
487
|
-
}
|
|
488
|
-
/**
|
|
489
|
-
* Optimize the LanceDB table to reduce storage.
|
|
490
|
-
* This compacts files, prunes old versions, and optimizes indices.
|
|
491
|
-
* Should be called periodically (e.g., after many inserts or on cleanup command).
|
|
492
|
-
*
|
|
493
|
-
* @param cleanupOlderThan - Date before which old versions should be pruned (default: now)
|
|
494
|
-
*/
|
|
495
|
-
async optimize(cleanupOlderThan) {
|
|
496
|
-
if (!this.table)
|
|
497
|
-
return null;
|
|
498
516
|
try {
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
return {
|
|
502
|
-
compacted: stats?.compaction?.filesRemoved ?? 0,
|
|
503
|
-
pruned: stats?.prune?.versionsRemoved ?? 0
|
|
504
|
-
};
|
|
505
|
-
} catch (error) {
|
|
506
|
-
console.error("[memextend] LanceDB optimize failed:", error);
|
|
507
|
-
return null;
|
|
517
|
+
this.db.pragma("wal_checkpoint(TRUNCATE)");
|
|
518
|
+
} catch {
|
|
508
519
|
}
|
|
520
|
+
this.db.close();
|
|
521
|
+
}
|
|
522
|
+
// No optimize needed - SQLite handles this automatically!
|
|
523
|
+
async optimize() {
|
|
524
|
+
this.db.exec("VACUUM");
|
|
525
|
+
return { compacted: 0, pruned: 0 };
|
|
509
526
|
}
|
|
510
527
|
};
|
|
511
528
|
|
|
@@ -957,7 +974,7 @@ async function main() {
|
|
|
957
974
|
log("Stop", `Found ${captures.length} captures to save`, { reasoning: reasoningCount, tools: toolCount });
|
|
958
975
|
const projectId = getProjectId2(input.cwd);
|
|
959
976
|
const sqlite = new SQLiteStorage(DB_PATH);
|
|
960
|
-
const
|
|
977
|
+
const vectorStore = await SQLiteVecStorage.create(VECTORS_PATH);
|
|
961
978
|
const project = sqlite.getProject(projectId);
|
|
962
979
|
if (!project) {
|
|
963
980
|
sqlite.insertProject({
|
|
@@ -1000,14 +1017,14 @@ async function main() {
|
|
|
1000
1017
|
}
|
|
1001
1018
|
sqlite.insertMemory(memory);
|
|
1002
1019
|
const vector = await embedder.embed(content);
|
|
1003
|
-
await
|
|
1020
|
+
await vectorStore.insertVector(memoryId, vector);
|
|
1004
1021
|
}
|
|
1005
1022
|
const dedupeOnPrune = config.storage?.deduplicateOnPrune ?? true;
|
|
1006
1023
|
const dedupeThreshold = config.retrieval?.deduplicationThreshold ?? 0.85;
|
|
1007
1024
|
if (dedupeOnPrune) {
|
|
1008
1025
|
const dedupedIds = await deduplicateStoredMemories(
|
|
1009
1026
|
sqlite,
|
|
1010
|
-
|
|
1027
|
+
vectorStore,
|
|
1011
1028
|
projectId,
|
|
1012
1029
|
dedupeThreshold
|
|
1013
1030
|
);
|
|
@@ -1026,12 +1043,12 @@ async function main() {
|
|
|
1026
1043
|
if (pruneResult.deleted > 0) {
|
|
1027
1044
|
log("Stop", `Pruned ${pruneResult.deleted} old memories to stay within limits`);
|
|
1028
1045
|
for (const id of pruneResult.deletedIds) {
|
|
1029
|
-
await
|
|
1046
|
+
await vectorStore.deleteVector(id);
|
|
1030
1047
|
}
|
|
1031
1048
|
}
|
|
1032
1049
|
}
|
|
1033
1050
|
sqlite.close();
|
|
1034
|
-
await
|
|
1051
|
+
await vectorStore.close();
|
|
1035
1052
|
await embedder.close();
|
|
1036
1053
|
log("Stop", `SUCCESS: Saved ${captures.length} memories`);
|
|
1037
1054
|
outputResult({ continue: true, suppressOutput: true });
|
|
@@ -1066,13 +1083,13 @@ async function loadConfig() {
|
|
|
1066
1083
|
function outputResult(result) {
|
|
1067
1084
|
console.log(JSON.stringify(result));
|
|
1068
1085
|
}
|
|
1069
|
-
async function deduplicateStoredMemories(sqlite,
|
|
1086
|
+
async function deduplicateStoredMemories(sqlite, vectorStore, projectId, threshold) {
|
|
1070
1087
|
const deletedIds = [];
|
|
1071
1088
|
const memories = sqlite.getRecentMemories(projectId, 0, 0);
|
|
1072
1089
|
if (memories.length < 2)
|
|
1073
1090
|
return deletedIds;
|
|
1074
1091
|
const memoryIds = memories.map((m) => m.id);
|
|
1075
|
-
const vectors = await
|
|
1092
|
+
const vectors = await vectorStore.getVectorsByIds(memoryIds);
|
|
1076
1093
|
if (vectors.size < 2)
|
|
1077
1094
|
return deletedIds;
|
|
1078
1095
|
const keepIds = /* @__PURE__ */ new Set();
|
|
@@ -1096,7 +1113,7 @@ async function deduplicateStoredMemories(sqlite, lancedb2, projectId, threshold)
|
|
|
1096
1113
|
keptVectors.push({ id: memory.id, vector });
|
|
1097
1114
|
} else {
|
|
1098
1115
|
sqlite.deleteMemory(memory.id);
|
|
1099
|
-
await
|
|
1116
|
+
await vectorStore.deleteVector(memory.id);
|
|
1100
1117
|
deletedIds.push(memory.id);
|
|
1101
1118
|
}
|
|
1102
1119
|
}
|