@equationalapplications/core-llm-wiki 4.6.1 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +1463 -664
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1463 -664
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -62,6 +62,19 @@ async function setupDatabase(db, prefix) {
|
|
|
62
62
|
key TEXT PRIMARY KEY,
|
|
63
63
|
value TEXT NOT NULL
|
|
64
64
|
);
|
|
65
|
+
|
|
66
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
67
|
+
id TEXT PRIMARY KEY,
|
|
68
|
+
entity_id TEXT NOT NULL,
|
|
69
|
+
table_name TEXT NOT NULL,
|
|
70
|
+
record_id TEXT NOT NULL,
|
|
71
|
+
operation TEXT NOT NULL,
|
|
72
|
+
payload TEXT NOT NULL,
|
|
73
|
+
created_at INTEGER NOT NULL
|
|
74
|
+
);
|
|
75
|
+
|
|
76
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
77
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
65
78
|
`);
|
|
66
79
|
}
|
|
67
80
|
|
|
@@ -77,8 +90,8 @@ var MIGRATIONS = [
|
|
|
77
90
|
version: 2,
|
|
78
91
|
description: "Remove FTS5; add embedding column for semantic retrieval",
|
|
79
92
|
run: async (db, prefix) => {
|
|
80
|
-
await db.withTransactionAsync(async () => {
|
|
81
|
-
await
|
|
93
|
+
await db.withTransactionAsync(async (tx) => {
|
|
94
|
+
await tx.execAsync(`
|
|
82
95
|
DROP TRIGGER IF EXISTS ${prefix}entries_ai;
|
|
83
96
|
DROP TRIGGER IF EXISTS ${prefix}entries_ad;
|
|
84
97
|
DROP TRIGGER IF EXISTS ${prefix}entries_au;
|
|
@@ -106,6 +119,25 @@ var MIGRATIONS = [
|
|
|
106
119
|
);
|
|
107
120
|
}
|
|
108
121
|
}
|
|
122
|
+
},
|
|
123
|
+
{
|
|
124
|
+
version: 4,
|
|
125
|
+
description: "Create outbox table for change data capture",
|
|
126
|
+
run: async (db, prefix) => {
|
|
127
|
+
await db.execAsync(`
|
|
128
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
129
|
+
id TEXT PRIMARY KEY,
|
|
130
|
+
entity_id TEXT NOT NULL,
|
|
131
|
+
table_name TEXT NOT NULL,
|
|
132
|
+
record_id TEXT NOT NULL,
|
|
133
|
+
operation TEXT NOT NULL,
|
|
134
|
+
payload TEXT NOT NULL,
|
|
135
|
+
created_at INTEGER NOT NULL
|
|
136
|
+
);
|
|
137
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
138
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
139
|
+
`);
|
|
140
|
+
}
|
|
109
141
|
}
|
|
110
142
|
];
|
|
111
143
|
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
@@ -183,8 +215,9 @@ function mapRowToFact(row) {
|
|
|
183
215
|
};
|
|
184
216
|
}
|
|
185
217
|
var EntryRepository = class extends BaseRepository {
|
|
186
|
-
constructor() {
|
|
187
|
-
super(
|
|
218
|
+
constructor(db, prefix, outbox) {
|
|
219
|
+
super(db, prefix);
|
|
220
|
+
this.outbox = outbox;
|
|
188
221
|
this.chunkSize = 500;
|
|
189
222
|
}
|
|
190
223
|
/**
|
|
@@ -205,89 +238,1180 @@ var EntryRepository = class extends BaseRepository {
|
|
|
205
238
|
);
|
|
206
239
|
rows.push(...chunkRows);
|
|
207
240
|
}
|
|
208
|
-
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
209
|
-
return ids.map((id) => byId.get(id)).filter((r) => r !== void 0).map(mapRowToFact);
|
|
241
|
+
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
242
|
+
return ids.map((id) => byId.get(id)).filter((r) => r !== void 0).map(mapRowToFact);
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Upsert a WikiFact. Nullable fields set to null when fact value is null.
|
|
246
|
+
* Returns { changes, lastInsertRowId }.
|
|
247
|
+
* `tx` is REQUIRED to ensure atomic outbox staging.
|
|
248
|
+
*/
|
|
249
|
+
async upsert(fact, tx) {
|
|
250
|
+
const executor = this.getExecutor(tx);
|
|
251
|
+
const now = Date.now();
|
|
252
|
+
const tagsJson = JSON.stringify(fact.tags);
|
|
253
|
+
const embeddingBlob = this.normalizeEmbeddingBlob(fact.embedding_blob);
|
|
254
|
+
const existingRow = await executor.getFirstAsync(
|
|
255
|
+
`SELECT id FROM ${this.prefix}entries WHERE id = ?`,
|
|
256
|
+
[fact.id]
|
|
257
|
+
);
|
|
258
|
+
const operation = fact.deleted_at ? "DELETE" : existingRow ? "UPDATE" : "INSERT";
|
|
259
|
+
const result = await executor.runAsync(
|
|
260
|
+
`INSERT INTO ${this.prefix}entries (
|
|
261
|
+
id, entity_id, title, body, tags, confidence, source_type,
|
|
262
|
+
source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count,
|
|
263
|
+
deleted_at, embedding_blob, embedding
|
|
264
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
265
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
266
|
+
entity_id = excluded.entity_id,
|
|
267
|
+
title = excluded.title,
|
|
268
|
+
body = excluded.body,
|
|
269
|
+
tags = excluded.tags,
|
|
270
|
+
confidence = excluded.confidence,
|
|
271
|
+
source_type = excluded.source_type,
|
|
272
|
+
source_hash = excluded.source_hash,
|
|
273
|
+
source_ref = excluded.source_ref,
|
|
274
|
+
updated_at = excluded.updated_at,
|
|
275
|
+
last_accessed_at = excluded.last_accessed_at,
|
|
276
|
+
access_count = excluded.access_count,
|
|
277
|
+
deleted_at = excluded.deleted_at,
|
|
278
|
+
embedding_blob = CASE WHEN excluded.embedding_blob IS NULL THEN embedding_blob ELSE excluded.embedding_blob END,
|
|
279
|
+
embedding = NULL`,
|
|
280
|
+
[
|
|
281
|
+
fact.id,
|
|
282
|
+
fact.entity_id,
|
|
283
|
+
fact.title,
|
|
284
|
+
fact.body,
|
|
285
|
+
tagsJson,
|
|
286
|
+
fact.confidence,
|
|
287
|
+
fact.source_type,
|
|
288
|
+
fact.source_hash,
|
|
289
|
+
fact.source_ref,
|
|
290
|
+
fact.created_at,
|
|
291
|
+
now,
|
|
292
|
+
fact.last_accessed_at === null ? null : fact.last_accessed_at,
|
|
293
|
+
fact.access_count,
|
|
294
|
+
fact.deleted_at ?? null,
|
|
295
|
+
embeddingBlob ?? null,
|
|
296
|
+
null
|
|
297
|
+
]
|
|
298
|
+
);
|
|
299
|
+
await this.outbox.push({
|
|
300
|
+
entityId: fact.entity_id,
|
|
301
|
+
tableName: "entries",
|
|
302
|
+
recordId: fact.id,
|
|
303
|
+
operation,
|
|
304
|
+
payload: fact
|
|
305
|
+
}, tx);
|
|
306
|
+
return result;
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Normalize an embedding blob value to Uint8Array or null.
|
|
310
|
+
*/
|
|
311
|
+
normalizeEmbeddingBlob(blob) {
|
|
312
|
+
if (blob instanceof Uint8Array) return blob;
|
|
313
|
+
if (blob !== null && blob !== void 0 && typeof blob === "object") {
|
|
314
|
+
const obj = blob;
|
|
315
|
+
if (obj["type"] === "Buffer" && Array.isArray(obj["data"])) {
|
|
316
|
+
return new Uint8Array(obj["data"]);
|
|
317
|
+
}
|
|
318
|
+
const entries = Object.keys(obj);
|
|
319
|
+
if (entries.length > 0 && entries.every((k) => /^\d+$/.test(k))) {
|
|
320
|
+
const len = entries.length;
|
|
321
|
+
const arr = new Uint8Array(len);
|
|
322
|
+
for (let i = 0; i < len; i++) arr[i] = obj[String(i)] ?? 0;
|
|
323
|
+
return arr;
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
return null;
|
|
327
|
+
}
|
|
328
|
+
/**
|
|
329
|
+
* Fetch existing rows by IDs and return id/entity_id/updated_at for import collision resolution.
|
|
330
|
+
*/
|
|
331
|
+
async findExistingMetadataByIds(ids, tx) {
|
|
332
|
+
const executor = this.getExecutor(tx);
|
|
333
|
+
const rows = [];
|
|
334
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
335
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
336
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
337
|
+
const chunkRows = await executor.getAllAsync(
|
|
338
|
+
`SELECT id, entity_id, updated_at FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
|
|
339
|
+
chunk
|
|
340
|
+
);
|
|
341
|
+
rows.push(...chunkRows.map((row) => ({ id: row.id, entity_id: row.entity_id, updated_at: Number(row.updated_at) })));
|
|
342
|
+
}
|
|
343
|
+
return rows;
|
|
344
|
+
}
|
|
345
|
+
async findIdById(id, entityId, tx) {
|
|
346
|
+
const executor = this.getExecutor(tx);
|
|
347
|
+
const row = await executor.getFirstAsync(
|
|
348
|
+
`SELECT id FROM ${this.prefix}entries WHERE id = ? AND entity_id = ?`,
|
|
349
|
+
[id, entityId]
|
|
350
|
+
);
|
|
351
|
+
return row?.id ?? null;
|
|
352
|
+
}
|
|
353
|
+
async findIdsBySource(entityId, sourceRef, sourceHash, tx, includeDeleted = false) {
|
|
354
|
+
const executor = this.getExecutor(tx);
|
|
355
|
+
let sql = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ?`;
|
|
356
|
+
const args = [entityId];
|
|
357
|
+
if (sourceRef !== null) {
|
|
358
|
+
sql += ` AND source_ref = ?`;
|
|
359
|
+
args.push(sourceRef);
|
|
360
|
+
}
|
|
361
|
+
if (sourceHash !== null) {
|
|
362
|
+
sql += ` AND source_hash = ?`;
|
|
363
|
+
args.push(sourceHash);
|
|
364
|
+
}
|
|
365
|
+
if (!includeDeleted) {
|
|
366
|
+
sql += ` AND deleted_at IS NULL`;
|
|
367
|
+
}
|
|
368
|
+
const rows = await executor.getAllAsync(sql, args);
|
|
369
|
+
return rows.map((row) => row.id);
|
|
370
|
+
}
|
|
371
|
+
async upsertForImport(fact, tx) {
|
|
372
|
+
const executor = this.getExecutor(tx);
|
|
373
|
+
const tagsJson = JSON.stringify(fact.tags);
|
|
374
|
+
const embeddingBlob = this.normalizeEmbeddingBlob(fact.embedding_blob);
|
|
375
|
+
const result = await executor.runAsync(
|
|
376
|
+
`INSERT INTO ${this.prefix}entries (
|
|
377
|
+
id, entity_id, title, body, tags, confidence, source_type,
|
|
378
|
+
source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count,
|
|
379
|
+
deleted_at, embedding_blob, embedding
|
|
380
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
381
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
382
|
+
entity_id = excluded.entity_id,
|
|
383
|
+
title = excluded.title,
|
|
384
|
+
body = excluded.body,
|
|
385
|
+
tags = excluded.tags,
|
|
386
|
+
confidence = excluded.confidence,
|
|
387
|
+
source_type = excluded.source_type,
|
|
388
|
+
source_hash = excluded.source_hash,
|
|
389
|
+
source_ref = excluded.source_ref,
|
|
390
|
+
created_at = excluded.created_at,
|
|
391
|
+
updated_at = excluded.updated_at,
|
|
392
|
+
last_accessed_at = excluded.last_accessed_at,
|
|
393
|
+
access_count = excluded.access_count,
|
|
394
|
+
deleted_at = excluded.deleted_at,
|
|
395
|
+
embedding_blob = excluded.embedding_blob,
|
|
396
|
+
embedding = NULL`,
|
|
397
|
+
[
|
|
398
|
+
fact.id,
|
|
399
|
+
fact.entity_id,
|
|
400
|
+
fact.title,
|
|
401
|
+
fact.body,
|
|
402
|
+
tagsJson,
|
|
403
|
+
fact.confidence,
|
|
404
|
+
fact.source_type,
|
|
405
|
+
fact.source_hash,
|
|
406
|
+
fact.source_ref,
|
|
407
|
+
fact.created_at,
|
|
408
|
+
fact.updated_at,
|
|
409
|
+
fact.last_accessed_at === null ? null : fact.last_accessed_at,
|
|
410
|
+
fact.access_count,
|
|
411
|
+
fact.deleted_at ?? null,
|
|
412
|
+
embeddingBlob ?? null,
|
|
413
|
+
null
|
|
414
|
+
]
|
|
415
|
+
);
|
|
416
|
+
return result;
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* Soft-delete a single entry by ID scoped to entityId. Sets deleted_at + updated_at.
|
|
420
|
+
* `tx` is REQUIRED to ensure atomic outbox staging.
|
|
421
|
+
*/
|
|
422
|
+
async softDelete(entryId, entityId, tx) {
|
|
423
|
+
const executor = this.getExecutor(tx);
|
|
424
|
+
const now = Date.now();
|
|
425
|
+
const result = await executor.runAsync(
|
|
426
|
+
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
427
|
+
[now, now, entryId, entityId]
|
|
428
|
+
);
|
|
429
|
+
await this.outbox.push({
|
|
430
|
+
entityId,
|
|
431
|
+
tableName: "entries",
|
|
432
|
+
recordId: entryId,
|
|
433
|
+
operation: "DELETE",
|
|
434
|
+
payload: { id: entryId, entity_id: entityId, deleted_at: now }
|
|
435
|
+
}, tx);
|
|
436
|
+
return result;
|
|
437
|
+
}
|
|
438
|
+
/**
|
|
439
|
+
* Soft-delete entries by source_ref and/or source_hash within a transaction.
|
|
440
|
+
* Stages a DELETE outbox entry for each row in the same transaction.
|
|
441
|
+
* `tx` is REQUIRED.
|
|
442
|
+
* Returns the number of rows deleted.
|
|
443
|
+
*/
|
|
444
|
+
async softDeleteBySource(entityId, tx, sourceRef, sourceHash) {
|
|
445
|
+
const executor = this.getExecutor(tx);
|
|
446
|
+
const now = Date.now();
|
|
447
|
+
let q = `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`;
|
|
448
|
+
const args = [now, now, entityId];
|
|
449
|
+
if (sourceRef) {
|
|
450
|
+
q += ` AND source_ref = ?`;
|
|
451
|
+
args.push(sourceRef);
|
|
452
|
+
}
|
|
453
|
+
if (sourceHash) {
|
|
454
|
+
q += ` AND source_hash = ?`;
|
|
455
|
+
args.push(sourceHash);
|
|
456
|
+
}
|
|
457
|
+
let selectQ = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`;
|
|
458
|
+
const selectArgs = [entityId];
|
|
459
|
+
if (sourceRef) {
|
|
460
|
+
selectQ += ` AND source_ref = ?`;
|
|
461
|
+
selectArgs.push(sourceRef);
|
|
462
|
+
}
|
|
463
|
+
if (sourceHash) {
|
|
464
|
+
selectQ += ` AND source_hash = ?`;
|
|
465
|
+
selectArgs.push(sourceHash);
|
|
466
|
+
}
|
|
467
|
+
const idsToDelete = await executor.getAllAsync(selectQ, selectArgs);
|
|
468
|
+
const result = await executor.runAsync(q, args);
|
|
469
|
+
for (const row of idsToDelete) {
|
|
470
|
+
await this.outbox.push({
|
|
471
|
+
entityId,
|
|
472
|
+
tableName: "entries",
|
|
473
|
+
recordId: row.id,
|
|
474
|
+
operation: "DELETE",
|
|
475
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: now }
|
|
476
|
+
}, tx);
|
|
477
|
+
}
|
|
478
|
+
return result.changes;
|
|
479
|
+
}
|
|
480
|
+
/**
|
|
481
|
+
* Fetch IDs + entity_ids of soft-deleted rows older than cutoff for a given entity.
|
|
482
|
+
* Used by runPrune().
|
|
483
|
+
*/
|
|
484
|
+
async getPrunableMetadata(entityId, cutoff, tx) {
|
|
485
|
+
const executor = this.getExecutor(tx);
|
|
486
|
+
return executor.getAllAsync(
|
|
487
|
+
`SELECT id, entity_id FROM ${this.prefix}entries
|
|
488
|
+
WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
|
|
489
|
+
[entityId, cutoff]
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
/**
|
|
493
|
+
* Fetch all non-deleted entries for an entity, ordered by updated_at DESC.
|
|
494
|
+
* Used by _getFullBundle().
|
|
495
|
+
*/
|
|
496
|
+
async findAllByEntityId(entityId, tx) {
|
|
497
|
+
const executor = this.getExecutor(tx);
|
|
498
|
+
const rows = await executor.getAllAsync(
|
|
499
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC`,
|
|
500
|
+
[entityId]
|
|
501
|
+
);
|
|
502
|
+
return rows.map(mapRowToFact);
|
|
503
|
+
}
|
|
504
|
+
/**
|
|
505
|
+
* Fetch recent non-deleted entries for an entity (limited), ordered by updated_at DESC.
|
|
506
|
+
* Used by _doRunLibrarian().
|
|
507
|
+
*/
|
|
508
|
+
async findRecentByEntityId(entityId, limit, tx) {
|
|
509
|
+
const executor = this.getExecutor(tx);
|
|
510
|
+
const rows = await executor.getAllAsync(
|
|
511
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC LIMIT ?`,
|
|
512
|
+
[entityId, limit]
|
|
513
|
+
);
|
|
514
|
+
return rows.map(mapRowToFact);
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* Count non-deleted entries for the given entities whose embedding_blob dimension
|
|
518
|
+
* doesn't match queryVecLength. Used by read() to detect model-switch mismatches.
|
|
519
|
+
*/
|
|
520
|
+
async countDimensionMismatched(entityIds, queryVecLength, tx) {
|
|
521
|
+
if (entityIds.length === 0) return 0;
|
|
522
|
+
const executor = this.getExecutor(tx);
|
|
523
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
524
|
+
const row = await executor.getFirstAsync(
|
|
525
|
+
`SELECT COUNT(*) AS cnt FROM ${this.prefix}entries
|
|
526
|
+
WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL
|
|
527
|
+
AND embedding_blob IS NOT NULL
|
|
528
|
+
AND (CAST(length(embedding_blob) AS INTEGER) % 4 = 0)
|
|
529
|
+
AND (CAST(length(embedding_blob) AS INTEGER) / 4) != ?`,
|
|
530
|
+
[...entityIds, queryVecLength]
|
|
531
|
+
);
|
|
532
|
+
return row?.cnt ?? 0;
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* Count non-deleted entries for entityId that are stale relative to targetDim
|
|
536
|
+
* (either no blob or wrong dimension). Used by runReembed() per-entity skip logic.
|
|
537
|
+
*/
|
|
538
|
+
async countStaleForEntity(entityId, targetDim, tx) {
|
|
539
|
+
const executor = this.getExecutor(tx);
|
|
540
|
+
const row = await executor.getFirstAsync(
|
|
541
|
+
`SELECT COUNT(*) AS cnt FROM ${this.prefix}entries
|
|
542
|
+
WHERE entity_id = ? AND deleted_at IS NULL
|
|
543
|
+
AND (
|
|
544
|
+
embedding_blob IS NULL
|
|
545
|
+
OR (CAST(length(embedding_blob) AS INTEGER) / 4) != ?
|
|
546
|
+
)`,
|
|
547
|
+
[entityId, targetDim]
|
|
548
|
+
);
|
|
549
|
+
return row?.cnt ?? 0;
|
|
550
|
+
}
|
|
551
|
+
/**
|
|
552
|
+
* Count non-deleted entries with stale or unconverted embeddings relative to `dim`.
|
|
553
|
+
* Used by _reconcileEmbeddingDimension() to decide when to promote the pending
|
|
554
|
+
* embedding_dimension value.
|
|
555
|
+
*/
|
|
556
|
+
async countStaleEmbeddings(dim, tx) {
|
|
557
|
+
const executor = this.getExecutor(tx);
|
|
558
|
+
const row = await executor.getFirstAsync(
|
|
559
|
+
`SELECT COUNT(*) AS cnt FROM ${this.prefix}entries
|
|
560
|
+
WHERE deleted_at IS NULL
|
|
561
|
+
AND (
|
|
562
|
+
(embedding_blob IS NOT NULL AND (CAST(length(embedding_blob) AS INTEGER) / 4) != ?)
|
|
563
|
+
OR (embedding_blob IS NULL AND embedding IS NOT NULL)
|
|
564
|
+
)`,
|
|
565
|
+
[dim]
|
|
566
|
+
);
|
|
567
|
+
return row?.cnt ?? 0;
|
|
568
|
+
}
|
|
569
|
+
/**
|
|
570
|
+
* Bulk delete pruned entries (already soft-deleted) by IDs.
|
|
571
|
+
* Used by runPrune(). Returns total number of deleted rows.
|
|
572
|
+
* `tx` is REQUIRED so outbox deletion events are staged atomically.
|
|
573
|
+
*/
|
|
574
|
+
async bulkDeletePruned(entityId, cutoff, ids, tx) {
|
|
575
|
+
const executor = this.getExecutor(tx);
|
|
576
|
+
let totalChanges = 0;
|
|
577
|
+
const chunkSize = 500;
|
|
578
|
+
for (let i = 0; i < ids.length; i += chunkSize) {
|
|
579
|
+
const chunk = ids.slice(i, i + chunkSize);
|
|
580
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
581
|
+
const result = await executor.runAsync(
|
|
582
|
+
`DELETE FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ? AND id IN (${placeholders})`,
|
|
583
|
+
[entityId, cutoff, ...chunk]
|
|
584
|
+
);
|
|
585
|
+
totalChanges += result.changes;
|
|
586
|
+
if (result.changes > 0) {
|
|
587
|
+
for (const id of chunk) {
|
|
588
|
+
await this.outbox.push({
|
|
589
|
+
entityId,
|
|
590
|
+
tableName: "entries",
|
|
591
|
+
recordId: id,
|
|
592
|
+
operation: "DELETE",
|
|
593
|
+
payload: { id, entity_id: entityId, deleted_at: cutoff }
|
|
594
|
+
}, tx);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
return totalChanges;
|
|
599
|
+
}
|
|
600
|
+
/**
|
|
601
|
+
* Mark orphaned entries (never accessed, old) as deleted.
|
|
602
|
+
* Used by _doRunHeal().
|
|
603
|
+
*/
|
|
604
|
+
async markOrphaned(entityId, orphanThreshold, tx) {
|
|
605
|
+
const executor = this.getExecutor(tx);
|
|
606
|
+
const now = Date.now();
|
|
607
|
+
const orphanedRows = await executor.getAllAsync(
|
|
608
|
+
`SELECT id FROM ${this.prefix}entries
|
|
609
|
+
WHERE entity_id = ? AND access_count = 0 AND created_at <= ? AND source_type != 'immutable_document' AND deleted_at IS NULL`,
|
|
610
|
+
[entityId, orphanThreshold]
|
|
611
|
+
);
|
|
612
|
+
if (orphanedRows.length === 0) return 0;
|
|
613
|
+
const result = await executor.runAsync(
|
|
614
|
+
`UPDATE ${this.prefix}entries
|
|
615
|
+
SET deleted_at = ?, updated_at = ?
|
|
616
|
+
WHERE entity_id = ? AND access_count = 0 AND created_at <= ? AND source_type != 'immutable_document' AND deleted_at IS NULL`,
|
|
617
|
+
[now, now, entityId, orphanThreshold]
|
|
618
|
+
);
|
|
619
|
+
for (const row of orphanedRows) {
|
|
620
|
+
await this.outbox.push({
|
|
621
|
+
entityId,
|
|
622
|
+
tableName: "entries",
|
|
623
|
+
recordId: row.id,
|
|
624
|
+
operation: "DELETE",
|
|
625
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: now }
|
|
626
|
+
}, tx);
|
|
627
|
+
}
|
|
628
|
+
return result.changes;
|
|
629
|
+
}
|
|
630
|
+
/**
|
|
631
|
+
* Downgrade stale inferred entries to 'tentative'.
|
|
632
|
+
* Used by _doRunHeal().
|
|
633
|
+
*/
|
|
634
|
+
async downgradeStaleInferred(entityId, staleThreshold, tx) {
|
|
635
|
+
const executor = this.getExecutor(tx);
|
|
636
|
+
const now = Date.now();
|
|
637
|
+
const eligibleRows = await executor.getAllAsync(
|
|
638
|
+
`SELECT id FROM ${this.prefix}entries
|
|
639
|
+
WHERE entity_id = ? AND confidence = 'inferred'
|
|
640
|
+
AND (last_accessed_at <= ? OR (last_accessed_at IS NULL AND created_at <= ?))
|
|
641
|
+
AND source_type != 'immutable_document' AND deleted_at IS NULL`,
|
|
642
|
+
[entityId, staleThreshold, staleThreshold]
|
|
643
|
+
);
|
|
644
|
+
if (eligibleRows.length === 0) return 0;
|
|
645
|
+
const result = await executor.runAsync(
|
|
646
|
+
`UPDATE ${this.prefix}entries
|
|
647
|
+
SET confidence = 'tentative', updated_at = ?
|
|
648
|
+
WHERE entity_id = ? AND confidence = 'inferred' AND (last_accessed_at <= ? OR (last_accessed_at IS NULL AND created_at <= ?)) AND source_type != 'immutable_document' AND deleted_at IS NULL`,
|
|
649
|
+
[now, entityId, staleThreshold, staleThreshold]
|
|
650
|
+
);
|
|
651
|
+
for (const row of eligibleRows) {
|
|
652
|
+
await this.outbox.push({
|
|
653
|
+
entityId,
|
|
654
|
+
tableName: "entries",
|
|
655
|
+
recordId: row.id,
|
|
656
|
+
operation: "UPDATE",
|
|
657
|
+
payload: { id: row.id, entity_id: entityId, confidence: "tentative", updated_at: now }
|
|
658
|
+
}, tx);
|
|
659
|
+
}
|
|
660
|
+
return result.changes;
|
|
661
|
+
}
|
|
662
|
+
/**
|
|
663
|
+
* Downgrade specific entries to 'tentative' by IDs.
|
|
664
|
+
* Used by _doRunHeal().
|
|
665
|
+
*/
|
|
666
|
+
async downgradeByIds(ids, entityId, tx) {
|
|
667
|
+
if (ids.length === 0) return;
|
|
668
|
+
const executor = this.getExecutor(tx);
|
|
669
|
+
const now = Date.now();
|
|
670
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
671
|
+
await executor.runAsync(
|
|
672
|
+
`UPDATE ${this.prefix}entries SET confidence = 'tentative', updated_at = ? WHERE id IN (${placeholders}) AND entity_id = ?`,
|
|
673
|
+
[now, ...ids, entityId]
|
|
674
|
+
);
|
|
675
|
+
for (const id of ids) {
|
|
676
|
+
await this.outbox.push({
|
|
677
|
+
entityId,
|
|
678
|
+
tableName: "entries",
|
|
679
|
+
recordId: id,
|
|
680
|
+
operation: "UPDATE",
|
|
681
|
+
payload: { id, entity_id: entityId, confidence: "tentative", updated_at: now }
|
|
682
|
+
}, tx);
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
/**
|
|
686
|
+
* Soft-delete specific entries by IDs.
|
|
687
|
+
* Used by _doRunHeal().
|
|
688
|
+
*/
|
|
689
|
+
async softDeleteByIds(ids, entityId, tx) {
|
|
690
|
+
if (ids.length === 0) return;
|
|
691
|
+
const executor = this.getExecutor(tx);
|
|
692
|
+
const now = Date.now();
|
|
693
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
694
|
+
await executor.runAsync(
|
|
695
|
+
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id IN (${placeholders}) AND entity_id = ?`,
|
|
696
|
+
[now, now, ...ids, entityId]
|
|
697
|
+
);
|
|
698
|
+
for (const id of ids) {
|
|
699
|
+
await this.outbox.push({
|
|
700
|
+
entityId,
|
|
701
|
+
tableName: "entries",
|
|
702
|
+
recordId: id,
|
|
703
|
+
operation: "DELETE",
|
|
704
|
+
payload: { id, entity_id: entityId, deleted_at: now }
|
|
705
|
+
}, tx);
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
/**
|
|
709
|
+
* Bulk soft-delete all entries for an entity.
|
|
710
|
+
* Stages DELETE outbox entries for each row in the same transaction.
|
|
711
|
+
* `tx` is REQUIRED.
|
|
712
|
+
*/
|
|
713
|
+
async bulkSoftDeleteByEntityId(entityId, tx) {
|
|
714
|
+
const executor = this.getExecutor(tx);
|
|
715
|
+
const now = Date.now();
|
|
716
|
+
const idsToDelete = await executor.getAllAsync(
|
|
717
|
+
`SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
718
|
+
[entityId]
|
|
719
|
+
);
|
|
720
|
+
const result = await executor.runAsync(
|
|
721
|
+
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
722
|
+
[now, now, entityId]
|
|
723
|
+
);
|
|
724
|
+
for (const row of idsToDelete) {
|
|
725
|
+
await this.outbox.push({
|
|
726
|
+
entityId,
|
|
727
|
+
tableName: "entries",
|
|
728
|
+
recordId: row.id,
|
|
729
|
+
operation: "DELETE",
|
|
730
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: now }
|
|
731
|
+
}, tx);
|
|
732
|
+
}
|
|
733
|
+
return result.changes;
|
|
734
|
+
}
|
|
735
|
+
async findMiniSearchRows(entityId, tx) {
|
|
736
|
+
const executor = this.getExecutor(tx);
|
|
737
|
+
if (entityId !== void 0) {
|
|
738
|
+
return executor.getAllAsync(
|
|
739
|
+
`SELECT id, entity_id, title, body, tags FROM ${this.prefix}entries WHERE deleted_at IS NULL AND entity_id = ?`,
|
|
740
|
+
[entityId]
|
|
741
|
+
);
|
|
742
|
+
}
|
|
743
|
+
return executor.getAllAsync(
|
|
744
|
+
`SELECT id, entity_id, title, body, tags FROM ${this.prefix}entries WHERE deleted_at IS NULL`
|
|
745
|
+
);
|
|
746
|
+
}
|
|
747
|
+
async updateEmbeddingBlob(id, blob, tx) {
|
|
748
|
+
const executor = this.getExecutor(tx);
|
|
749
|
+
await executor.runAsync(
|
|
750
|
+
`UPDATE ${this.prefix}entries SET embedding_blob = ?, embedding = NULL WHERE id = ?`,
|
|
751
|
+
[blob, id]
|
|
752
|
+
);
|
|
753
|
+
}
|
|
754
|
+
async hasLegacySourceTypes(tx) {
|
|
755
|
+
const executor = this.getExecutor(tx);
|
|
756
|
+
const row = await executor.getFirstAsync(
|
|
757
|
+
`SELECT 1 AS one FROM ${this.prefix}entries WHERE source_type IN ('user_document', 'agent_inferred') LIMIT 1`,
|
|
758
|
+
[]
|
|
759
|
+
);
|
|
760
|
+
return row != null;
|
|
761
|
+
}
|
|
762
|
+
async countLegacySourceTypes(tx) {
|
|
763
|
+
const executor = this.getExecutor(tx);
|
|
764
|
+
const row = await executor.getFirstAsync(
|
|
765
|
+
`SELECT COUNT(*) as count FROM ${this.prefix}entries WHERE source_type IN ('user_document', 'agent_inferred')`,
|
|
766
|
+
[]
|
|
767
|
+
);
|
|
768
|
+
return row?.count ?? 0;
|
|
769
|
+
}
|
|
770
|
+
async findAllForReembed(entityId, tx) {
|
|
771
|
+
const executor = this.getExecutor(tx);
|
|
772
|
+
if (entityId !== void 0) {
|
|
773
|
+
return executor.getAllAsync(
|
|
774
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
775
|
+
[entityId]
|
|
776
|
+
);
|
|
777
|
+
}
|
|
778
|
+
return executor.getAllAsync(
|
|
779
|
+
`SELECT * FROM ${this.prefix}entries WHERE deleted_at IS NULL`
|
|
780
|
+
);
|
|
781
|
+
}
|
|
782
|
+
async findRowsForSourceRefMigration(tx) {
|
|
783
|
+
const executor = this.getExecutor(tx);
|
|
784
|
+
return executor.getAllAsync(
|
|
785
|
+
`SELECT rowid, source_ref FROM ${this.prefix}entries
|
|
786
|
+
WHERE source_ref IS NOT NULL
|
|
787
|
+
AND (
|
|
788
|
+
TRIM(source_ref) != source_ref
|
|
789
|
+
OR INSTR(source_ref, '/') > 0
|
|
790
|
+
OR INSTR(source_ref, '\\') > 0
|
|
791
|
+
OR INSTR(source_ref, CHAR(0)) > 0
|
|
792
|
+
OR source_ref GLOB '*[^-A-Za-z0-9._ ]*'
|
|
793
|
+
)`
|
|
794
|
+
);
|
|
795
|
+
}
|
|
796
|
+
async updateSourceRefByRowid(rowid, sourceRef, tx) {
|
|
797
|
+
const executor = this.getExecutor(tx);
|
|
798
|
+
await executor.runAsync(
|
|
799
|
+
`UPDATE ${this.prefix}entries SET source_ref = ? WHERE rowid = ?`,
|
|
800
|
+
[sourceRef, rowid]
|
|
801
|
+
);
|
|
802
|
+
}
|
|
803
|
+
async findLatestSourceHash(entityId, sourceRef, tx) {
|
|
804
|
+
const executor = this.getExecutor(tx);
|
|
805
|
+
const row = await executor.getFirstAsync(
|
|
806
|
+
`SELECT source_hash FROM ${this.prefix}entries
|
|
807
|
+
WHERE entity_id = ? AND source_ref = ? AND deleted_at IS NULL
|
|
808
|
+
ORDER BY updated_at DESC
|
|
809
|
+
LIMIT 1`,
|
|
810
|
+
[entityId, sourceRef]
|
|
811
|
+
);
|
|
812
|
+
return row?.source_hash ?? null;
|
|
813
|
+
}
|
|
814
|
+
async findMetadataByIds(ids, tx) {
|
|
815
|
+
if (ids.length === 0) return [];
|
|
816
|
+
const executor = this.getExecutor(tx);
|
|
817
|
+
const rows = [];
|
|
818
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
819
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
820
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
821
|
+
const chunkRows = await executor.getAllAsync(
|
|
822
|
+
`SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
823
|
+
chunk
|
|
824
|
+
);
|
|
825
|
+
rows.push(...chunkRows);
|
|
826
|
+
}
|
|
827
|
+
return rows;
|
|
828
|
+
}
|
|
829
|
+
async findWithEmbeddingsByIds(ids, tx) {
|
|
830
|
+
if (ids.length === 0) return [];
|
|
831
|
+
const executor = this.getExecutor(tx);
|
|
832
|
+
const rows = [];
|
|
833
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
834
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
835
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
836
|
+
const chunkRows = await executor.getAllAsync(
|
|
837
|
+
`SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
838
|
+
chunk
|
|
839
|
+
);
|
|
840
|
+
rows.push(...chunkRows);
|
|
841
|
+
}
|
|
842
|
+
return rows;
|
|
843
|
+
}
|
|
844
|
+
async findMetadataByEntityIds(entityIds, tx) {
|
|
845
|
+
if (entityIds.length === 0) return [];
|
|
846
|
+
const executor = this.getExecutor(tx);
|
|
847
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
848
|
+
return executor.getAllAsync(
|
|
849
|
+
`SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
850
|
+
[...entityIds]
|
|
851
|
+
);
|
|
852
|
+
}
|
|
853
|
+
async findWithEmbeddingsByEntityIds(entityIds, tx) {
|
|
854
|
+
if (entityIds.length === 0) return [];
|
|
855
|
+
const executor = this.getExecutor(tx);
|
|
856
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
857
|
+
return executor.getAllAsync(
|
|
858
|
+
`SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
859
|
+
[...entityIds]
|
|
860
|
+
);
|
|
861
|
+
}
|
|
862
|
+
async findEmbeddingsByIds(ids, tx) {
|
|
863
|
+
if (ids.length === 0) return [];
|
|
864
|
+
const executor = this.getExecutor(tx);
|
|
865
|
+
const rows = [];
|
|
866
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
867
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
868
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
869
|
+
const chunkRows = await executor.getAllAsync(
|
|
870
|
+
`SELECT id, embedding_blob, embedding FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
871
|
+
chunk
|
|
872
|
+
);
|
|
873
|
+
rows.push(...chunkRows);
|
|
874
|
+
}
|
|
875
|
+
return rows;
|
|
876
|
+
}
|
|
877
|
+
async trackAccess(ids, now, tx) {
|
|
878
|
+
if (ids.length === 0) return;
|
|
879
|
+
const executor = this.getExecutor(tx);
|
|
880
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
881
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
882
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
883
|
+
await executor.runAsync(
|
|
884
|
+
`UPDATE ${this.prefix}entries SET access_count = access_count + 1, last_accessed_at = ? WHERE id IN (${placeholders})`,
|
|
885
|
+
[now, ...chunk]
|
|
886
|
+
);
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
getLegacyMigrationSQL() {
|
|
890
|
+
return [
|
|
891
|
+
`-- Migrate legacy source_type values (targets your WikiMemory prefix: ${this.prefix})`,
|
|
892
|
+
`UPDATE ${this.prefix}entries SET source_type = 'immutable_document' WHERE source_type = 'user_document';`,
|
|
893
|
+
`UPDATE ${this.prefix}entries SET source_type = 'librarian_inferred' WHERE source_type = 'agent_inferred';`
|
|
894
|
+
].join("\n");
|
|
895
|
+
}
|
|
896
|
+
async findRecentByEntityIds(entityIds, limit, tx) {
|
|
897
|
+
if (entityIds.length === 0) return [];
|
|
898
|
+
const executor = this.getExecutor(tx);
|
|
899
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
900
|
+
const rows = await executor.getAllAsync(
|
|
901
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL ORDER BY updated_at DESC LIMIT ?`,
|
|
902
|
+
[...entityIds, limit]
|
|
903
|
+
);
|
|
904
|
+
return rows.map(mapRowToFact);
|
|
905
|
+
}
|
|
906
|
+
};
|
|
907
|
+
|
|
908
|
+
// src/utils/ids.ts
|
|
909
|
+
function generateId(prefix = "") {
|
|
910
|
+
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
|
|
911
|
+
return prefix + crypto.randomUUID().replace(/-/g, "").substring(0, 24);
|
|
912
|
+
}
|
|
913
|
+
if (typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function") {
|
|
914
|
+
const bytes = new Uint8Array(16);
|
|
915
|
+
crypto.getRandomValues(bytes);
|
|
916
|
+
return prefix + Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("").substring(0, 24);
|
|
917
|
+
}
|
|
918
|
+
return prefix + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
// src/repositories/OutboxRepository.ts
|
|
922
|
+
var OutboxRepository = class extends BaseRepository {
|
|
923
|
+
/**
|
|
924
|
+
* Insert a new outbox event within the provided transaction.
|
|
925
|
+
* `tx` is required — callers must always pass the active transaction
|
|
926
|
+
* so the write is atomic with the main table mutation.
|
|
927
|
+
*/
|
|
928
|
+
async push(params, tx) {
|
|
929
|
+
const executor = this.getExecutor(tx);
|
|
930
|
+
const id = generateId("out_");
|
|
931
|
+
const now = Date.now();
|
|
932
|
+
await executor.runAsync(
|
|
933
|
+
`INSERT INTO ${this.prefix}outbox (id, entity_id, table_name, record_id, operation, payload, created_at)
|
|
934
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
935
|
+
[id, params.entityId, params.tableName, params.recordId, params.operation, JSON.stringify(params.payload), now]
|
|
936
|
+
);
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Fetch pending outbox rows ordered by created_at ASC.
|
|
940
|
+
* Reads directly from `this.db` (not a transaction).
|
|
941
|
+
*/
|
|
942
|
+
async fetchPending(limit = 50) {
|
|
943
|
+
return this.db.getAllAsync(
|
|
944
|
+
`SELECT * FROM ${this.prefix}outbox ORDER BY created_at ASC LIMIT ?`,
|
|
945
|
+
[limit]
|
|
946
|
+
);
|
|
947
|
+
}
|
|
948
|
+
/**
|
|
949
|
+
* Delete acknowledged outbox rows by their IDs.
|
|
950
|
+
* No-op when `ids` is empty.
|
|
951
|
+
* Deletes directly from `this.db` (not a transaction).
|
|
952
|
+
*/
|
|
953
|
+
async acknowledge(ids) {
|
|
954
|
+
if (ids.length === 0) return;
|
|
955
|
+
const placeholders = ids.map(() => "?").join(", ");
|
|
956
|
+
await this.db.runAsync(
|
|
957
|
+
`DELETE FROM ${this.prefix}outbox WHERE id IN (${placeholders})`,
|
|
958
|
+
ids
|
|
959
|
+
);
|
|
960
|
+
}
|
|
961
|
+
};
|
|
962
|
+
|
|
963
|
+
// src/repositories/TaskRepository.ts
|
|
964
|
+
function mapRowToTask(row) {
|
|
965
|
+
return {
|
|
966
|
+
id: row.id,
|
|
967
|
+
entity_id: row.entity_id,
|
|
968
|
+
description: row.description,
|
|
969
|
+
status: row.status,
|
|
970
|
+
priority: Number(row.priority),
|
|
971
|
+
created_at: Number(row.created_at),
|
|
972
|
+
updated_at: Number(row.updated_at),
|
|
973
|
+
resolved_at: row.resolved_at != null ? Number(row.resolved_at) : null,
|
|
974
|
+
deleted_at: row.deleted_at != null ? Number(row.deleted_at) : null
|
|
975
|
+
};
|
|
976
|
+
}
|
|
977
|
+
var TaskRepository = class extends BaseRepository {
|
|
978
|
+
constructor(db, prefix, outbox) {
|
|
979
|
+
super(db, prefix);
|
|
980
|
+
this.outbox = outbox;
|
|
981
|
+
}
|
|
982
|
+
/**
|
|
983
|
+
* Fetch a single task by ID. Returns null if not found or soft-deleted.
|
|
984
|
+
*/
|
|
985
|
+
async findById(id) {
|
|
986
|
+
const row = await this.db.getFirstAsync(
|
|
987
|
+
`SELECT * FROM ${this.prefix}tasks WHERE id = ? AND deleted_at IS NULL`,
|
|
988
|
+
[id]
|
|
989
|
+
);
|
|
990
|
+
return row ? mapRowToTask(row) : null;
|
|
991
|
+
}
|
|
992
|
+
/**
|
|
993
|
+
* Fetch all pending/in_progress tasks for the given entity IDs.
|
|
994
|
+
* Returns empty array when entityIds is empty.
|
|
995
|
+
*/
|
|
996
|
+
async findAllPending(entityIds, limit) {
|
|
997
|
+
if (entityIds.length === 0) return [];
|
|
998
|
+
const placeholders = entityIds.map(() => "?").join(", ");
|
|
999
|
+
const sql = `SELECT * FROM ${this.prefix}tasks WHERE entity_id IN (${placeholders}) AND status IN ('pending', 'in_progress') AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC` + (limit != null ? ` LIMIT ?` : "");
|
|
1000
|
+
const params = limit != null ? [...entityIds, limit] : [...entityIds];
|
|
1001
|
+
const rows = await this.db.getAllAsync(sql, params);
|
|
1002
|
+
return rows.map(mapRowToTask);
|
|
1003
|
+
}
|
|
1004
|
+
async findExistingMetadataByIds(ids, tx) {
|
|
1005
|
+
const executor = this.getExecutor(tx);
|
|
1006
|
+
const rows = [];
|
|
1007
|
+
const chunkSize = 500;
|
|
1008
|
+
for (let i = 0; i < ids.length; i += chunkSize) {
|
|
1009
|
+
const chunk = ids.slice(i, i + chunkSize);
|
|
1010
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1011
|
+
const chunkRows = await executor.getAllAsync(
|
|
1012
|
+
`SELECT id, entity_id, updated_at FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
|
|
1013
|
+
chunk
|
|
1014
|
+
);
|
|
1015
|
+
rows.push(...chunkRows.map((row) => ({ id: row.id, entity_id: row.entity_id, updated_at: Number(row.updated_at) })));
|
|
1016
|
+
}
|
|
1017
|
+
return rows;
|
|
1018
|
+
}
|
|
1019
|
+
/**
|
|
1020
|
+
* Upsert a WikiTask within the provided transaction.
|
|
1021
|
+
* Uses ON CONFLICT(id) DO UPDATE (not INSERT OR REPLACE).
|
|
1022
|
+
* Stages an outbox entry in the same transaction.
|
|
1023
|
+
* `tx` is REQUIRED.
|
|
1024
|
+
*/
|
|
1025
|
+
async upsert(task, tx, updatedAt) {
|
|
1026
|
+
const executor = this.getExecutor(tx);
|
|
1027
|
+
const now = Number.isFinite(updatedAt) ? updatedAt : Date.now();
|
|
1028
|
+
const existingRow = await executor.getFirstAsync(
|
|
1029
|
+
`SELECT id FROM ${this.prefix}tasks WHERE id = ?`,
|
|
1030
|
+
[task.id]
|
|
1031
|
+
);
|
|
1032
|
+
const operation = task.deleted_at != null ? "DELETE" : existingRow ? "UPDATE" : "INSERT";
|
|
1033
|
+
await executor.runAsync(
|
|
1034
|
+
`INSERT INTO ${this.prefix}tasks (
|
|
1035
|
+
id, entity_id, description, status, priority,
|
|
1036
|
+
created_at, updated_at, resolved_at, deleted_at
|
|
1037
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1038
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1039
|
+
entity_id = excluded.entity_id,
|
|
1040
|
+
description = excluded.description,
|
|
1041
|
+
status = excluded.status,
|
|
1042
|
+
priority = excluded.priority,
|
|
1043
|
+
updated_at = excluded.updated_at,
|
|
1044
|
+
resolved_at = excluded.resolved_at,
|
|
1045
|
+
deleted_at = excluded.deleted_at`,
|
|
1046
|
+
[
|
|
1047
|
+
task.id,
|
|
1048
|
+
task.entity_id,
|
|
1049
|
+
task.description,
|
|
1050
|
+
task.status,
|
|
1051
|
+
task.priority,
|
|
1052
|
+
task.created_at,
|
|
1053
|
+
now,
|
|
1054
|
+
// updated_at set by repo or import override
|
|
1055
|
+
task.resolved_at ?? null,
|
|
1056
|
+
task.deleted_at ?? null
|
|
1057
|
+
]
|
|
1058
|
+
);
|
|
1059
|
+
await this.outbox.push(
|
|
1060
|
+
{
|
|
1061
|
+
entityId: task.entity_id,
|
|
1062
|
+
tableName: "tasks",
|
|
1063
|
+
recordId: task.id,
|
|
1064
|
+
operation,
|
|
1065
|
+
payload: task
|
|
1066
|
+
},
|
|
1067
|
+
tx
|
|
1068
|
+
);
|
|
1069
|
+
}
|
|
1070
|
+
async upsertForImport(task, tx, updatedAt) {
|
|
1071
|
+
const executor = this.getExecutor(tx);
|
|
1072
|
+
const now = Number.isFinite(updatedAt) ? updatedAt : Date.now();
|
|
1073
|
+
await executor.runAsync(
|
|
1074
|
+
`INSERT INTO ${this.prefix}tasks (
|
|
1075
|
+
id, entity_id, description, status, priority,
|
|
1076
|
+
created_at, updated_at, resolved_at, deleted_at
|
|
1077
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1078
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1079
|
+
entity_id = excluded.entity_id,
|
|
1080
|
+
description = excluded.description,
|
|
1081
|
+
status = excluded.status,
|
|
1082
|
+
priority = excluded.priority,
|
|
1083
|
+
updated_at = excluded.updated_at,
|
|
1084
|
+
resolved_at = excluded.resolved_at,
|
|
1085
|
+
deleted_at = excluded.deleted_at`,
|
|
1086
|
+
[
|
|
1087
|
+
task.id,
|
|
1088
|
+
task.entity_id,
|
|
1089
|
+
task.description,
|
|
1090
|
+
task.status,
|
|
1091
|
+
task.priority,
|
|
1092
|
+
task.created_at,
|
|
1093
|
+
now,
|
|
1094
|
+
task.resolved_at ?? null,
|
|
1095
|
+
task.deleted_at ?? null
|
|
1096
|
+
]
|
|
1097
|
+
);
|
|
1098
|
+
}
|
|
1099
|
+
/**
|
|
1100
|
+
* Soft-delete a task by ID. Sets deleted_at and updated_at.
|
|
1101
|
+
* Stages a DELETE outbox entry in the same transaction.
|
|
1102
|
+
* `tx` is REQUIRED.
|
|
1103
|
+
*/
|
|
1104
|
+
async softDelete(id, entityId, tx) {
|
|
1105
|
+
const executor = this.getExecutor(tx);
|
|
1106
|
+
const now = Date.now();
|
|
1107
|
+
await executor.runAsync(
|
|
1108
|
+
`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
1109
|
+
[now, now, id, entityId]
|
|
1110
|
+
);
|
|
1111
|
+
await this.outbox.push(
|
|
1112
|
+
{
|
|
1113
|
+
entityId,
|
|
1114
|
+
tableName: "tasks",
|
|
1115
|
+
recordId: id,
|
|
1116
|
+
operation: "DELETE",
|
|
1117
|
+
payload: { id, entity_id: entityId, deleted_at: now }
|
|
1118
|
+
},
|
|
1119
|
+
tx
|
|
1120
|
+
);
|
|
1121
|
+
}
|
|
1122
|
+
/**
|
|
1123
|
+
* Fetch all non-deleted tasks for an entity, ordered by priority DESC, created_at ASC.
|
|
1124
|
+
* Used by _getFullBundle().
|
|
1125
|
+
*/
|
|
1126
|
+
async findAllByEntityId(entityId, tx) {
|
|
1127
|
+
const executor = this.getExecutor(tx);
|
|
1128
|
+
const rows = await executor.getAllAsync(
|
|
1129
|
+
`SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC`,
|
|
1130
|
+
[entityId]
|
|
1131
|
+
);
|
|
1132
|
+
return rows.map(mapRowToTask);
|
|
1133
|
+
}
|
|
1134
|
+
/**
|
|
1135
|
+
* Bulk delete pruned tasks (already soft-deleted) by cutoff date.
|
|
1136
|
+
* Used by runPrune(). Returns number of deleted rows.
|
|
1137
|
+
*/
|
|
1138
|
+
async bulkDeletePruned(entityId, cutoff, tx) {
|
|
1139
|
+
const executor = this.getExecutor(tx);
|
|
1140
|
+
const rowsToDelete = await executor.getAllAsync(
|
|
1141
|
+
`SELECT id, deleted_at FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
|
|
1142
|
+
[entityId, cutoff]
|
|
1143
|
+
);
|
|
1144
|
+
if (rowsToDelete.length === 0) return 0;
|
|
1145
|
+
const result = await executor.runAsync(
|
|
1146
|
+
`DELETE FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
|
|
1147
|
+
[entityId, cutoff]
|
|
1148
|
+
);
|
|
1149
|
+
for (const row of rowsToDelete) {
|
|
1150
|
+
await this.outbox.push(
|
|
1151
|
+
{
|
|
1152
|
+
entityId,
|
|
1153
|
+
tableName: "tasks",
|
|
1154
|
+
recordId: row.id,
|
|
1155
|
+
operation: "DELETE",
|
|
1156
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: row.deleted_at }
|
|
1157
|
+
},
|
|
1158
|
+
tx
|
|
1159
|
+
);
|
|
1160
|
+
}
|
|
1161
|
+
return result.changes;
|
|
1162
|
+
}
|
|
1163
|
+
/**
|
|
1164
|
+
* Soft-delete a task by ID within a transaction.
|
|
1165
|
+
* Stages a DELETE outbox entry in the same transaction.
|
|
1166
|
+
* `tx` is REQUIRED.
|
|
1167
|
+
*/
|
|
1168
|
+
async softDeleteById(id, entityId, tx) {
|
|
1169
|
+
const executor = this.getExecutor(tx);
|
|
1170
|
+
const now = Date.now();
|
|
1171
|
+
const result = await executor.runAsync(
|
|
1172
|
+
`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
1173
|
+
[now, now, id, entityId]
|
|
1174
|
+
);
|
|
1175
|
+
if (result.changes > 0) {
|
|
1176
|
+
await this.outbox.push(
|
|
1177
|
+
{
|
|
1178
|
+
entityId,
|
|
1179
|
+
tableName: "tasks",
|
|
1180
|
+
recordId: id,
|
|
1181
|
+
operation: "DELETE",
|
|
1182
|
+
payload: { id, entity_id: entityId, deleted_at: now }
|
|
1183
|
+
},
|
|
1184
|
+
tx
|
|
1185
|
+
);
|
|
1186
|
+
}
|
|
1187
|
+
return result;
|
|
1188
|
+
}
|
|
1189
|
+
/**
|
|
1190
|
+
* Bulk soft-delete all tasks for an entity.
|
|
1191
|
+
* Stages DELETE outbox entries for each row in the same transaction.
|
|
1192
|
+
* `tx` is REQUIRED.
|
|
1193
|
+
*/
|
|
1194
|
+
async bulkSoftDeleteByEntityId(entityId, tx) {
|
|
1195
|
+
const executor = this.getExecutor(tx);
|
|
1196
|
+
const now = Date.now();
|
|
1197
|
+
const idsToDelete = await executor.getAllAsync(
|
|
1198
|
+
`SELECT id FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
1199
|
+
[entityId]
|
|
1200
|
+
);
|
|
1201
|
+
const result = await executor.runAsync(
|
|
1202
|
+
`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
1203
|
+
[now, now, entityId]
|
|
1204
|
+
);
|
|
1205
|
+
for (const row of idsToDelete) {
|
|
1206
|
+
await this.outbox.push({
|
|
1207
|
+
entityId,
|
|
1208
|
+
tableName: "tasks",
|
|
1209
|
+
recordId: row.id,
|
|
1210
|
+
operation: "DELETE",
|
|
1211
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: now }
|
|
1212
|
+
}, tx);
|
|
1213
|
+
}
|
|
1214
|
+
return result.changes;
|
|
210
1215
|
}
|
|
1216
|
+
};
|
|
1217
|
+
|
|
1218
|
+
// src/repositories/EventRepository.ts
|
|
1219
|
+
var EventRepository = class extends BaseRepository {
|
|
211
1220
|
/**
|
|
212
|
-
*
|
|
213
|
-
*
|
|
1221
|
+
* Insert a new event row.
|
|
1222
|
+
* Pass `tx` to participate in a caller-owned transaction; omit to run against the default db.
|
|
214
1223
|
*/
|
|
215
|
-
async
|
|
1224
|
+
async add(event, tx) {
|
|
216
1225
|
const executor = this.getExecutor(tx);
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
const obj = fact.embedding_blob;
|
|
221
|
-
const keys = Object.keys(obj).map(Number).sort((a, b) => a - b);
|
|
222
|
-
const arr = new Uint8Array(keys.length);
|
|
223
|
-
for (let i = 0; i < keys.length; i++) arr[i] = obj[String(keys[i])];
|
|
224
|
-
return arr;
|
|
225
|
-
})() : void 0;
|
|
226
|
-
return executor.runAsync(
|
|
227
|
-
`INSERT INTO ${this.prefix}entries (
|
|
228
|
-
id, entity_id, title, body, tags, confidence, source_type,
|
|
229
|
-
source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count,
|
|
230
|
-
deleted_at, embedding_blob, embedding
|
|
231
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
232
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
233
|
-
entity_id = excluded.entity_id,
|
|
234
|
-
title = excluded.title,
|
|
235
|
-
body = excluded.body,
|
|
236
|
-
tags = excluded.tags,
|
|
237
|
-
confidence = excluded.confidence,
|
|
238
|
-
source_type = excluded.source_type,
|
|
239
|
-
source_hash = excluded.source_hash,
|
|
240
|
-
source_ref = excluded.source_ref,
|
|
241
|
-
updated_at = excluded.updated_at,
|
|
242
|
-
last_accessed_at = excluded.last_accessed_at,
|
|
243
|
-
access_count = excluded.access_count,
|
|
244
|
-
deleted_at = excluded.deleted_at,
|
|
245
|
-
embedding_blob = CASE WHEN excluded.embedding_blob IS NULL THEN embedding_blob ELSE excluded.embedding_blob END,
|
|
246
|
-
embedding = NULL`,
|
|
1226
|
+
await executor.runAsync(
|
|
1227
|
+
`INSERT INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
|
|
1228
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
247
1229
|
[
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
1230
|
+
event.id,
|
|
1231
|
+
event.entity_id,
|
|
1232
|
+
event.event_type,
|
|
1233
|
+
event.summary,
|
|
1234
|
+
event.related_entry_id ?? null,
|
|
1235
|
+
event.created_at
|
|
1236
|
+
]
|
|
1237
|
+
);
|
|
1238
|
+
}
|
|
1239
|
+
async addIgnoreDuplicate(event, tx) {
|
|
1240
|
+
const executor = this.getExecutor(tx);
|
|
1241
|
+
await executor.runAsync(
|
|
1242
|
+
`INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
|
|
1243
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
1244
|
+
[
|
|
1245
|
+
event.id,
|
|
1246
|
+
event.entity_id,
|
|
1247
|
+
event.event_type,
|
|
1248
|
+
event.summary,
|
|
1249
|
+
event.related_entry_id ?? null,
|
|
1250
|
+
event.created_at
|
|
265
1251
|
]
|
|
266
1252
|
);
|
|
267
1253
|
}
|
|
268
1254
|
/**
|
|
269
|
-
*
|
|
1255
|
+
* Return the most recent events for an entity, newest first.
|
|
1256
|
+
* Defaults to a limit of 50.
|
|
270
1257
|
*/
|
|
271
|
-
async
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
276
|
-
[now, now, entryId, entityId]
|
|
1258
|
+
async getRecent(entityId, limit = 50) {
|
|
1259
|
+
return this.db.getAllAsync(
|
|
1260
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?`,
|
|
1261
|
+
[entityId, limit]
|
|
277
1262
|
);
|
|
278
1263
|
}
|
|
279
1264
|
/**
|
|
280
|
-
*
|
|
281
|
-
*
|
|
1265
|
+
* Return the most recent events for the given entity IDs, newest first.
|
|
1266
|
+
* Defaults to a limit of 50.
|
|
282
1267
|
*/
|
|
283
|
-
async
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
1268
|
+
async getRecentForEntities(entityIds, limit = 50) {
|
|
1269
|
+
if (entityIds.length === 0) return [];
|
|
1270
|
+
const placeholders = entityIds.map(() => "?").join(", ");
|
|
1271
|
+
return this.db.getAllAsync(
|
|
1272
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id IN (${placeholders}) ORDER BY created_at DESC LIMIT ?`,
|
|
1273
|
+
[...entityIds, limit]
|
|
1274
|
+
);
|
|
1275
|
+
}
|
|
1276
|
+
/**
|
|
1277
|
+
* Delete events for an entity that were created at or before the given cutoff timestamp.
|
|
1278
|
+
* Returns the number of deleted rows.
|
|
1279
|
+
*/
|
|
1280
|
+
async prune(entityId, cutoff) {
|
|
1281
|
+
return this.db.runAsync(
|
|
1282
|
+
`DELETE FROM ${this.prefix}events WHERE entity_id = ? AND created_at <= ?`,
|
|
288
1283
|
[entityId, cutoff]
|
|
289
1284
|
);
|
|
290
1285
|
}
|
|
1286
|
+
/**
|
|
1287
|
+
* Return the total number of events stored for an entity.
|
|
1288
|
+
* `tx` is optional — pass an active transaction handle for atomic reads.
|
|
1289
|
+
*/
|
|
1290
|
+
async count(entityId, tx) {
|
|
1291
|
+
const executor = tx ?? this.db;
|
|
1292
|
+
const row = await executor.getFirstAsync(
|
|
1293
|
+
`SELECT COUNT(*) as count FROM ${this.prefix}events WHERE entity_id = ?`,
|
|
1294
|
+
[entityId]
|
|
1295
|
+
);
|
|
1296
|
+
return row?.count ?? 0;
|
|
1297
|
+
}
|
|
1298
|
+
/**
|
|
1299
|
+
* Return all events for an entity in chronological (ASC) order.
|
|
1300
|
+
* When limit is provided, fetches newest-first then reverses to preserve chronological order.
|
|
1301
|
+
*/
|
|
1302
|
+
async getByEntityId(entityId, limit) {
|
|
1303
|
+
if (limit != null) {
|
|
1304
|
+
const rows = await this.db.getAllAsync(
|
|
1305
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?`,
|
|
1306
|
+
[entityId, limit]
|
|
1307
|
+
);
|
|
1308
|
+
return rows.slice().reverse();
|
|
1309
|
+
}
|
|
1310
|
+
return this.db.getAllAsync(
|
|
1311
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC`,
|
|
1312
|
+
[entityId]
|
|
1313
|
+
);
|
|
1314
|
+
}
|
|
1315
|
+
};
|
|
1316
|
+
|
|
1317
|
+
// src/repositories/MetadataRepository.ts
|
|
1318
|
+
var MetadataRepository = class extends BaseRepository {
|
|
1319
|
+
// CHECKPOINTS TABLE METHODS
|
|
1320
|
+
async getCheckpoint(entityId, tx) {
|
|
1321
|
+
const executor = this.getExecutor(tx);
|
|
1322
|
+
const row = await executor.getFirstAsync(
|
|
1323
|
+
`SELECT memory_checkpoint, heal_checkpoint FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
1324
|
+
[entityId]
|
|
1325
|
+
);
|
|
1326
|
+
if (!row) return {};
|
|
1327
|
+
return {
|
|
1328
|
+
memory: row.memory_checkpoint ?? void 0,
|
|
1329
|
+
heal: row.heal_checkpoint ?? void 0
|
|
1330
|
+
};
|
|
1331
|
+
}
|
|
1332
|
+
async updateCheckpoint(entityId, updates, tx) {
|
|
1333
|
+
const fields = [];
|
|
1334
|
+
const values = [];
|
|
1335
|
+
if (updates.memory !== void 0) {
|
|
1336
|
+
fields.push("memory_checkpoint = ?");
|
|
1337
|
+
values.push(updates.memory);
|
|
1338
|
+
}
|
|
1339
|
+
if (updates.heal !== void 0) {
|
|
1340
|
+
fields.push("heal_checkpoint = ?");
|
|
1341
|
+
values.push(updates.heal);
|
|
1342
|
+
}
|
|
1343
|
+
if (fields.length === 0) return;
|
|
1344
|
+
const executor = this.getExecutor(tx);
|
|
1345
|
+
await executor.runAsync(
|
|
1346
|
+
`INSERT INTO ${this.prefix}checkpoints (entity_id, memory_checkpoint, heal_checkpoint)
|
|
1347
|
+
VALUES (?, ?, ?)
|
|
1348
|
+
ON CONFLICT(entity_id) DO UPDATE SET ${fields.join(", ")}`,
|
|
1349
|
+
[entityId, updates.memory ?? 0, updates.heal ?? 0, ...values]
|
|
1350
|
+
);
|
|
1351
|
+
}
|
|
1352
|
+
async deleteCheckpoint(entityId, tx) {
|
|
1353
|
+
const executor = this.getExecutor(tx);
|
|
1354
|
+
await executor.runAsync(
|
|
1355
|
+
`DELETE FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
1356
|
+
[entityId]
|
|
1357
|
+
);
|
|
1358
|
+
}
|
|
1359
|
+
// META TABLE METHODS
|
|
1360
|
+
async getMeta(key, tx) {
|
|
1361
|
+
const executor = this.getExecutor(tx);
|
|
1362
|
+
const row = await executor.getFirstAsync(
|
|
1363
|
+
`SELECT value FROM ${this.prefix}meta WHERE key = ?`,
|
|
1364
|
+
[key]
|
|
1365
|
+
);
|
|
1366
|
+
return row ? row.value : null;
|
|
1367
|
+
}
|
|
1368
|
+
async setMeta(key, value, tx) {
|
|
1369
|
+
const executor = this.getExecutor(tx);
|
|
1370
|
+
await executor.runAsync(
|
|
1371
|
+
`INSERT INTO ${this.prefix}meta (key, value) VALUES (?, ?)
|
|
1372
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
|
1373
|
+
[key, value]
|
|
1374
|
+
);
|
|
1375
|
+
}
|
|
1376
|
+
async clearDimensionMismatch(tx) {
|
|
1377
|
+
const executor = this.getExecutor(tx);
|
|
1378
|
+
await executor.runAsync(
|
|
1379
|
+
`DELETE FROM ${this.prefix}meta WHERE key = 'embedding_dimension_mismatch'`
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
async tableExists(tableName, tx) {
|
|
1383
|
+
const executor = this.getExecutor(tx);
|
|
1384
|
+
const row = await executor.getFirstAsync(
|
|
1385
|
+
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
|
|
1386
|
+
[tableName]
|
|
1387
|
+
);
|
|
1388
|
+
return row != null;
|
|
1389
|
+
}
|
|
1390
|
+
async getTableDdl(tableName, tx) {
|
|
1391
|
+
const executor = this.getExecutor(tx);
|
|
1392
|
+
const row = await executor.getFirstAsync(
|
|
1393
|
+
`SELECT sql FROM sqlite_master WHERE type='table' AND name=?`,
|
|
1394
|
+
[tableName]
|
|
1395
|
+
);
|
|
1396
|
+
return row?.sql ?? null;
|
|
1397
|
+
}
|
|
1398
|
+
async vacuum() {
|
|
1399
|
+
await this.db.execAsync(`PRAGMA wal_checkpoint(TRUNCATE)`);
|
|
1400
|
+
await this.db.execAsync(`VACUUM`);
|
|
1401
|
+
}
|
|
1402
|
+
async getDistinctEntityIds(tx) {
|
|
1403
|
+
const executor = this.getExecutor(tx);
|
|
1404
|
+
const rows = await executor.getAllAsync(
|
|
1405
|
+
`SELECT DISTINCT entity_id FROM (
|
|
1406
|
+
SELECT entity_id FROM ${this.prefix}entries WHERE deleted_at IS NULL
|
|
1407
|
+
UNION
|
|
1408
|
+
SELECT entity_id FROM ${this.prefix}tasks WHERE deleted_at IS NULL
|
|
1409
|
+
UNION
|
|
1410
|
+
SELECT entity_id FROM ${this.prefix}events
|
|
1411
|
+
) ORDER BY entity_id`
|
|
1412
|
+
);
|
|
1413
|
+
return rows.map((r) => r.entity_id);
|
|
1414
|
+
}
|
|
291
1415
|
};
|
|
292
1416
|
|
|
293
1417
|
// src/prompts.ts
|
|
@@ -441,9 +1565,6 @@ function parseJsonResponse(text) {
|
|
|
441
1565
|
if (end === -1) throw new SyntaxError("No JSON object/array found in LLM response");
|
|
442
1566
|
return JSON.parse(text.slice(start, end + 1));
|
|
443
1567
|
}
|
|
444
|
-
function generateId(prefix = "") {
|
|
445
|
-
return prefix + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
|
|
446
|
-
}
|
|
447
1568
|
function safeSlice(value, start, end) {
|
|
448
1569
|
const length = value.length;
|
|
449
1570
|
let safeStart = start < 0 ? Math.max(length + start, 0) : Math.min(start, length);
|
|
@@ -612,7 +1733,11 @@ var _WikiMemory = class _WikiMemory {
|
|
|
612
1733
|
this.db = db;
|
|
613
1734
|
this.options = options;
|
|
614
1735
|
this.prefix = options.config?.tablePrefix || "llm_wiki_";
|
|
615
|
-
this.
|
|
1736
|
+
this.outboxRepo = new OutboxRepository(db, this.prefix);
|
|
1737
|
+
this.entryRepo = new EntryRepository(db, this.prefix, this.outboxRepo);
|
|
1738
|
+
this.taskRepo = new TaskRepository(db, this.prefix, this.outboxRepo);
|
|
1739
|
+
this.eventRepo = new EventRepository(db, this.prefix);
|
|
1740
|
+
this.metadataRepo = new MetadataRepository(db, this.prefix);
|
|
616
1741
|
}
|
|
617
1742
|
normalizeMiniSearchRow(row) {
|
|
618
1743
|
return {
|
|
@@ -632,10 +1757,7 @@ var _WikiMemory = class _WikiMemory {
|
|
|
632
1757
|
}
|
|
633
1758
|
async rebuildMiniSearchIndex(entityId) {
|
|
634
1759
|
if (entityId) {
|
|
635
|
-
const rows2 = await this.
|
|
636
|
-
`SELECT id, entity_id, title, body, tags FROM ${this.prefix}entries WHERE deleted_at IS NULL AND entity_id = ?`,
|
|
637
|
-
[entityId]
|
|
638
|
-
);
|
|
1760
|
+
const rows2 = await this.entryRepo.findMiniSearchRows(entityId);
|
|
639
1761
|
const previousIds = this.miniSearchEntryIdsByEntity.get(entityId);
|
|
640
1762
|
if (previousIds) {
|
|
641
1763
|
for (const id of previousIds) {
|
|
@@ -649,7 +1771,7 @@ var _WikiMemory = class _WikiMemory {
|
|
|
649
1771
|
this.miniSearchEntryIdsByEntity.set(entityId, new Set(documents2.map((document) => document.id)));
|
|
650
1772
|
return;
|
|
651
1773
|
}
|
|
652
|
-
const rows = await this.
|
|
1774
|
+
const rows = await this.entryRepo.findMiniSearchRows();
|
|
653
1775
|
this.miniSearch.removeAll();
|
|
654
1776
|
this.miniSearchEntryIdsByEntity.clear();
|
|
655
1777
|
const documents = rows.map((row) => this.normalizeMiniSearchRow(row));
|
|
@@ -663,25 +1785,17 @@ var _WikiMemory = class _WikiMemory {
|
|
|
663
1785
|
}
|
|
664
1786
|
}
|
|
665
1787
|
async storeEmbeddingDimension(dim) {
|
|
666
|
-
const existing = await this.
|
|
667
|
-
`SELECT value FROM ${this.prefix}meta WHERE key = 'embedding_dimension'`
|
|
668
|
-
);
|
|
1788
|
+
const existing = await this.metadataRepo.getMeta("embedding_dimension");
|
|
669
1789
|
if (existing) {
|
|
670
|
-
const storedDim = parseInt(existing
|
|
1790
|
+
const storedDim = parseInt(existing, 10);
|
|
671
1791
|
if (storedDim !== dim) {
|
|
672
1792
|
console.warn(
|
|
673
1793
|
`[WikiMemory] Embedding dimension mismatch: stored ${storedDim}, got ${dim}. Call runReembed() to rebuild embeddings with the new model.`
|
|
674
1794
|
);
|
|
675
|
-
await this.
|
|
676
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
677
|
-
[String(dim)]
|
|
678
|
-
);
|
|
1795
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(dim), this.db);
|
|
679
1796
|
}
|
|
680
1797
|
} else {
|
|
681
|
-
await this.
|
|
682
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension', ?)`,
|
|
683
|
-
[String(dim)]
|
|
684
|
-
);
|
|
1798
|
+
await this.metadataRepo.setMeta("embedding_dimension", String(dim), this.db);
|
|
685
1799
|
}
|
|
686
1800
|
}
|
|
687
1801
|
/**
|
|
@@ -691,28 +1805,13 @@ var _WikiMemory = class _WikiMemory {
|
|
|
691
1805
|
* stuck on the MiniSearch fallback.
|
|
692
1806
|
*/
|
|
693
1807
|
async _reconcileEmbeddingDimension() {
|
|
694
|
-
const
|
|
695
|
-
|
|
696
|
-
);
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
WHERE deleted_at IS NULL
|
|
702
|
-
AND (
|
|
703
|
-
(embedding_blob IS NOT NULL AND (CAST(length(embedding_blob) AS INTEGER) / 4) != ?)
|
|
704
|
-
OR (embedding_blob IS NULL AND embedding IS NOT NULL)
|
|
705
|
-
)`,
|
|
706
|
-
[newDim]
|
|
707
|
-
);
|
|
708
|
-
if (!residual || residual.cnt === 0) {
|
|
709
|
-
await this.db.runAsync(
|
|
710
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension', ?)`,
|
|
711
|
-
[mismatch.value]
|
|
712
|
-
);
|
|
713
|
-
await this.db.runAsync(
|
|
714
|
-
`DELETE FROM ${this.prefix}meta WHERE key = 'embedding_dimension_mismatch'`
|
|
715
|
-
);
|
|
1808
|
+
const mismatchValue = await this.metadataRepo.getMeta("embedding_dimension_mismatch");
|
|
1809
|
+
if (!mismatchValue) return;
|
|
1810
|
+
const newDim = parseInt(mismatchValue, 10);
|
|
1811
|
+
const residualCount = await this.entryRepo.countStaleEmbeddings(newDim);
|
|
1812
|
+
if (residualCount === 0) {
|
|
1813
|
+
await this.metadataRepo.setMeta("embedding_dimension", mismatchValue, this.db);
|
|
1814
|
+
await this.metadataRepo.clearDimensionMismatch(this.db);
|
|
716
1815
|
}
|
|
717
1816
|
}
|
|
718
1817
|
async embedFact(fact) {
|
|
@@ -750,10 +1849,7 @@ var _WikiMemory = class _WikiMemory {
|
|
|
750
1849
|
}
|
|
751
1850
|
await this.storeEmbeddingDimension(float32Vector.length);
|
|
752
1851
|
const blob = new Uint8Array(float32Vector.buffer);
|
|
753
|
-
await this.
|
|
754
|
-
`UPDATE ${this.prefix}entries SET embedding_blob = ?, embedding = NULL WHERE id = ?`,
|
|
755
|
-
[blob, fact.id]
|
|
756
|
-
);
|
|
1852
|
+
await this.entryRepo.updateEmbeddingBlob(fact.id, blob);
|
|
757
1853
|
try {
|
|
758
1854
|
await this._notifyEmbeddingPersisted(fact.entity_id, fact.id, float32Vector);
|
|
759
1855
|
} catch (hookErr) {
|
|
@@ -786,28 +1882,12 @@ var _WikiMemory = class _WikiMemory {
|
|
|
786
1882
|
);
|
|
787
1883
|
}
|
|
788
1884
|
async assertNoLegacySourceTypes() {
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
WHERE source_type IN ('user_document', 'agent_inferred')
|
|
792
|
-
LIMIT 1`,
|
|
793
|
-
[]
|
|
794
|
-
);
|
|
795
|
-
if (!legacyProbe) return;
|
|
796
|
-
const legacyCount = await this.db.getFirstAsync(
|
|
797
|
-
`SELECT COUNT(*) as count FROM ${this.prefix}entries
|
|
798
|
-
WHERE source_type IN ('user_document', 'agent_inferred')`,
|
|
799
|
-
[]
|
|
800
|
-
);
|
|
801
|
-
const count = legacyCount?.count ?? 0;
|
|
802
|
-
const migrationSQL = `
|
|
803
|
-
-- Migrate legacy source_type values (targets your WikiMemory prefix: ${this.prefix})
|
|
804
|
-
UPDATE ${this.prefix}entries SET source_type = 'immutable_document' WHERE source_type = 'user_document';
|
|
805
|
-
UPDATE ${this.prefix}entries SET source_type = 'librarian_inferred' WHERE source_type = 'agent_inferred';
|
|
806
|
-
`.trim();
|
|
1885
|
+
if (!await this.entryRepo.hasLegacySourceTypes()) return;
|
|
1886
|
+
const count = await this.entryRepo.countLegacySourceTypes();
|
|
807
1887
|
throw new Error(
|
|
808
1888
|
`Database contains ${count} entries with legacy source_type values ('user_document' or 'agent_inferred'). These enum values were renamed in this release. Running without migration would allow legacy 'user_document' facts to bypass immutability guards, causing data corruption.
|
|
809
1889
|
|
|
810
|
-
${
|
|
1890
|
+
${this.entryRepo.getLegacyMigrationSQL()}
|
|
811
1891
|
|
|
812
1892
|
After running the migration SQL, restart your application.`
|
|
813
1893
|
);
|
|
@@ -864,77 +1944,45 @@ After running the migration SQL, restart your application.`
|
|
|
864
1944
|
}
|
|
865
1945
|
}
|
|
866
1946
|
async setup() {
|
|
867
|
-
const entriesExistedBeforeSetup = await this.
|
|
868
|
-
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
|
|
869
|
-
[`${this.prefix}entries`]
|
|
870
|
-
);
|
|
1947
|
+
const entriesExistedBeforeSetup = await this.metadataRepo.tableExists(`${this.prefix}entries`);
|
|
871
1948
|
await setupDatabase(this.db, this.prefix);
|
|
872
1949
|
let currentVersion;
|
|
873
1950
|
if (!entriesExistedBeforeSetup) {
|
|
874
|
-
await this.
|
|
875
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
|
|
876
|
-
[String(CURRENT_SCHEMA_VERSION)]
|
|
877
|
-
);
|
|
1951
|
+
await this.metadataRepo.setMeta("schema_version", String(CURRENT_SCHEMA_VERSION), this.db);
|
|
878
1952
|
currentVersion = CURRENT_SCHEMA_VERSION;
|
|
879
1953
|
} else {
|
|
880
|
-
const
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
if (metaRow) {
|
|
884
|
-
currentVersion = parseInt(metaRow.value, 10);
|
|
1954
|
+
const schemaVersionValue = await this.metadataRepo.getMeta("schema_version");
|
|
1955
|
+
if (schemaVersionValue) {
|
|
1956
|
+
currentVersion = parseInt(schemaVersionValue, 10);
|
|
885
1957
|
if (!Number.isFinite(currentVersion)) currentVersion = 0;
|
|
886
1958
|
} else {
|
|
887
|
-
const
|
|
888
|
-
|
|
889
|
-
[`${this.prefix}entries_fts`]
|
|
890
|
-
);
|
|
891
|
-
const hasPorter = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsMeta?.sql ?? "");
|
|
1959
|
+
const ftsDdl = await this.metadataRepo.getTableDdl(`${this.prefix}entries_fts`);
|
|
1960
|
+
const hasPorter = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl ?? "");
|
|
892
1961
|
currentVersion = hasPorter ? 1 : 0;
|
|
893
1962
|
}
|
|
894
1963
|
}
|
|
895
1964
|
for (const migration of MIGRATIONS) {
|
|
896
1965
|
if (migration.version > currentVersion) {
|
|
897
1966
|
await migration.run(this.db, this.prefix);
|
|
898
|
-
await this.
|
|
899
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
|
|
900
|
-
[String(migration.version)]
|
|
901
|
-
);
|
|
1967
|
+
await this.metadataRepo.setMeta("schema_version", String(migration.version), this.db);
|
|
902
1968
|
currentVersion = migration.version;
|
|
903
1969
|
}
|
|
904
1970
|
}
|
|
905
1971
|
if (entriesExistedBeforeSetup) {
|
|
906
|
-
const
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
if (!metaCheck) {
|
|
910
|
-
await this.db.runAsync(
|
|
911
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
|
|
912
|
-
[String(currentVersion)]
|
|
913
|
-
);
|
|
1972
|
+
const schemaVersionCheck = await this.metadataRepo.getMeta("schema_version");
|
|
1973
|
+
if (!schemaVersionCheck) {
|
|
1974
|
+
await this.metadataRepo.setMeta("schema_version", String(currentVersion), this.db);
|
|
914
1975
|
}
|
|
915
1976
|
}
|
|
916
1977
|
if (entriesExistedBeforeSetup) {
|
|
917
1978
|
await this.assertNoLegacySourceTypes();
|
|
918
1979
|
}
|
|
919
|
-
const rows = await this.
|
|
920
|
-
|
|
921
|
-
WHERE source_ref IS NOT NULL
|
|
922
|
-
AND (
|
|
923
|
-
TRIM(source_ref) != source_ref
|
|
924
|
-
OR INSTR(source_ref, '/') > 0
|
|
925
|
-
OR INSTR(source_ref, '\\') > 0
|
|
926
|
-
OR INSTR(source_ref, CHAR(0)) > 0
|
|
927
|
-
OR source_ref GLOB '*[^-A-Za-z0-9._ ]*'
|
|
928
|
-
)
|
|
929
|
-
`);
|
|
930
|
-
await this.db.withTransactionAsync(async () => {
|
|
1980
|
+
const rows = await this.entryRepo.findRowsForSourceRefMigration();
|
|
1981
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
931
1982
|
for (const row of rows) {
|
|
932
1983
|
const normalized = normalizeSourceRef(row.source_ref);
|
|
933
1984
|
if (normalized !== row.source_ref) {
|
|
934
|
-
await this.
|
|
935
|
-
`UPDATE ${this.prefix}entries SET source_ref = ? WHERE rowid = ?`,
|
|
936
|
-
[normalized, row.rowid]
|
|
937
|
-
);
|
|
1985
|
+
await this.entryRepo.updateSourceRefByRowid(row.rowid, normalized, tx);
|
|
938
1986
|
}
|
|
939
1987
|
}
|
|
940
1988
|
});
|
|
@@ -949,15 +1997,9 @@ After running the migration SQL, restart your application.`
|
|
|
949
1997
|
if (!normalizedHash) {
|
|
950
1998
|
throw new Error(`Invalid sourceHash: must be a 64-character hex string (normalized to lowercase)`);
|
|
951
1999
|
}
|
|
952
|
-
const
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
ORDER BY updated_at DESC
|
|
956
|
-
LIMIT 1`,
|
|
957
|
-
[entityId, normalizedRef]
|
|
958
|
-
);
|
|
959
|
-
if (!row) return true;
|
|
960
|
-
const normalizedStoredHash = row.source_hash ? normalizeSourceHash(row.source_hash) : null;
|
|
2000
|
+
const storedHash = await this.entryRepo.findLatestSourceHash(entityId, normalizedRef);
|
|
2001
|
+
if (storedHash === null) return true;
|
|
2002
|
+
const normalizedStoredHash = normalizeSourceHash(storedHash);
|
|
961
2003
|
return normalizedStoredHash !== normalizedHash;
|
|
962
2004
|
}
|
|
963
2005
|
_pruneKey(entityId) {
|
|
@@ -1080,24 +2122,13 @@ After running the migration SQL, restart your application.`
|
|
|
1080
2122
|
break;
|
|
1081
2123
|
}
|
|
1082
2124
|
}
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
1088
|
-
const entryResult = await this.db.runAsync(
|
|
1089
|
-
`DELETE FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ? AND id IN (${placeholders})`,
|
|
1090
|
-
[entityId, cutoff, ...chunk.map((r) => r.id)]
|
|
1091
|
-
);
|
|
1092
|
-
deletedEntries += entryResult.changes;
|
|
2125
|
+
const succeededIds = succeeded.map((r) => r.id);
|
|
2126
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2127
|
+
if (succeededIds.length > 0) {
|
|
2128
|
+
deletedEntries = await this.entryRepo.bulkDeletePruned(entityId, cutoff, succeededIds, tx);
|
|
1093
2129
|
}
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
`DELETE FROM ${this.prefix}tasks
|
|
1097
|
-
WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
|
|
1098
|
-
[entityId, cutoff]
|
|
1099
|
-
);
|
|
1100
|
-
deletedTasks = taskResult.changes;
|
|
2130
|
+
deletedTasks = await this.taskRepo.bulkDeletePruned(entityId, cutoff, tx);
|
|
2131
|
+
});
|
|
1101
2132
|
if (failure) {
|
|
1102
2133
|
await this.rebuildMiniSearchIndex(entityId);
|
|
1103
2134
|
this.vectorCache.delete(entityId);
|
|
@@ -1130,16 +2161,11 @@ After running the migration SQL, restart your application.`
|
|
|
1130
2161
|
}
|
|
1131
2162
|
if (retainEventsFor !== null) {
|
|
1132
2163
|
const cutoff = now - retainEventsFor * 864e5;
|
|
1133
|
-
const eventResult = await this.
|
|
1134
|
-
`DELETE FROM ${this.prefix}events
|
|
1135
|
-
WHERE entity_id = ? AND created_at <= ?`,
|
|
1136
|
-
[entityId, cutoff]
|
|
1137
|
-
);
|
|
2164
|
+
const eventResult = await this.eventRepo.prune(entityId, cutoff);
|
|
1138
2165
|
deletedEvents = eventResult.changes;
|
|
1139
2166
|
}
|
|
1140
2167
|
if (vacuum) {
|
|
1141
|
-
await this.
|
|
1142
|
-
await this.db.execAsync(`VACUUM`);
|
|
2168
|
+
await this.metadataRepo.vacuum();
|
|
1143
2169
|
}
|
|
1144
2170
|
await this.rebuildMiniSearchIndex(entityId);
|
|
1145
2171
|
this.vectorCache.delete(entityId);
|
|
@@ -1195,27 +2221,17 @@ After running the migration SQL, restart your application.`
|
|
|
1195
2221
|
"embed() returned an empty or non-finite vector. Falling back to keyword search."
|
|
1196
2222
|
);
|
|
1197
2223
|
}
|
|
1198
|
-
const
|
|
1199
|
-
|
|
1200
|
-
|
|
1201
|
-
if (storedDimRow) {
|
|
1202
|
-
const storedDim = parseInt(storedDimRow.value, 10);
|
|
2224
|
+
const storedDimValue = await this.metadataRepo.getMeta("embedding_dimension");
|
|
2225
|
+
if (storedDimValue) {
|
|
2226
|
+
const storedDim = parseInt(storedDimValue, 10);
|
|
1203
2227
|
if (storedDim !== queryVec.length) {
|
|
1204
2228
|
throw new Error(
|
|
1205
2229
|
`Embedding dimension mismatch: stored ${storedDim}, query has ${queryVec.length}. Call runReembed() to rebuild embeddings with the new model.`
|
|
1206
2230
|
);
|
|
1207
2231
|
}
|
|
1208
2232
|
}
|
|
1209
|
-
const
|
|
1210
|
-
|
|
1211
|
-
`SELECT COUNT(*) AS cnt FROM ${this.prefix}entries
|
|
1212
|
-
WHERE ${mismatchScope.clause} AND deleted_at IS NULL
|
|
1213
|
-
AND embedding_blob IS NOT NULL
|
|
1214
|
-
AND (CAST(length(embedding_blob) AS INTEGER) % 4 = 0)
|
|
1215
|
-
AND (CAST(length(embedding_blob) AS INTEGER) / 4) != ?`,
|
|
1216
|
-
[...mismatchScope.params, queryVec.length]
|
|
1217
|
-
);
|
|
1218
|
-
if (mismatchedCount && mismatchedCount.cnt > 0) {
|
|
2233
|
+
const mismatchedCount = await this.entryRepo.countDimensionMismatched(scoredEntityIds, queryVec.length);
|
|
2234
|
+
if (mismatchedCount > 0) {
|
|
1219
2235
|
throw new Error(
|
|
1220
2236
|
`Some facts have embeddings that do not match the current model dimension. Call runReembed() to rebuild all embeddings consistently.`
|
|
1221
2237
|
);
|
|
@@ -1239,31 +2255,10 @@ After running the migration SQL, restart your application.`
|
|
|
1239
2255
|
candidateRows = null;
|
|
1240
2256
|
} else {
|
|
1241
2257
|
const topKIds = topKResults.map((r) => r.id);
|
|
1242
|
-
const inClauseChunkSize = 500;
|
|
1243
2258
|
if (useRanker) {
|
|
1244
|
-
|
|
1245
|
-
for (let i = 0; i < topKIds.length; i += inClauseChunkSize) {
|
|
1246
|
-
const idChunk = topKIds.slice(i, i + inClauseChunkSize);
|
|
1247
|
-
const placeholders = idChunk.map(() => "?").join(",");
|
|
1248
|
-
const chunkRows = await this.db.getAllAsync(
|
|
1249
|
-
`SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
1250
|
-
idChunk
|
|
1251
|
-
);
|
|
1252
|
-
rows.push(...chunkRows);
|
|
1253
|
-
}
|
|
1254
|
-
candidateRows = rows;
|
|
2259
|
+
candidateRows = await this.entryRepo.findMetadataByIds(topKIds);
|
|
1255
2260
|
} else {
|
|
1256
|
-
|
|
1257
|
-
for (let i = 0; i < topKIds.length; i += inClauseChunkSize) {
|
|
1258
|
-
const idChunk = topKIds.slice(i, i + inClauseChunkSize);
|
|
1259
|
-
const placeholders = idChunk.map(() => "?").join(",");
|
|
1260
|
-
const chunkRows = await this.db.getAllAsync(
|
|
1261
|
-
`SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
1262
|
-
idChunk
|
|
1263
|
-
);
|
|
1264
|
-
rows.push(...chunkRows);
|
|
1265
|
-
}
|
|
1266
|
-
candidateRows = rows;
|
|
2261
|
+
candidateRows = await this.entryRepo.findWithEmbeddingsByIds(topKIds);
|
|
1267
2262
|
}
|
|
1268
2263
|
if (weight !== void 0 && weight < 1) {
|
|
1269
2264
|
const maxMsScore = Math.max(1, topKResults[0]?.score ?? 1);
|
|
@@ -1273,17 +2268,9 @@ After running the migration SQL, restart your application.`
|
|
|
1273
2268
|
}
|
|
1274
2269
|
} else {
|
|
1275
2270
|
if (useRanker) {
|
|
1276
|
-
|
|
1277
|
-
candidateRows = await this.db.getAllAsync(
|
|
1278
|
-
`SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE ${entityScope.clause} AND deleted_at IS NULL`,
|
|
1279
|
-
entityScope.params
|
|
1280
|
-
);
|
|
2271
|
+
candidateRows = await this.entryRepo.findMetadataByEntityIds(scoredEntityIds);
|
|
1281
2272
|
} else {
|
|
1282
|
-
|
|
1283
|
-
candidateRows = await this.db.getAllAsync(
|
|
1284
|
-
`SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE ${entityScope.clause} AND deleted_at IS NULL`,
|
|
1285
|
-
entityScope.params
|
|
1286
|
-
);
|
|
2273
|
+
candidateRows = await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds);
|
|
1287
2274
|
}
|
|
1288
2275
|
if (weight !== void 0 && weight < 1) {
|
|
1289
2276
|
const entityIdSet = new Set(scoredEntityIds);
|
|
@@ -1446,19 +2433,8 @@ After running the migration SQL, restart your application.`
|
|
|
1446
2433
|
let fallbackRows = candidateRows;
|
|
1447
2434
|
if (fallbackRows && fallbackRows.length > 0 && !("embedding_blob" in fallbackRows[0])) {
|
|
1448
2435
|
const rowIds = fallbackRows.map((r) => r.id);
|
|
1449
|
-
const
|
|
1450
|
-
const
|
|
1451
|
-
for (let i = 0; i < rowIds.length; i += chunkSize) {
|
|
1452
|
-
const idChunk = rowIds.slice(i, i + chunkSize);
|
|
1453
|
-
const placeholders = idChunk.map(() => "?").join(",");
|
|
1454
|
-
const embeddingRows = await this.db.getAllAsync(
|
|
1455
|
-
`SELECT id, embedding_blob, embedding FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
1456
|
-
idChunk
|
|
1457
|
-
);
|
|
1458
|
-
for (const row of embeddingRows) {
|
|
1459
|
-
embeddingsMap.set(row.id, { embedding_blob: row.embedding_blob, embedding: row.embedding });
|
|
1460
|
-
}
|
|
1461
|
-
}
|
|
2436
|
+
const embeddingRows = await this.entryRepo.findEmbeddingsByIds(rowIds);
|
|
2437
|
+
const embeddingsMap = new Map(embeddingRows.map((row) => [row.id, row]));
|
|
1462
2438
|
fallbackRows = fallbackRows.map((r) => ({
|
|
1463
2439
|
...r,
|
|
1464
2440
|
embedding_blob: embeddingsMap.get(r.id)?.embedding_blob ?? null,
|
|
@@ -1599,65 +2575,15 @@ After running the migration SQL, restart your application.`
|
|
|
1599
2575
|
if (facts.length > 0) {
|
|
1600
2576
|
const ids = facts.map((f) => f.id);
|
|
1601
2577
|
const now = Date.now();
|
|
1602
|
-
|
|
1603
|
-
for (let i = 0; i < ids.length; i += accessChunkSize) {
|
|
1604
|
-
const idChunk = ids.slice(i, i + accessChunkSize);
|
|
1605
|
-
const placeholders = idChunk.map(() => "?").join(",");
|
|
1606
|
-
await this.db.runAsync(
|
|
1607
|
-
`UPDATE ${this.prefix}entries
|
|
1608
|
-
SET access_count = access_count + 1, last_accessed_at = ?
|
|
1609
|
-
WHERE id IN (${placeholders})`,
|
|
1610
|
-
[now, ...idChunk]
|
|
1611
|
-
);
|
|
1612
|
-
}
|
|
2578
|
+
await this.entryRepo.trackAccess(ids, now);
|
|
1613
2579
|
}
|
|
1614
2580
|
} else {
|
|
1615
|
-
|
|
1616
|
-
const rawFacts = await this.db.getAllAsync(
|
|
1617
|
-
`SELECT * FROM ${this.prefix}entries
|
|
1618
|
-
WHERE ${entityScope.clause} AND deleted_at IS NULL
|
|
1619
|
-
ORDER BY updated_at DESC
|
|
1620
|
-
LIMIT ?`,
|
|
1621
|
-
[...entityScope.params, maxResults]
|
|
1622
|
-
);
|
|
1623
|
-
facts = rawFacts.map((f) => {
|
|
1624
|
-
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
1625
|
-
return {
|
|
1626
|
-
...rest,
|
|
1627
|
-
tags: (() => {
|
|
1628
|
-
if (Array.isArray(rest.tags)) return rest.tags;
|
|
1629
|
-
try {
|
|
1630
|
-
const p = JSON.parse(rest.tags);
|
|
1631
|
-
return Array.isArray(p) ? p : [];
|
|
1632
|
-
} catch {
|
|
1633
|
-
return [];
|
|
1634
|
-
}
|
|
1635
|
-
})()
|
|
1636
|
-
};
|
|
1637
|
-
});
|
|
2581
|
+
facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults);
|
|
1638
2582
|
}
|
|
2583
|
+
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
1639
2584
|
const [tasks, events] = await Promise.all([
|
|
1640
|
-
(
|
|
1641
|
-
|
|
1642
|
-
const tasksLimit = entityIds.length === 1 ? void 0 : Math.min(20 * entityIds.length, 200);
|
|
1643
|
-
return this.db.getAllAsync(
|
|
1644
|
-
`SELECT * FROM ${this.prefix}tasks
|
|
1645
|
-
WHERE ${entityScope.clause} AND status IN ('pending', 'in_progress') AND deleted_at IS NULL
|
|
1646
|
-
ORDER BY priority DESC, created_at ASC${tasksLimit !== void 0 ? "\n LIMIT ?" : ""}`,
|
|
1647
|
-
tasksLimit !== void 0 ? [...entityScope.params, tasksLimit] : entityScope.params
|
|
1648
|
-
);
|
|
1649
|
-
})(),
|
|
1650
|
-
(async () => {
|
|
1651
|
-
const entityScope = this._entityInClause(entityIds);
|
|
1652
|
-
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
1653
|
-
return this.db.getAllAsync(
|
|
1654
|
-
`SELECT * FROM ${this.prefix}events
|
|
1655
|
-
WHERE ${entityScope.clause}
|
|
1656
|
-
ORDER BY created_at DESC
|
|
1657
|
-
LIMIT ?`,
|
|
1658
|
-
[...entityScope.params, eventsLimit]
|
|
1659
|
-
);
|
|
1660
|
-
})()
|
|
2585
|
+
this.taskRepo.findAllPending(entityIds, entityIds.length === 1 ? void 0 : Math.min(20 * entityIds.length, 200)),
|
|
2586
|
+
entityIds.length === 1 ? this.eventRepo.getRecent(entityIds[0], eventsLimit) : this.eventRepo.getRecentForEntities(entityIds, eventsLimit)
|
|
1661
2587
|
]);
|
|
1662
2588
|
let factScores;
|
|
1663
2589
|
if (exposeMetadata && trimmedQuery && scoreByFactId) {
|
|
@@ -1700,14 +2626,6 @@ After running the migration SQL, restart your application.`
|
|
|
1700
2626
|
if (updatedAtDiff !== 0) return updatedAtDiff;
|
|
1701
2627
|
return a.id.localeCompare(b.id);
|
|
1702
2628
|
}
|
|
1703
|
-
/**
|
|
1704
|
-
* Build SQL IN clause with placeholders for multiple entity IDs.
|
|
1705
|
-
*/
|
|
1706
|
-
_entityInClause(entityIds) {
|
|
1707
|
-
if (entityIds.length === 0) return { clause: "1=0", params: [] };
|
|
1708
|
-
const placeholders = entityIds.map(() => "?").join(",");
|
|
1709
|
-
return { clause: `entity_id IN (${placeholders})`, params: [...entityIds] };
|
|
1710
|
-
}
|
|
1711
2629
|
/**
|
|
1712
2630
|
* Hydrate full facts by ID. Pass scopedEntityIds to restrict to requested namespaces in SQL
|
|
1713
2631
|
* (defense-in-depth against a rogue VectorRanker returning cross-entity IDs).
|
|
@@ -1845,53 +2763,60 @@ After running the migration SQL, restart your application.`
|
|
|
1845
2763
|
if (!["observation", "decision", "action", "outcome"].includes(eventType)) {
|
|
1846
2764
|
eventType = "observation";
|
|
1847
2765
|
}
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
let
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
const
|
|
1862
|
-
|
|
1863
|
-
this.
|
|
1864
|
-
this.
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
2766
|
+
const newEvent = {
|
|
2767
|
+
id,
|
|
2768
|
+
entity_id: entityId,
|
|
2769
|
+
event_type: eventType,
|
|
2770
|
+
summary: event.summary,
|
|
2771
|
+
related_entry_id: event.related_entry_id || null,
|
|
2772
|
+
created_at: now
|
|
2773
|
+
};
|
|
2774
|
+
let shouldRunLibrarian = false;
|
|
2775
|
+
let librarianCount = 0;
|
|
2776
|
+
let librarianJobKey = null;
|
|
2777
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2778
|
+
await this.eventRepo.add(newEvent, tx);
|
|
2779
|
+
const threshold = this.options.config?.autoLibrarianThreshold || 20;
|
|
2780
|
+
const [count, cp] = await Promise.all([
|
|
2781
|
+
this.eventRepo.count(entityId, tx),
|
|
2782
|
+
this.metadataRepo.getCheckpoint(entityId, tx)
|
|
2783
|
+
]);
|
|
2784
|
+
let memoryCheckpoint = cp.memory ?? 0;
|
|
2785
|
+
if (memoryCheckpoint > count) memoryCheckpoint = 0;
|
|
2786
|
+
if (count - memoryCheckpoint >= threshold) {
|
|
2787
|
+
const jobKey = this._librarianKey(entityId);
|
|
2788
|
+
if (!this.activeMaintenanceJobs.has(jobKey) && !this.activeMaintenanceJobs.has(this._pruneKey(entityId)) && !this._isReembedActive(entityId) && !this._isImportActiveFor(entityId) && !this._isForgetActiveFor(entityId)) {
|
|
2789
|
+
shouldRunLibrarian = true;
|
|
2790
|
+
librarianCount = count;
|
|
2791
|
+
librarianJobKey = jobKey;
|
|
2792
|
+
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
2793
|
+
}
|
|
1869
2794
|
}
|
|
2795
|
+
});
|
|
2796
|
+
if (shouldRunLibrarian && librarianJobKey !== null) {
|
|
2797
|
+
this.activeMaintenanceJobs.add(librarianJobKey);
|
|
2798
|
+
this._notifyStatusSubscribers(entityId);
|
|
2799
|
+
this.runLibrarianThenMaybeHeal(entityId, librarianCount).catch(console.error).finally(() => {
|
|
2800
|
+
this.activeMaintenanceJobs.delete(librarianJobKey);
|
|
2801
|
+
this._notifyStatusSubscribers(entityId);
|
|
2802
|
+
});
|
|
1870
2803
|
}
|
|
1871
2804
|
}
|
|
1872
2805
|
async runLibrarianThenMaybeHeal(entityId, currentEventCount) {
|
|
1873
2806
|
await this._doRunLibrarian(entityId);
|
|
1874
|
-
await this.db.runAsync(`
|
|
1875
|
-
INSERT INTO ${this.prefix}checkpoints (entity_id, memory_checkpoint)
|
|
1876
|
-
VALUES (?, ?)
|
|
1877
|
-
ON CONFLICT(entity_id) DO UPDATE SET memory_checkpoint = ?
|
|
1878
|
-
`, [entityId, currentEventCount, currentEventCount]);
|
|
1879
2807
|
const autoHealThreshold = this.options.config?.autoHealThreshold || 100;
|
|
1880
|
-
const cp = await this.
|
|
1881
|
-
let healCheckpoint = cp
|
|
2808
|
+
const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
|
|
2809
|
+
let healCheckpoint = cp.heal ?? 0;
|
|
1882
2810
|
if (healCheckpoint > currentEventCount) healCheckpoint = 0;
|
|
1883
|
-
|
|
2811
|
+
const shouldRunHeal = currentEventCount - healCheckpoint >= autoHealThreshold;
|
|
2812
|
+
if (shouldRunHeal) {
|
|
1884
2813
|
const healKey = this._healKey(entityId);
|
|
1885
2814
|
if (!this.activeMaintenanceJobs.has(healKey)) {
|
|
1886
2815
|
this.activeMaintenanceJobs.add(healKey);
|
|
1887
2816
|
this._notifyStatusSubscribers(entityId);
|
|
1888
2817
|
try {
|
|
1889
2818
|
await this._doRunHeal(entityId);
|
|
1890
|
-
await this.
|
|
1891
|
-
INSERT INTO ${this.prefix}checkpoints (entity_id, heal_checkpoint)
|
|
1892
|
-
VALUES (?, ?)
|
|
1893
|
-
ON CONFLICT(entity_id) DO UPDATE SET heal_checkpoint = ?
|
|
1894
|
-
`, [entityId, currentEventCount, currentEventCount]);
|
|
2819
|
+
await this.metadataRepo.updateCheckpoint(entityId, { heal: currentEventCount }, this.db);
|
|
1895
2820
|
} finally {
|
|
1896
2821
|
this.activeMaintenanceJobs.delete(healKey);
|
|
1897
2822
|
this._notifyStatusSubscribers(entityId);
|
|
@@ -1900,18 +2825,8 @@ After running the migration SQL, restart your application.`
|
|
|
1900
2825
|
}
|
|
1901
2826
|
}
|
|
1902
2827
|
async _doRunLibrarian(entityId) {
|
|
1903
|
-
const events = await this.
|
|
1904
|
-
|
|
1905
|
-
WHERE entity_id = ?
|
|
1906
|
-
ORDER BY created_at DESC
|
|
1907
|
-
LIMIT 50
|
|
1908
|
-
`, [entityId]);
|
|
1909
|
-
const currentFactsRows = await this.db.getAllAsync(`
|
|
1910
|
-
SELECT * FROM ${this.prefix}entries
|
|
1911
|
-
WHERE entity_id = ? AND deleted_at IS NULL
|
|
1912
|
-
ORDER BY updated_at DESC
|
|
1913
|
-
LIMIT 100
|
|
1914
|
-
`, [entityId]);
|
|
2828
|
+
const events = await this.eventRepo.getRecent(entityId, 50);
|
|
2829
|
+
const currentFactsRows = await this.entryRepo.findRecentByEntityId(entityId, 100);
|
|
1915
2830
|
const currentFacts = currentFactsRows.map((f) => {
|
|
1916
2831
|
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
1917
2832
|
return {
|
|
@@ -1935,12 +2850,13 @@ ${JSON.stringify(currentFacts, null, 2)}`;
|
|
|
1935
2850
|
const validTasks = tasks.map(validateTask).filter((t) => t !== null);
|
|
1936
2851
|
const now = Date.now();
|
|
1937
2852
|
const insertedFacts = [];
|
|
1938
|
-
await this.db.withTransactionAsync(async () => {
|
|
2853
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2854
|
+
const factsForDedupe = await this.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
1939
2855
|
for (const fact of validFacts) {
|
|
1940
2856
|
const newTokens = titleTokens(fact.title);
|
|
1941
2857
|
let skip = false;
|
|
1942
2858
|
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
1943
|
-
for (const existing of
|
|
2859
|
+
for (const existing of factsForDedupe) {
|
|
1944
2860
|
if (existing.source_type !== "librarian_inferred") continue;
|
|
1945
2861
|
const existingTokens = titleTokens(existing.title);
|
|
1946
2862
|
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
@@ -1953,18 +2869,29 @@ ${JSON.stringify(currentFacts, null, 2)}`;
|
|
|
1953
2869
|
}
|
|
1954
2870
|
if (skip) continue;
|
|
1955
2871
|
const id = generateId("fact_");
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
2872
|
+
const factObj = {
|
|
2873
|
+
id,
|
|
2874
|
+
entity_id: entityId,
|
|
2875
|
+
title: fact.title,
|
|
2876
|
+
body: fact.body,
|
|
2877
|
+
tags: fact.tags,
|
|
2878
|
+
confidence: fact.confidence,
|
|
2879
|
+
source_type: "librarian_inferred",
|
|
2880
|
+
source_hash: null,
|
|
2881
|
+
source_ref: null,
|
|
2882
|
+
created_at: now,
|
|
2883
|
+
updated_at: now,
|
|
2884
|
+
last_accessed_at: null,
|
|
2885
|
+
access_count: 0,
|
|
2886
|
+
deleted_at: null
|
|
2887
|
+
};
|
|
2888
|
+
await this.entryRepo.upsert(factObj, tx);
|
|
1960
2889
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
1961
2890
|
}
|
|
1962
2891
|
for (const task of validTasks) {
|
|
1963
2892
|
const id = generateId("task_");
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1967
|
-
`, [id, entityId, task.description, "pending", task.priority, now, now]);
|
|
2893
|
+
const taskObj = { id, entity_id: entityId, description: task.description, status: "pending", priority: task.priority, created_at: now, updated_at: now, resolved_at: null, deleted_at: null };
|
|
2894
|
+
await this.taskRepo.upsert(taskObj, tx);
|
|
1968
2895
|
}
|
|
1969
2896
|
});
|
|
1970
2897
|
await this.rebuildMiniSearchIndex(entityId);
|
|
@@ -1985,27 +2912,19 @@ ${JSON.stringify(currentFacts, null, 2)}`;
|
|
|
1985
2912
|
if (staleInferredAfterDays !== null && (typeof staleInferredAfterDays !== "number" || !Number.isFinite(staleInferredAfterDays) || staleInferredAfterDays < 0)) {
|
|
1986
2913
|
throw new Error("Invalid staleInferredAfterDays: must be a finite number >= 0 or null");
|
|
1987
2914
|
}
|
|
1988
|
-
await this.db.withTransactionAsync(async () => {
|
|
2915
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
1989
2916
|
if (orphanAfterDays !== null) {
|
|
1990
2917
|
const orphanThreshold = now - orphanAfterDays * MS_PER_DAY;
|
|
1991
|
-
await this.
|
|
1992
|
-
UPDATE ${this.prefix}entries
|
|
1993
|
-
SET deleted_at = ?, updated_at = ?
|
|
1994
|
-
WHERE entity_id = ? AND access_count = 0 AND created_at <= ? AND source_type != 'immutable_document' AND deleted_at IS NULL
|
|
1995
|
-
`, [now, now, entityId, orphanThreshold]);
|
|
2918
|
+
await this.entryRepo.markOrphaned(entityId, orphanThreshold, tx);
|
|
1996
2919
|
}
|
|
1997
2920
|
if (staleInferredAfterDays !== null) {
|
|
1998
2921
|
const staleThreshold = now - staleInferredAfterDays * MS_PER_DAY;
|
|
1999
|
-
await this.
|
|
2000
|
-
UPDATE ${this.prefix}entries
|
|
2001
|
-
SET confidence = 'tentative', updated_at = ?
|
|
2002
|
-
WHERE entity_id = ? AND confidence = 'inferred' AND (last_accessed_at <= ? OR (last_accessed_at IS NULL AND created_at <= ?)) AND source_type != 'immutable_document' AND deleted_at IS NULL
|
|
2003
|
-
`, [now, entityId, staleThreshold, staleThreshold]);
|
|
2922
|
+
await this.entryRepo.downgradeStaleInferred(entityId, staleThreshold, tx);
|
|
2004
2923
|
}
|
|
2005
2924
|
});
|
|
2006
|
-
const allFactsRows = await this.
|
|
2007
|
-
const allTasks = await this.
|
|
2008
|
-
const recentEvents = await this.
|
|
2925
|
+
const allFactsRows = await this.entryRepo.findAllByEntityId(entityId);
|
|
2926
|
+
const allTasks = await this.taskRepo.findAllPending([entityId]);
|
|
2927
|
+
const recentEvents = await this.eventRepo.getRecent(entityId, 20);
|
|
2009
2928
|
const healCandidates = allFactsRows.filter((f) => f.source_type !== "immutable_document");
|
|
2010
2929
|
const documentAnchors = allFactsRows.filter((f) => f.source_type === "immutable_document").map(({ id, title, source_ref }) => ({ id, title, source_ref }));
|
|
2011
2930
|
const userPrompt = `Heal Candidates:
|
|
@@ -2038,19 +2957,28 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2038
2957
|
const validNewFacts = newFacts.map(validateFact).filter((f) => f !== null);
|
|
2039
2958
|
const insertedFacts = [];
|
|
2040
2959
|
const uniqueDeletedFactIds = Array.from(new Set(safeDeleted));
|
|
2041
|
-
await this.db.withTransactionAsync(async () => {
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
}
|
|
2045
|
-
for (const id of safeDeleted) {
|
|
2046
|
-
await this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ?`, [now, now, id, entityId]);
|
|
2047
|
-
}
|
|
2960
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2961
|
+
await this.entryRepo.downgradeByIds(safeDowngraded, entityId, tx);
|
|
2962
|
+
await this.entryRepo.softDeleteByIds(safeDeleted, entityId, tx);
|
|
2048
2963
|
for (const fact of validNewFacts) {
|
|
2049
2964
|
const id = generateId("fact_");
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2965
|
+
const factObj = {
|
|
2966
|
+
id,
|
|
2967
|
+
entity_id: entityId,
|
|
2968
|
+
title: fact.title,
|
|
2969
|
+
body: fact.body,
|
|
2970
|
+
tags: fact.tags,
|
|
2971
|
+
confidence: fact.confidence,
|
|
2972
|
+
source_type: "librarian_inferred",
|
|
2973
|
+
source_hash: null,
|
|
2974
|
+
source_ref: null,
|
|
2975
|
+
created_at: now,
|
|
2976
|
+
updated_at: now,
|
|
2977
|
+
last_accessed_at: null,
|
|
2978
|
+
access_count: 0,
|
|
2979
|
+
deleted_at: null
|
|
2980
|
+
};
|
|
2981
|
+
await this.entryRepo.upsert(factObj, tx);
|
|
2054
2982
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
2055
2983
|
}
|
|
2056
2984
|
});
|
|
@@ -2174,12 +3102,7 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2174
3102
|
}
|
|
2175
3103
|
this.activeMaintenanceJobs.add(reembedKey);
|
|
2176
3104
|
try {
|
|
2177
|
-
const
|
|
2178
|
-
const params = entityId ? [entityId] : [];
|
|
2179
|
-
const rows = await this.db.getAllAsync(
|
|
2180
|
-
`SELECT * FROM ${this.prefix}entries WHERE ${where}`,
|
|
2181
|
-
params
|
|
2182
|
-
);
|
|
3105
|
+
const rows = await this.entryRepo.findAllForReembed(entityId);
|
|
2183
3106
|
if (entityId) {
|
|
2184
3107
|
this.vectorCache.delete(entityId);
|
|
2185
3108
|
} else {
|
|
@@ -2188,22 +3111,12 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2188
3111
|
const skipExisting = opts?.skipExisting ?? false;
|
|
2189
3112
|
let effectiveSkip = skipExisting;
|
|
2190
3113
|
if (skipExisting) {
|
|
2191
|
-
const
|
|
2192
|
-
|
|
2193
|
-
);
|
|
2194
|
-
if (mismatchRow) {
|
|
3114
|
+
const mismatchValue = await this.metadataRepo.getMeta("embedding_dimension_mismatch");
|
|
3115
|
+
if (mismatchValue) {
|
|
2195
3116
|
if (entityId) {
|
|
2196
|
-
const mismatchDim = parseInt(
|
|
2197
|
-
const
|
|
2198
|
-
|
|
2199
|
-
WHERE entity_id = ? AND deleted_at IS NULL
|
|
2200
|
-
AND (
|
|
2201
|
-
embedding_blob IS NULL
|
|
2202
|
-
OR (CAST(length(embedding_blob) AS INTEGER) / 4) != ?
|
|
2203
|
-
)`,
|
|
2204
|
-
[entityId, mismatchDim]
|
|
2205
|
-
);
|
|
2206
|
-
if (staleForEntity && staleForEntity.cnt > 0) effectiveSkip = false;
|
|
3117
|
+
const mismatchDim = parseInt(mismatchValue, 10);
|
|
3118
|
+
const staleCount = await this.entryRepo.countStaleForEntity(entityId, mismatchDim);
|
|
3119
|
+
if (staleCount > 0) effectiveSkip = false;
|
|
2207
3120
|
} else {
|
|
2208
3121
|
effectiveSkip = false;
|
|
2209
3122
|
}
|
|
@@ -2295,19 +3208,10 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2295
3208
|
this.vectorCache.clear();
|
|
2296
3209
|
}
|
|
2297
3210
|
async _getFullBundle(entityId, opts) {
|
|
2298
|
-
const
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
this.db.getAllAsync(
|
|
2303
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC`,
|
|
2304
|
-
[entityId]
|
|
2305
|
-
),
|
|
2306
|
-
this.db.getAllAsync(
|
|
2307
|
-
`SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC`,
|
|
2308
|
-
[entityId]
|
|
2309
|
-
),
|
|
2310
|
-
this.db.getAllAsync(eventsQuery, eventsParams)
|
|
3211
|
+
const [factsRaw, tasks, events] = await Promise.all([
|
|
3212
|
+
this.entryRepo.findAllByEntityId(entityId),
|
|
3213
|
+
this.taskRepo.findAllByEntityId(entityId),
|
|
3214
|
+
this.eventRepo.getByEntityId(entityId, opts?.maxEvents)
|
|
2311
3215
|
]);
|
|
2312
3216
|
const facts = factsRaw.map((f) => {
|
|
2313
3217
|
const { embedding: _embedding, embedding_blob, ...rest } = f;
|
|
@@ -2322,7 +3226,6 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2322
3226
|
tags: typeof factBase.tags === "string" ? JSON.parse(factBase.tags) : factBase.tags
|
|
2323
3227
|
};
|
|
2324
3228
|
});
|
|
2325
|
-
const events = maxEvents != null ? eventsRaw.slice().reverse() : eventsRaw;
|
|
2326
3229
|
return { facts, tasks, events };
|
|
2327
3230
|
}
|
|
2328
3231
|
async exportDump(entityIds) {
|
|
@@ -2330,16 +3233,7 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2330
3233
|
if (entityIds && entityIds.length > 0) {
|
|
2331
3234
|
ids = Array.from(new Set(entityIds));
|
|
2332
3235
|
} else {
|
|
2333
|
-
|
|
2334
|
-
SELECT DISTINCT entity_id FROM (
|
|
2335
|
-
SELECT entity_id FROM ${this.prefix}entries WHERE deleted_at IS NULL
|
|
2336
|
-
UNION
|
|
2337
|
-
SELECT entity_id FROM ${this.prefix}tasks WHERE deleted_at IS NULL
|
|
2338
|
-
UNION
|
|
2339
|
-
SELECT entity_id FROM ${this.prefix}events
|
|
2340
|
-
) ORDER BY entity_id
|
|
2341
|
-
`);
|
|
2342
|
-
ids = rows.map((r) => r.entity_id);
|
|
3236
|
+
ids = await this.metadataRepo.getDistinctEntityIds();
|
|
2343
3237
|
}
|
|
2344
3238
|
const entities = {};
|
|
2345
3239
|
const BATCH = 3;
|
|
@@ -2405,48 +3299,26 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2405
3299
|
const factsWithPreservedBlob = /* @__PURE__ */ new Map();
|
|
2406
3300
|
const preservedBlobDims = /* @__PURE__ */ new Set();
|
|
2407
3301
|
const softDeletedFactIds = [];
|
|
2408
|
-
await this.db.withTransactionAsync(async () => {
|
|
3302
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2409
3303
|
if (!merge) {
|
|
2410
|
-
const
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
);
|
|
2414
|
-
|
|
2415
|
-
const now = Date.now();
|
|
2416
|
-
await this.db.runAsync(
|
|
2417
|
-
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
2418
|
-
[now, now, entityId]
|
|
2419
|
-
);
|
|
2420
|
-
await this.db.runAsync(
|
|
2421
|
-
`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
2422
|
-
[now, now, entityId]
|
|
2423
|
-
);
|
|
2424
|
-
await this.db.runAsync(
|
|
2425
|
-
`DELETE FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
2426
|
-
[entityId]
|
|
2427
|
-
);
|
|
3304
|
+
const deletedLiveFactIds = await this.entryRepo.findIdsBySource(entityId, null, null, tx, false);
|
|
3305
|
+
softDeletedFactIds.push(...deletedLiveFactIds);
|
|
3306
|
+
await this.entryRepo.bulkSoftDeleteByEntityId(entityId, tx);
|
|
3307
|
+
await this.taskRepo.bulkSoftDeleteByEntityId(entityId, tx);
|
|
3308
|
+
await this.metadataRepo.deleteCheckpoint(entityId, tx);
|
|
2428
3309
|
}
|
|
2429
3310
|
const factIds = bundle.facts.map((fact) => fact.id);
|
|
2430
3311
|
const existingFactsById = /* @__PURE__ */ new Map();
|
|
2431
|
-
const
|
|
2432
|
-
for (
|
|
2433
|
-
|
|
2434
|
-
if (factIdChunk.length === 0) continue;
|
|
2435
|
-
const placeholders = factIdChunk.map(() => "?").join(", ");
|
|
2436
|
-
const existingFacts = await this.db.getAllAsync(
|
|
2437
|
-
`SELECT id, entity_id, updated_at FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
|
|
2438
|
-
factIdChunk
|
|
2439
|
-
);
|
|
2440
|
-
for (const existingFact of existingFacts) {
|
|
2441
|
-
existingFactsById.set(existingFact.id, existingFact);
|
|
2442
|
-
}
|
|
3312
|
+
const existingFacts = await this.entryRepo.findExistingMetadataByIds(factIds, tx);
|
|
3313
|
+
for (const existingFact of existingFacts) {
|
|
3314
|
+
existingFactsById.set(existingFact.id, existingFact);
|
|
2443
3315
|
}
|
|
2444
3316
|
for (const fact of bundle.facts) {
|
|
2445
3317
|
const sourceType = this._normalizeImportedSourceType(String(fact.source_type), {
|
|
2446
3318
|
entityId,
|
|
2447
3319
|
factId: fact.id
|
|
2448
3320
|
});
|
|
2449
|
-
|
|
3321
|
+
JSON.stringify(Array.isArray(fact.tags) ? fact.tags : []);
|
|
2450
3322
|
const safeUpdatedAt = Number.isFinite(fact.updated_at) ? fact.updated_at : 0;
|
|
2451
3323
|
const existing = existingFactsById.get(fact.id);
|
|
2452
3324
|
const rawBlobRaw = fact.embedding_blob;
|
|
@@ -2491,55 +3363,38 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2491
3363
|
if (merge) {
|
|
2492
3364
|
if (safeUpdatedAt <= existing.updated_at) continue;
|
|
2493
3365
|
}
|
|
2494
|
-
if (blobData != null) {
|
|
2495
|
-
await this.db.runAsync(
|
|
2496
|
-
`UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ?, embedding_blob = ?, embedding = NULL WHERE id = ?`,
|
|
2497
|
-
[entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData, fact.id]
|
|
2498
|
-
);
|
|
2499
|
-
factsWithPreservedBlob.set(fact.id, blobData);
|
|
2500
|
-
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
2501
|
-
} else {
|
|
2502
|
-
await this.db.runAsync(
|
|
2503
|
-
`UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ?, embedding_blob = NULL, embedding = NULL WHERE id = ?`,
|
|
2504
|
-
[entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
|
|
2505
|
-
);
|
|
2506
|
-
}
|
|
2507
|
-
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2508
|
-
upsertedFactIds.add(fact.id);
|
|
2509
|
-
if (fact.deleted_at) upsertedDeletedFactIds.add(fact.id);
|
|
2510
|
-
} else {
|
|
2511
|
-
if (blobData != null) {
|
|
2512
|
-
await this.db.runAsync(
|
|
2513
|
-
`INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at, embedding_blob) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2514
|
-
[fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData]
|
|
2515
|
-
);
|
|
2516
|
-
factsWithPreservedBlob.set(fact.id, blobData);
|
|
2517
|
-
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
2518
|
-
} else {
|
|
2519
|
-
await this.db.runAsync(
|
|
2520
|
-
`INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2521
|
-
[fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at]
|
|
2522
|
-
);
|
|
2523
|
-
}
|
|
2524
|
-
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2525
|
-
upsertedFactIds.add(fact.id);
|
|
2526
|
-
if (fact.deleted_at) upsertedDeletedFactIds.add(fact.id);
|
|
2527
3366
|
}
|
|
3367
|
+
const factObj = {
|
|
3368
|
+
id: fact.id,
|
|
3369
|
+
entity_id: entityId,
|
|
3370
|
+
title: fact.title,
|
|
3371
|
+
body: fact.body,
|
|
3372
|
+
tags: Array.isArray(fact.tags) ? fact.tags : [],
|
|
3373
|
+
confidence: fact.confidence,
|
|
3374
|
+
source_type: sourceType,
|
|
3375
|
+
source_hash: fact.source_hash,
|
|
3376
|
+
source_ref: fact.source_ref,
|
|
3377
|
+
created_at: fact.created_at,
|
|
3378
|
+
updated_at: safeUpdatedAt,
|
|
3379
|
+
last_accessed_at: fact.last_accessed_at,
|
|
3380
|
+
access_count: fact.access_count,
|
|
3381
|
+
deleted_at: fact.deleted_at,
|
|
3382
|
+
embedding_blob: blobData ?? void 0
|
|
3383
|
+
};
|
|
3384
|
+
await this.entryRepo.upsertForImport(factObj, tx);
|
|
3385
|
+
if (blobData != null) {
|
|
3386
|
+
factsWithPreservedBlob.set(fact.id, blobData);
|
|
3387
|
+
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
3388
|
+
}
|
|
3389
|
+
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
3390
|
+
upsertedFactIds.add(fact.id);
|
|
3391
|
+
if (fact.deleted_at) upsertedDeletedFactIds.add(fact.id);
|
|
2528
3392
|
}
|
|
2529
3393
|
const taskIds = bundle.tasks.map((task) => task.id);
|
|
2530
3394
|
const existingTasksById = /* @__PURE__ */ new Map();
|
|
2531
|
-
const
|
|
2532
|
-
for (
|
|
2533
|
-
|
|
2534
|
-
if (taskIdChunk.length === 0) continue;
|
|
2535
|
-
const placeholders = taskIdChunk.map(() => "?").join(", ");
|
|
2536
|
-
const existingTasks = await this.db.getAllAsync(
|
|
2537
|
-
`SELECT id, entity_id, updated_at FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
|
|
2538
|
-
taskIdChunk
|
|
2539
|
-
);
|
|
2540
|
-
for (const existingTask of existingTasks) {
|
|
2541
|
-
existingTasksById.set(existingTask.id, existingTask);
|
|
2542
|
-
}
|
|
3395
|
+
const existingTasks = await this.taskRepo.findExistingMetadataByIds(taskIds, tx);
|
|
3396
|
+
for (const existingTask of existingTasks) {
|
|
3397
|
+
existingTasksById.set(existingTask.id, existingTask);
|
|
2543
3398
|
}
|
|
2544
3399
|
for (const task of bundle.tasks) {
|
|
2545
3400
|
const safeUpdatedAt = Number.isFinite(task.updated_at) ? task.updated_at : 0;
|
|
@@ -2552,25 +3407,29 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2552
3407
|
if (merge) {
|
|
2553
3408
|
if (safeUpdatedAt <= existing.updated_at) continue;
|
|
2554
3409
|
}
|
|
2555
|
-
await this.db.runAsync(
|
|
2556
|
-
`UPDATE ${this.prefix}tasks SET entity_id = ?, description = ?, status = ?, priority = ?, created_at = ?, updated_at = ?, resolved_at = ?, deleted_at = ? WHERE id = ?`,
|
|
2557
|
-
[entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at, task.id]
|
|
2558
|
-
);
|
|
2559
|
-
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2560
|
-
} else {
|
|
2561
|
-
await this.db.runAsync(
|
|
2562
|
-
`INSERT INTO ${this.prefix}tasks (id, entity_id, description, status, priority, created_at, updated_at, resolved_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2563
|
-
[task.id, entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at]
|
|
2564
|
-
);
|
|
2565
|
-
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2566
3410
|
}
|
|
3411
|
+
await this.taskRepo.upsertForImport({
|
|
3412
|
+
id: task.id,
|
|
3413
|
+
entity_id: entityId,
|
|
3414
|
+
description: task.description,
|
|
3415
|
+
status: task.status,
|
|
3416
|
+
priority: task.priority,
|
|
3417
|
+
created_at: task.created_at,
|
|
3418
|
+
updated_at: safeUpdatedAt,
|
|
3419
|
+
resolved_at: task.resolved_at,
|
|
3420
|
+
deleted_at: task.deleted_at
|
|
3421
|
+
}, tx, safeUpdatedAt);
|
|
3422
|
+
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2567
3423
|
}
|
|
2568
3424
|
for (const event of bundle.events) {
|
|
2569
|
-
await this.
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
3425
|
+
await this.eventRepo.addIgnoreDuplicate({
|
|
3426
|
+
id: event.id,
|
|
3427
|
+
entity_id: entityId,
|
|
3428
|
+
event_type: event.event_type,
|
|
3429
|
+
summary: event.summary,
|
|
3430
|
+
related_entry_id: event.related_entry_id ?? null,
|
|
3431
|
+
created_at: event.created_at
|
|
3432
|
+
}, tx);
|
|
2574
3433
|
}
|
|
2575
3434
|
});
|
|
2576
3435
|
this.vectorCache.delete(entityId);
|
|
@@ -2608,43 +3467,27 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2608
3467
|
}
|
|
2609
3468
|
}
|
|
2610
3469
|
try {
|
|
2611
|
-
const
|
|
2612
|
-
|
|
2613
|
-
);
|
|
2614
|
-
const canonicalDim = canonicalRow ? parseInt(canonicalRow.value, 10) : null;
|
|
3470
|
+
const canonicalDimValue = await this.metadataRepo.getMeta("embedding_dimension");
|
|
3471
|
+
const canonicalDim = canonicalDimValue ? parseInt(canonicalDimValue, 10) : null;
|
|
2615
3472
|
if (preservedBlobDims.size === 1) {
|
|
2616
3473
|
const preservedDim = [...preservedBlobDims][0];
|
|
2617
3474
|
if (canonicalDim === null || canonicalDim === preservedDim) {
|
|
2618
3475
|
await this.storeEmbeddingDimension(preservedDim);
|
|
2619
|
-
const
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
if (staleMismatch && parseInt(staleMismatch.value, 10) !== preservedDim) {
|
|
2623
|
-
await this.db.runAsync(
|
|
2624
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
2625
|
-
[String(preservedDim)]
|
|
2626
|
-
);
|
|
3476
|
+
const staleMismatchValue = await this.metadataRepo.getMeta("embedding_dimension_mismatch");
|
|
3477
|
+
if (staleMismatchValue && parseInt(staleMismatchValue, 10) !== preservedDim) {
|
|
3478
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(preservedDim), this.db);
|
|
2627
3479
|
}
|
|
2628
3480
|
await this._reconcileEmbeddingDimension();
|
|
2629
3481
|
} else {
|
|
2630
|
-
await this.
|
|
2631
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
2632
|
-
[String(canonicalDim)]
|
|
2633
|
-
);
|
|
3482
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(canonicalDim), this.db);
|
|
2634
3483
|
}
|
|
2635
3484
|
} else if (preservedBlobDims.size > 1) {
|
|
2636
3485
|
if (canonicalDim === null) {
|
|
2637
3486
|
const sortedPreservedBlobDims = [...preservedBlobDims].sort((a, b) => a - b);
|
|
2638
3487
|
await this.storeEmbeddingDimension(sortedPreservedBlobDims[0]);
|
|
2639
|
-
await this.
|
|
2640
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
2641
|
-
[String(sortedPreservedBlobDims[0])]
|
|
2642
|
-
);
|
|
3488
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(sortedPreservedBlobDims[0]), this.db);
|
|
2643
3489
|
} else {
|
|
2644
|
-
await this.
|
|
2645
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
2646
|
-
[String(canonicalDim)]
|
|
2647
|
-
);
|
|
3490
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(canonicalDim), this.db);
|
|
2648
3491
|
}
|
|
2649
3492
|
}
|
|
2650
3493
|
} finally {
|
|
@@ -2678,79 +3521,44 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2678
3521
|
let deletedEntries = 0;
|
|
2679
3522
|
let deletedTasks = 0;
|
|
2680
3523
|
const deletedEntryIds = [];
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
]);
|
|
2695
|
-
await this.db.runAsync(`UPDATE ${this.prefix}checkpoints SET memory_checkpoint = 0, heal_checkpoint = 0 WHERE entity_id = ?`, [entityId]);
|
|
2696
|
-
deletedEntries = entriesRes.changes;
|
|
2697
|
-
deletedTasks = tasksRes.changes;
|
|
2698
|
-
} else {
|
|
2699
|
-
const hasIdSelectors = params.entryId !== void 0 || params.taskId !== void 0;
|
|
2700
|
-
const hasSourceSelectors = params.sourceRef !== void 0 || params.sourceHash !== void 0;
|
|
2701
|
-
if (hasIdSelectors && hasSourceSelectors) {
|
|
2702
|
-
throw new Error("forget() params are mutually exclusive: use entryId/taskId together, or sourceRef/sourceHash together, but not both in the same call");
|
|
2703
|
-
}
|
|
2704
|
-
const sourceRef = params.sourceRef !== void 0 ? normalizeSourceRef(params.sourceRef) : null;
|
|
2705
|
-
if (params.sourceRef !== void 0 && !sourceRef) throw new Error("Invalid sourceRef");
|
|
2706
|
-
const sourceHash = params.sourceHash !== void 0 ? normalizeSourceHash(params.sourceHash) : null;
|
|
2707
|
-
if (params.sourceHash !== void 0 && !sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
2708
|
-
if (params.entryId) {
|
|
2709
|
-
const entry = await this.db.getFirstAsync(
|
|
2710
|
-
`SELECT id FROM ${this.prefix}entries WHERE id = ? AND entity_id = ?`,
|
|
2711
|
-
[params.entryId, entityId]
|
|
2712
|
-
);
|
|
2713
|
-
if (entry) deletedEntryIds.push(entry.id);
|
|
2714
|
-
}
|
|
2715
|
-
if (sourceRef || sourceHash) {
|
|
2716
|
-
let q = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ?`;
|
|
2717
|
-
const args = [entityId];
|
|
2718
|
-
if (sourceRef) {
|
|
2719
|
-
q += ` AND source_ref = ?`;
|
|
2720
|
-
args.push(sourceRef);
|
|
2721
|
-
}
|
|
2722
|
-
if (sourceHash) {
|
|
2723
|
-
q += ` AND source_hash = ?`;
|
|
2724
|
-
args.push(sourceHash);
|
|
3524
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3525
|
+
if (params.clearAll) {
|
|
3526
|
+
deletedEntryIds.push(...await this.entryRepo.findIdsBySource(entityId, null, null, tx, true));
|
|
3527
|
+
const entriesRes = await this.entryRepo.bulkSoftDeleteByEntityId(entityId, tx);
|
|
3528
|
+
const tasksRes = await this.taskRepo.bulkSoftDeleteByEntityId(entityId, tx);
|
|
3529
|
+
await this.metadataRepo.updateCheckpoint(entityId, { memory: 0, heal: 0 }, tx);
|
|
3530
|
+
deletedEntries = entriesRes;
|
|
3531
|
+
deletedTasks = tasksRes;
|
|
3532
|
+
} else {
|
|
3533
|
+
const hasIdSelectors = params.entryId !== void 0 || params.taskId !== void 0;
|
|
3534
|
+
const hasSourceSelectors = params.sourceRef !== void 0 || params.sourceHash !== void 0;
|
|
3535
|
+
if (hasIdSelectors && hasSourceSelectors) {
|
|
3536
|
+
throw new Error("forget() params are mutually exclusive: use entryId/taskId together, or sourceRef/sourceHash together, but not both in the same call");
|
|
2725
3537
|
}
|
|
2726
|
-
const
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
let q = `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`;
|
|
2734
|
-
const args = [now, now, entityId];
|
|
2735
|
-
if (sourceRef) {
|
|
2736
|
-
q += ` AND source_ref = ?`;
|
|
2737
|
-
args.push(sourceRef);
|
|
3538
|
+
const sourceRef = params.sourceRef !== void 0 ? normalizeSourceRef(params.sourceRef) : null;
|
|
3539
|
+
if (params.sourceRef !== void 0 && !sourceRef) throw new Error("Invalid sourceRef");
|
|
3540
|
+
const sourceHash = params.sourceHash !== void 0 ? normalizeSourceHash(params.sourceHash) : null;
|
|
3541
|
+
if (params.sourceHash !== void 0 && !sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
3542
|
+
if (params.entryId) {
|
|
3543
|
+
const entryId = await this.entryRepo.findIdById(params.entryId, entityId, tx);
|
|
3544
|
+
if (entryId) deletedEntryIds.push(entryId);
|
|
2738
3545
|
}
|
|
2739
|
-
if (sourceHash) {
|
|
2740
|
-
|
|
2741
|
-
args.push(sourceHash);
|
|
3546
|
+
if (sourceRef || sourceHash) {
|
|
3547
|
+
deletedEntryIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, sourceHash, tx, true));
|
|
2742
3548
|
}
|
|
2743
|
-
|
|
3549
|
+
const entryPromise = params.entryId ? this.entryRepo.softDelete(params.entryId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
3550
|
+
const taskDeletedPromise = params.taskId ? this.taskRepo.softDeleteById(params.taskId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
3551
|
+
const refPromise = sourceRef || sourceHash ? this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, sourceHash) : null;
|
|
3552
|
+
const [entryResult, taskResult, refResult] = await Promise.all([
|
|
3553
|
+
entryPromise ?? Promise.resolve(false),
|
|
3554
|
+
taskDeletedPromise ?? Promise.resolve(false),
|
|
3555
|
+
refPromise ?? Promise.resolve(0)
|
|
3556
|
+
]);
|
|
3557
|
+
if (entryResult) deletedEntries++;
|
|
3558
|
+
if (taskResult) deletedTasks++;
|
|
3559
|
+
deletedEntries += refResult;
|
|
2744
3560
|
}
|
|
2745
|
-
|
|
2746
|
-
entryPromise ?? Promise.resolve(null),
|
|
2747
|
-
taskPromise ?? Promise.resolve(null),
|
|
2748
|
-
refPromise ?? Promise.resolve(null)
|
|
2749
|
-
]);
|
|
2750
|
-
if (entryResult) deletedEntries += entryResult.changes;
|
|
2751
|
-
if (taskResult) deletedTasks += taskResult.changes;
|
|
2752
|
-
if (refResult) deletedEntries += refResult.changes;
|
|
2753
|
-
}
|
|
3561
|
+
});
|
|
2754
3562
|
await this.rebuildMiniSearchIndex(entityId);
|
|
2755
3563
|
this.vectorCache.delete(entityId);
|
|
2756
3564
|
const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
|
|
@@ -2849,18 +3657,9 @@ ${chunk}`;
|
|
|
2849
3657
|
const now = Date.now();
|
|
2850
3658
|
const insertedFacts = [];
|
|
2851
3659
|
const deletedSourceFactIds = [];
|
|
2852
|
-
await this.db.withTransactionAsync(async () => {
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
[sourceRef, entityId]
|
|
2856
|
-
);
|
|
2857
|
-
for (const row of existingSourceFacts) {
|
|
2858
|
-
deletedSourceFactIds.push(row.id);
|
|
2859
|
-
}
|
|
2860
|
-
await this.db.runAsync(
|
|
2861
|
-
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
2862
|
-
[now, now, sourceRef, entityId]
|
|
2863
|
-
);
|
|
3660
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3661
|
+
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
3662
|
+
await this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, null);
|
|
2864
3663
|
for (const fact of allValidFacts) {
|
|
2865
3664
|
const id = generateId("fact_");
|
|
2866
3665
|
const wikiFact = {
|
|
@@ -2879,7 +3678,7 @@ ${chunk}`;
|
|
|
2879
3678
|
access_count: 0,
|
|
2880
3679
|
deleted_at: null
|
|
2881
3680
|
};
|
|
2882
|
-
await this.entryRepo.upsert(wikiFact);
|
|
3681
|
+
await this.entryRepo.upsert(wikiFact, tx);
|
|
2883
3682
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
2884
3683
|
}
|
|
2885
3684
|
});
|