@equationalapplications/core-llm-wiki 4.6.1 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +1463 -664
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1463 -664
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -68,6 +68,19 @@ async function setupDatabase(db, prefix) {
|
|
|
68
68
|
key TEXT PRIMARY KEY,
|
|
69
69
|
value TEXT NOT NULL
|
|
70
70
|
);
|
|
71
|
+
|
|
72
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
73
|
+
id TEXT PRIMARY KEY,
|
|
74
|
+
entity_id TEXT NOT NULL,
|
|
75
|
+
table_name TEXT NOT NULL,
|
|
76
|
+
record_id TEXT NOT NULL,
|
|
77
|
+
operation TEXT NOT NULL,
|
|
78
|
+
payload TEXT NOT NULL,
|
|
79
|
+
created_at INTEGER NOT NULL
|
|
80
|
+
);
|
|
81
|
+
|
|
82
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
83
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
71
84
|
`);
|
|
72
85
|
}
|
|
73
86
|
|
|
@@ -83,8 +96,8 @@ var MIGRATIONS = [
|
|
|
83
96
|
version: 2,
|
|
84
97
|
description: "Remove FTS5; add embedding column for semantic retrieval",
|
|
85
98
|
run: async (db, prefix) => {
|
|
86
|
-
await db.withTransactionAsync(async () => {
|
|
87
|
-
await
|
|
99
|
+
await db.withTransactionAsync(async (tx) => {
|
|
100
|
+
await tx.execAsync(`
|
|
88
101
|
DROP TRIGGER IF EXISTS ${prefix}entries_ai;
|
|
89
102
|
DROP TRIGGER IF EXISTS ${prefix}entries_ad;
|
|
90
103
|
DROP TRIGGER IF EXISTS ${prefix}entries_au;
|
|
@@ -112,6 +125,25 @@ var MIGRATIONS = [
|
|
|
112
125
|
);
|
|
113
126
|
}
|
|
114
127
|
}
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
version: 4,
|
|
131
|
+
description: "Create outbox table for change data capture",
|
|
132
|
+
run: async (db, prefix) => {
|
|
133
|
+
await db.execAsync(`
|
|
134
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
135
|
+
id TEXT PRIMARY KEY,
|
|
136
|
+
entity_id TEXT NOT NULL,
|
|
137
|
+
table_name TEXT NOT NULL,
|
|
138
|
+
record_id TEXT NOT NULL,
|
|
139
|
+
operation TEXT NOT NULL,
|
|
140
|
+
payload TEXT NOT NULL,
|
|
141
|
+
created_at INTEGER NOT NULL
|
|
142
|
+
);
|
|
143
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
144
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
145
|
+
`);
|
|
146
|
+
}
|
|
115
147
|
}
|
|
116
148
|
];
|
|
117
149
|
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
@@ -189,8 +221,9 @@ function mapRowToFact(row) {
|
|
|
189
221
|
};
|
|
190
222
|
}
|
|
191
223
|
var EntryRepository = class extends BaseRepository {
|
|
192
|
-
constructor() {
|
|
193
|
-
super(
|
|
224
|
+
constructor(db, prefix, outbox) {
|
|
225
|
+
super(db, prefix);
|
|
226
|
+
this.outbox = outbox;
|
|
194
227
|
this.chunkSize = 500;
|
|
195
228
|
}
|
|
196
229
|
/**
|
|
@@ -211,89 +244,1180 @@ var EntryRepository = class extends BaseRepository {
|
|
|
211
244
|
);
|
|
212
245
|
rows.push(...chunkRows);
|
|
213
246
|
}
|
|
214
|
-
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
215
|
-
return ids.map((id) => byId.get(id)).filter((r) => r !== void 0).map(mapRowToFact);
|
|
247
|
+
const byId = new Map(rows.map((r) => [r.id, r]));
|
|
248
|
+
return ids.map((id) => byId.get(id)).filter((r) => r !== void 0).map(mapRowToFact);
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Upsert a WikiFact. Nullable fields set to null when fact value is null.
|
|
252
|
+
* Returns { changes, lastInsertRowId }.
|
|
253
|
+
* `tx` is REQUIRED to ensure atomic outbox staging.
|
|
254
|
+
*/
|
|
255
|
+
async upsert(fact, tx) {
|
|
256
|
+
const executor = this.getExecutor(tx);
|
|
257
|
+
const now = Date.now();
|
|
258
|
+
const tagsJson = JSON.stringify(fact.tags);
|
|
259
|
+
const embeddingBlob = this.normalizeEmbeddingBlob(fact.embedding_blob);
|
|
260
|
+
const existingRow = await executor.getFirstAsync(
|
|
261
|
+
`SELECT id FROM ${this.prefix}entries WHERE id = ?`,
|
|
262
|
+
[fact.id]
|
|
263
|
+
);
|
|
264
|
+
const operation = fact.deleted_at ? "DELETE" : existingRow ? "UPDATE" : "INSERT";
|
|
265
|
+
const result = await executor.runAsync(
|
|
266
|
+
`INSERT INTO ${this.prefix}entries (
|
|
267
|
+
id, entity_id, title, body, tags, confidence, source_type,
|
|
268
|
+
source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count,
|
|
269
|
+
deleted_at, embedding_blob, embedding
|
|
270
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
271
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
272
|
+
entity_id = excluded.entity_id,
|
|
273
|
+
title = excluded.title,
|
|
274
|
+
body = excluded.body,
|
|
275
|
+
tags = excluded.tags,
|
|
276
|
+
confidence = excluded.confidence,
|
|
277
|
+
source_type = excluded.source_type,
|
|
278
|
+
source_hash = excluded.source_hash,
|
|
279
|
+
source_ref = excluded.source_ref,
|
|
280
|
+
updated_at = excluded.updated_at,
|
|
281
|
+
last_accessed_at = excluded.last_accessed_at,
|
|
282
|
+
access_count = excluded.access_count,
|
|
283
|
+
deleted_at = excluded.deleted_at,
|
|
284
|
+
embedding_blob = CASE WHEN excluded.embedding_blob IS NULL THEN embedding_blob ELSE excluded.embedding_blob END,
|
|
285
|
+
embedding = NULL`,
|
|
286
|
+
[
|
|
287
|
+
fact.id,
|
|
288
|
+
fact.entity_id,
|
|
289
|
+
fact.title,
|
|
290
|
+
fact.body,
|
|
291
|
+
tagsJson,
|
|
292
|
+
fact.confidence,
|
|
293
|
+
fact.source_type,
|
|
294
|
+
fact.source_hash,
|
|
295
|
+
fact.source_ref,
|
|
296
|
+
fact.created_at,
|
|
297
|
+
now,
|
|
298
|
+
fact.last_accessed_at === null ? null : fact.last_accessed_at,
|
|
299
|
+
fact.access_count,
|
|
300
|
+
fact.deleted_at ?? null,
|
|
301
|
+
embeddingBlob ?? null,
|
|
302
|
+
null
|
|
303
|
+
]
|
|
304
|
+
);
|
|
305
|
+
await this.outbox.push({
|
|
306
|
+
entityId: fact.entity_id,
|
|
307
|
+
tableName: "entries",
|
|
308
|
+
recordId: fact.id,
|
|
309
|
+
operation,
|
|
310
|
+
payload: fact
|
|
311
|
+
}, tx);
|
|
312
|
+
return result;
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Normalize an embedding blob value to Uint8Array or null.
|
|
316
|
+
*/
|
|
317
|
+
normalizeEmbeddingBlob(blob) {
|
|
318
|
+
if (blob instanceof Uint8Array) return blob;
|
|
319
|
+
if (blob !== null && blob !== void 0 && typeof blob === "object") {
|
|
320
|
+
const obj = blob;
|
|
321
|
+
if (obj["type"] === "Buffer" && Array.isArray(obj["data"])) {
|
|
322
|
+
return new Uint8Array(obj["data"]);
|
|
323
|
+
}
|
|
324
|
+
const entries = Object.keys(obj);
|
|
325
|
+
if (entries.length > 0 && entries.every((k) => /^\d+$/.test(k))) {
|
|
326
|
+
const len = entries.length;
|
|
327
|
+
const arr = new Uint8Array(len);
|
|
328
|
+
for (let i = 0; i < len; i++) arr[i] = obj[String(i)] ?? 0;
|
|
329
|
+
return arr;
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
return null;
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Fetch existing rows by IDs and return id/entity_id/updated_at for import collision resolution.
|
|
336
|
+
*/
|
|
337
|
+
async findExistingMetadataByIds(ids, tx) {
|
|
338
|
+
const executor = this.getExecutor(tx);
|
|
339
|
+
const rows = [];
|
|
340
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
341
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
342
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
343
|
+
const chunkRows = await executor.getAllAsync(
|
|
344
|
+
`SELECT id, entity_id, updated_at FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
|
|
345
|
+
chunk
|
|
346
|
+
);
|
|
347
|
+
rows.push(...chunkRows.map((row) => ({ id: row.id, entity_id: row.entity_id, updated_at: Number(row.updated_at) })));
|
|
348
|
+
}
|
|
349
|
+
return rows;
|
|
350
|
+
}
|
|
351
|
+
async findIdById(id, entityId, tx) {
|
|
352
|
+
const executor = this.getExecutor(tx);
|
|
353
|
+
const row = await executor.getFirstAsync(
|
|
354
|
+
`SELECT id FROM ${this.prefix}entries WHERE id = ? AND entity_id = ?`,
|
|
355
|
+
[id, entityId]
|
|
356
|
+
);
|
|
357
|
+
return row?.id ?? null;
|
|
358
|
+
}
|
|
359
|
+
async findIdsBySource(entityId, sourceRef, sourceHash, tx, includeDeleted = false) {
|
|
360
|
+
const executor = this.getExecutor(tx);
|
|
361
|
+
let sql = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ?`;
|
|
362
|
+
const args = [entityId];
|
|
363
|
+
if (sourceRef !== null) {
|
|
364
|
+
sql += ` AND source_ref = ?`;
|
|
365
|
+
args.push(sourceRef);
|
|
366
|
+
}
|
|
367
|
+
if (sourceHash !== null) {
|
|
368
|
+
sql += ` AND source_hash = ?`;
|
|
369
|
+
args.push(sourceHash);
|
|
370
|
+
}
|
|
371
|
+
if (!includeDeleted) {
|
|
372
|
+
sql += ` AND deleted_at IS NULL`;
|
|
373
|
+
}
|
|
374
|
+
const rows = await executor.getAllAsync(sql, args);
|
|
375
|
+
return rows.map((row) => row.id);
|
|
376
|
+
}
|
|
377
|
+
async upsertForImport(fact, tx) {
|
|
378
|
+
const executor = this.getExecutor(tx);
|
|
379
|
+
const tagsJson = JSON.stringify(fact.tags);
|
|
380
|
+
const embeddingBlob = this.normalizeEmbeddingBlob(fact.embedding_blob);
|
|
381
|
+
const result = await executor.runAsync(
|
|
382
|
+
`INSERT INTO ${this.prefix}entries (
|
|
383
|
+
id, entity_id, title, body, tags, confidence, source_type,
|
|
384
|
+
source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count,
|
|
385
|
+
deleted_at, embedding_blob, embedding
|
|
386
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
387
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
388
|
+
entity_id = excluded.entity_id,
|
|
389
|
+
title = excluded.title,
|
|
390
|
+
body = excluded.body,
|
|
391
|
+
tags = excluded.tags,
|
|
392
|
+
confidence = excluded.confidence,
|
|
393
|
+
source_type = excluded.source_type,
|
|
394
|
+
source_hash = excluded.source_hash,
|
|
395
|
+
source_ref = excluded.source_ref,
|
|
396
|
+
created_at = excluded.created_at,
|
|
397
|
+
updated_at = excluded.updated_at,
|
|
398
|
+
last_accessed_at = excluded.last_accessed_at,
|
|
399
|
+
access_count = excluded.access_count,
|
|
400
|
+
deleted_at = excluded.deleted_at,
|
|
401
|
+
embedding_blob = excluded.embedding_blob,
|
|
402
|
+
embedding = NULL`,
|
|
403
|
+
[
|
|
404
|
+
fact.id,
|
|
405
|
+
fact.entity_id,
|
|
406
|
+
fact.title,
|
|
407
|
+
fact.body,
|
|
408
|
+
tagsJson,
|
|
409
|
+
fact.confidence,
|
|
410
|
+
fact.source_type,
|
|
411
|
+
fact.source_hash,
|
|
412
|
+
fact.source_ref,
|
|
413
|
+
fact.created_at,
|
|
414
|
+
fact.updated_at,
|
|
415
|
+
fact.last_accessed_at === null ? null : fact.last_accessed_at,
|
|
416
|
+
fact.access_count,
|
|
417
|
+
fact.deleted_at ?? null,
|
|
418
|
+
embeddingBlob ?? null,
|
|
419
|
+
null
|
|
420
|
+
]
|
|
421
|
+
);
|
|
422
|
+
return result;
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* Soft-delete a single entry by ID scoped to entityId. Sets deleted_at + updated_at.
|
|
426
|
+
* `tx` is REQUIRED to ensure atomic outbox staging.
|
|
427
|
+
*/
|
|
428
|
+
async softDelete(entryId, entityId, tx) {
|
|
429
|
+
const executor = this.getExecutor(tx);
|
|
430
|
+
const now = Date.now();
|
|
431
|
+
const result = await executor.runAsync(
|
|
432
|
+
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
433
|
+
[now, now, entryId, entityId]
|
|
434
|
+
);
|
|
435
|
+
await this.outbox.push({
|
|
436
|
+
entityId,
|
|
437
|
+
tableName: "entries",
|
|
438
|
+
recordId: entryId,
|
|
439
|
+
operation: "DELETE",
|
|
440
|
+
payload: { id: entryId, entity_id: entityId, deleted_at: now }
|
|
441
|
+
}, tx);
|
|
442
|
+
return result;
|
|
443
|
+
}
|
|
444
|
+
/**
|
|
445
|
+
* Soft-delete entries by source_ref and/or source_hash within a transaction.
|
|
446
|
+
* Stages a DELETE outbox entry for each row in the same transaction.
|
|
447
|
+
* `tx` is REQUIRED.
|
|
448
|
+
* Returns the number of rows deleted.
|
|
449
|
+
*/
|
|
450
|
+
async softDeleteBySource(entityId, tx, sourceRef, sourceHash) {
|
|
451
|
+
const executor = this.getExecutor(tx);
|
|
452
|
+
const now = Date.now();
|
|
453
|
+
let q = `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`;
|
|
454
|
+
const args = [now, now, entityId];
|
|
455
|
+
if (sourceRef) {
|
|
456
|
+
q += ` AND source_ref = ?`;
|
|
457
|
+
args.push(sourceRef);
|
|
458
|
+
}
|
|
459
|
+
if (sourceHash) {
|
|
460
|
+
q += ` AND source_hash = ?`;
|
|
461
|
+
args.push(sourceHash);
|
|
462
|
+
}
|
|
463
|
+
let selectQ = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`;
|
|
464
|
+
const selectArgs = [entityId];
|
|
465
|
+
if (sourceRef) {
|
|
466
|
+
selectQ += ` AND source_ref = ?`;
|
|
467
|
+
selectArgs.push(sourceRef);
|
|
468
|
+
}
|
|
469
|
+
if (sourceHash) {
|
|
470
|
+
selectQ += ` AND source_hash = ?`;
|
|
471
|
+
selectArgs.push(sourceHash);
|
|
472
|
+
}
|
|
473
|
+
const idsToDelete = await executor.getAllAsync(selectQ, selectArgs);
|
|
474
|
+
const result = await executor.runAsync(q, args);
|
|
475
|
+
for (const row of idsToDelete) {
|
|
476
|
+
await this.outbox.push({
|
|
477
|
+
entityId,
|
|
478
|
+
tableName: "entries",
|
|
479
|
+
recordId: row.id,
|
|
480
|
+
operation: "DELETE",
|
|
481
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: now }
|
|
482
|
+
}, tx);
|
|
483
|
+
}
|
|
484
|
+
return result.changes;
|
|
485
|
+
}
|
|
486
|
+
/**
|
|
487
|
+
* Fetch IDs + entity_ids of soft-deleted rows older than cutoff for a given entity.
|
|
488
|
+
* Used by runPrune().
|
|
489
|
+
*/
|
|
490
|
+
async getPrunableMetadata(entityId, cutoff, tx) {
|
|
491
|
+
const executor = this.getExecutor(tx);
|
|
492
|
+
return executor.getAllAsync(
|
|
493
|
+
`SELECT id, entity_id FROM ${this.prefix}entries
|
|
494
|
+
WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
|
|
495
|
+
[entityId, cutoff]
|
|
496
|
+
);
|
|
497
|
+
}
|
|
498
|
+
/**
|
|
499
|
+
* Fetch all non-deleted entries for an entity, ordered by updated_at DESC.
|
|
500
|
+
* Used by _getFullBundle().
|
|
501
|
+
*/
|
|
502
|
+
async findAllByEntityId(entityId, tx) {
|
|
503
|
+
const executor = this.getExecutor(tx);
|
|
504
|
+
const rows = await executor.getAllAsync(
|
|
505
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC`,
|
|
506
|
+
[entityId]
|
|
507
|
+
);
|
|
508
|
+
return rows.map(mapRowToFact);
|
|
509
|
+
}
|
|
510
|
+
/**
|
|
511
|
+
* Fetch recent non-deleted entries for an entity (limited), ordered by updated_at DESC.
|
|
512
|
+
* Used by _doRunLibrarian().
|
|
513
|
+
*/
|
|
514
|
+
async findRecentByEntityId(entityId, limit, tx) {
|
|
515
|
+
const executor = this.getExecutor(tx);
|
|
516
|
+
const rows = await executor.getAllAsync(
|
|
517
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC LIMIT ?`,
|
|
518
|
+
[entityId, limit]
|
|
519
|
+
);
|
|
520
|
+
return rows.map(mapRowToFact);
|
|
521
|
+
}
|
|
522
|
+
/**
|
|
523
|
+
* Count non-deleted entries for the given entities whose embedding_blob dimension
|
|
524
|
+
* doesn't match queryVecLength. Used by read() to detect model-switch mismatches.
|
|
525
|
+
*/
|
|
526
|
+
async countDimensionMismatched(entityIds, queryVecLength, tx) {
|
|
527
|
+
if (entityIds.length === 0) return 0;
|
|
528
|
+
const executor = this.getExecutor(tx);
|
|
529
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
530
|
+
const row = await executor.getFirstAsync(
|
|
531
|
+
`SELECT COUNT(*) AS cnt FROM ${this.prefix}entries
|
|
532
|
+
WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL
|
|
533
|
+
AND embedding_blob IS NOT NULL
|
|
534
|
+
AND (CAST(length(embedding_blob) AS INTEGER) % 4 = 0)
|
|
535
|
+
AND (CAST(length(embedding_blob) AS INTEGER) / 4) != ?`,
|
|
536
|
+
[...entityIds, queryVecLength]
|
|
537
|
+
);
|
|
538
|
+
return row?.cnt ?? 0;
|
|
539
|
+
}
|
|
540
|
+
/**
|
|
541
|
+
* Count non-deleted entries for entityId that are stale relative to targetDim
|
|
542
|
+
* (either no blob or wrong dimension). Used by runReembed() per-entity skip logic.
|
|
543
|
+
*/
|
|
544
|
+
async countStaleForEntity(entityId, targetDim, tx) {
|
|
545
|
+
const executor = this.getExecutor(tx);
|
|
546
|
+
const row = await executor.getFirstAsync(
|
|
547
|
+
`SELECT COUNT(*) AS cnt FROM ${this.prefix}entries
|
|
548
|
+
WHERE entity_id = ? AND deleted_at IS NULL
|
|
549
|
+
AND (
|
|
550
|
+
embedding_blob IS NULL
|
|
551
|
+
OR (CAST(length(embedding_blob) AS INTEGER) / 4) != ?
|
|
552
|
+
)`,
|
|
553
|
+
[entityId, targetDim]
|
|
554
|
+
);
|
|
555
|
+
return row?.cnt ?? 0;
|
|
556
|
+
}
|
|
557
|
+
/**
|
|
558
|
+
* Count non-deleted entries with stale or unconverted embeddings relative to `dim`.
|
|
559
|
+
* Used by _reconcileEmbeddingDimension() to decide when to promote the pending
|
|
560
|
+
* embedding_dimension value.
|
|
561
|
+
*/
|
|
562
|
+
async countStaleEmbeddings(dim, tx) {
|
|
563
|
+
const executor = this.getExecutor(tx);
|
|
564
|
+
const row = await executor.getFirstAsync(
|
|
565
|
+
`SELECT COUNT(*) AS cnt FROM ${this.prefix}entries
|
|
566
|
+
WHERE deleted_at IS NULL
|
|
567
|
+
AND (
|
|
568
|
+
(embedding_blob IS NOT NULL AND (CAST(length(embedding_blob) AS INTEGER) / 4) != ?)
|
|
569
|
+
OR (embedding_blob IS NULL AND embedding IS NOT NULL)
|
|
570
|
+
)`,
|
|
571
|
+
[dim]
|
|
572
|
+
);
|
|
573
|
+
return row?.cnt ?? 0;
|
|
574
|
+
}
|
|
575
|
+
/**
|
|
576
|
+
* Bulk delete pruned entries (already soft-deleted) by IDs.
|
|
577
|
+
* Used by runPrune(). Returns total number of deleted rows.
|
|
578
|
+
* `tx` is REQUIRED so outbox deletion events are staged atomically.
|
|
579
|
+
*/
|
|
580
|
+
async bulkDeletePruned(entityId, cutoff, ids, tx) {
|
|
581
|
+
const executor = this.getExecutor(tx);
|
|
582
|
+
let totalChanges = 0;
|
|
583
|
+
const chunkSize = 500;
|
|
584
|
+
for (let i = 0; i < ids.length; i += chunkSize) {
|
|
585
|
+
const chunk = ids.slice(i, i + chunkSize);
|
|
586
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
587
|
+
const result = await executor.runAsync(
|
|
588
|
+
`DELETE FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ? AND id IN (${placeholders})`,
|
|
589
|
+
[entityId, cutoff, ...chunk]
|
|
590
|
+
);
|
|
591
|
+
totalChanges += result.changes;
|
|
592
|
+
if (result.changes > 0) {
|
|
593
|
+
for (const id of chunk) {
|
|
594
|
+
await this.outbox.push({
|
|
595
|
+
entityId,
|
|
596
|
+
tableName: "entries",
|
|
597
|
+
recordId: id,
|
|
598
|
+
operation: "DELETE",
|
|
599
|
+
payload: { id, entity_id: entityId, deleted_at: cutoff }
|
|
600
|
+
}, tx);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
return totalChanges;
|
|
605
|
+
}
|
|
606
|
+
/**
|
|
607
|
+
* Mark orphaned entries (never accessed, old) as deleted.
|
|
608
|
+
* Used by _doRunHeal().
|
|
609
|
+
*/
|
|
610
|
+
async markOrphaned(entityId, orphanThreshold, tx) {
|
|
611
|
+
const executor = this.getExecutor(tx);
|
|
612
|
+
const now = Date.now();
|
|
613
|
+
const orphanedRows = await executor.getAllAsync(
|
|
614
|
+
`SELECT id FROM ${this.prefix}entries
|
|
615
|
+
WHERE entity_id = ? AND access_count = 0 AND created_at <= ? AND source_type != 'immutable_document' AND deleted_at IS NULL`,
|
|
616
|
+
[entityId, orphanThreshold]
|
|
617
|
+
);
|
|
618
|
+
if (orphanedRows.length === 0) return 0;
|
|
619
|
+
const result = await executor.runAsync(
|
|
620
|
+
`UPDATE ${this.prefix}entries
|
|
621
|
+
SET deleted_at = ?, updated_at = ?
|
|
622
|
+
WHERE entity_id = ? AND access_count = 0 AND created_at <= ? AND source_type != 'immutable_document' AND deleted_at IS NULL`,
|
|
623
|
+
[now, now, entityId, orphanThreshold]
|
|
624
|
+
);
|
|
625
|
+
for (const row of orphanedRows) {
|
|
626
|
+
await this.outbox.push({
|
|
627
|
+
entityId,
|
|
628
|
+
tableName: "entries",
|
|
629
|
+
recordId: row.id,
|
|
630
|
+
operation: "DELETE",
|
|
631
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: now }
|
|
632
|
+
}, tx);
|
|
633
|
+
}
|
|
634
|
+
return result.changes;
|
|
635
|
+
}
|
|
636
|
+
/**
|
|
637
|
+
* Downgrade stale inferred entries to 'tentative'.
|
|
638
|
+
* Used by _doRunHeal().
|
|
639
|
+
*/
|
|
640
|
+
async downgradeStaleInferred(entityId, staleThreshold, tx) {
|
|
641
|
+
const executor = this.getExecutor(tx);
|
|
642
|
+
const now = Date.now();
|
|
643
|
+
const eligibleRows = await executor.getAllAsync(
|
|
644
|
+
`SELECT id FROM ${this.prefix}entries
|
|
645
|
+
WHERE entity_id = ? AND confidence = 'inferred'
|
|
646
|
+
AND (last_accessed_at <= ? OR (last_accessed_at IS NULL AND created_at <= ?))
|
|
647
|
+
AND source_type != 'immutable_document' AND deleted_at IS NULL`,
|
|
648
|
+
[entityId, staleThreshold, staleThreshold]
|
|
649
|
+
);
|
|
650
|
+
if (eligibleRows.length === 0) return 0;
|
|
651
|
+
const result = await executor.runAsync(
|
|
652
|
+
`UPDATE ${this.prefix}entries
|
|
653
|
+
SET confidence = 'tentative', updated_at = ?
|
|
654
|
+
WHERE entity_id = ? AND confidence = 'inferred' AND (last_accessed_at <= ? OR (last_accessed_at IS NULL AND created_at <= ?)) AND source_type != 'immutable_document' AND deleted_at IS NULL`,
|
|
655
|
+
[now, entityId, staleThreshold, staleThreshold]
|
|
656
|
+
);
|
|
657
|
+
for (const row of eligibleRows) {
|
|
658
|
+
await this.outbox.push({
|
|
659
|
+
entityId,
|
|
660
|
+
tableName: "entries",
|
|
661
|
+
recordId: row.id,
|
|
662
|
+
operation: "UPDATE",
|
|
663
|
+
payload: { id: row.id, entity_id: entityId, confidence: "tentative", updated_at: now }
|
|
664
|
+
}, tx);
|
|
665
|
+
}
|
|
666
|
+
return result.changes;
|
|
667
|
+
}
|
|
668
|
+
/**
|
|
669
|
+
* Downgrade specific entries to 'tentative' by IDs.
|
|
670
|
+
* Used by _doRunHeal().
|
|
671
|
+
*/
|
|
672
|
+
async downgradeByIds(ids, entityId, tx) {
|
|
673
|
+
if (ids.length === 0) return;
|
|
674
|
+
const executor = this.getExecutor(tx);
|
|
675
|
+
const now = Date.now();
|
|
676
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
677
|
+
await executor.runAsync(
|
|
678
|
+
`UPDATE ${this.prefix}entries SET confidence = 'tentative', updated_at = ? WHERE id IN (${placeholders}) AND entity_id = ?`,
|
|
679
|
+
[now, ...ids, entityId]
|
|
680
|
+
);
|
|
681
|
+
for (const id of ids) {
|
|
682
|
+
await this.outbox.push({
|
|
683
|
+
entityId,
|
|
684
|
+
tableName: "entries",
|
|
685
|
+
recordId: id,
|
|
686
|
+
operation: "UPDATE",
|
|
687
|
+
payload: { id, entity_id: entityId, confidence: "tentative", updated_at: now }
|
|
688
|
+
}, tx);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
/**
|
|
692
|
+
* Soft-delete specific entries by IDs.
|
|
693
|
+
* Used by _doRunHeal().
|
|
694
|
+
*/
|
|
695
|
+
async softDeleteByIds(ids, entityId, tx) {
|
|
696
|
+
if (ids.length === 0) return;
|
|
697
|
+
const executor = this.getExecutor(tx);
|
|
698
|
+
const now = Date.now();
|
|
699
|
+
const placeholders = ids.map(() => "?").join(",");
|
|
700
|
+
await executor.runAsync(
|
|
701
|
+
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id IN (${placeholders}) AND entity_id = ?`,
|
|
702
|
+
[now, now, ...ids, entityId]
|
|
703
|
+
);
|
|
704
|
+
for (const id of ids) {
|
|
705
|
+
await this.outbox.push({
|
|
706
|
+
entityId,
|
|
707
|
+
tableName: "entries",
|
|
708
|
+
recordId: id,
|
|
709
|
+
operation: "DELETE",
|
|
710
|
+
payload: { id, entity_id: entityId, deleted_at: now }
|
|
711
|
+
}, tx);
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
/**
|
|
715
|
+
* Bulk soft-delete all entries for an entity.
|
|
716
|
+
* Stages DELETE outbox entries for each row in the same transaction.
|
|
717
|
+
* `tx` is REQUIRED.
|
|
718
|
+
*/
|
|
719
|
+
async bulkSoftDeleteByEntityId(entityId, tx) {
|
|
720
|
+
const executor = this.getExecutor(tx);
|
|
721
|
+
const now = Date.now();
|
|
722
|
+
const idsToDelete = await executor.getAllAsync(
|
|
723
|
+
`SELECT id FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
724
|
+
[entityId]
|
|
725
|
+
);
|
|
726
|
+
const result = await executor.runAsync(
|
|
727
|
+
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
728
|
+
[now, now, entityId]
|
|
729
|
+
);
|
|
730
|
+
for (const row of idsToDelete) {
|
|
731
|
+
await this.outbox.push({
|
|
732
|
+
entityId,
|
|
733
|
+
tableName: "entries",
|
|
734
|
+
recordId: row.id,
|
|
735
|
+
operation: "DELETE",
|
|
736
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: now }
|
|
737
|
+
}, tx);
|
|
738
|
+
}
|
|
739
|
+
return result.changes;
|
|
740
|
+
}
|
|
741
|
+
async findMiniSearchRows(entityId, tx) {
|
|
742
|
+
const executor = this.getExecutor(tx);
|
|
743
|
+
if (entityId !== void 0) {
|
|
744
|
+
return executor.getAllAsync(
|
|
745
|
+
`SELECT id, entity_id, title, body, tags FROM ${this.prefix}entries WHERE deleted_at IS NULL AND entity_id = ?`,
|
|
746
|
+
[entityId]
|
|
747
|
+
);
|
|
748
|
+
}
|
|
749
|
+
return executor.getAllAsync(
|
|
750
|
+
`SELECT id, entity_id, title, body, tags FROM ${this.prefix}entries WHERE deleted_at IS NULL`
|
|
751
|
+
);
|
|
752
|
+
}
|
|
753
|
+
async updateEmbeddingBlob(id, blob, tx) {
|
|
754
|
+
const executor = this.getExecutor(tx);
|
|
755
|
+
await executor.runAsync(
|
|
756
|
+
`UPDATE ${this.prefix}entries SET embedding_blob = ?, embedding = NULL WHERE id = ?`,
|
|
757
|
+
[blob, id]
|
|
758
|
+
);
|
|
759
|
+
}
|
|
760
|
+
async hasLegacySourceTypes(tx) {
|
|
761
|
+
const executor = this.getExecutor(tx);
|
|
762
|
+
const row = await executor.getFirstAsync(
|
|
763
|
+
`SELECT 1 AS one FROM ${this.prefix}entries WHERE source_type IN ('user_document', 'agent_inferred') LIMIT 1`,
|
|
764
|
+
[]
|
|
765
|
+
);
|
|
766
|
+
return row != null;
|
|
767
|
+
}
|
|
768
|
+
async countLegacySourceTypes(tx) {
|
|
769
|
+
const executor = this.getExecutor(tx);
|
|
770
|
+
const row = await executor.getFirstAsync(
|
|
771
|
+
`SELECT COUNT(*) as count FROM ${this.prefix}entries WHERE source_type IN ('user_document', 'agent_inferred')`,
|
|
772
|
+
[]
|
|
773
|
+
);
|
|
774
|
+
return row?.count ?? 0;
|
|
775
|
+
}
|
|
776
|
+
async findAllForReembed(entityId, tx) {
|
|
777
|
+
const executor = this.getExecutor(tx);
|
|
778
|
+
if (entityId !== void 0) {
|
|
779
|
+
return executor.getAllAsync(
|
|
780
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
781
|
+
[entityId]
|
|
782
|
+
);
|
|
783
|
+
}
|
|
784
|
+
return executor.getAllAsync(
|
|
785
|
+
`SELECT * FROM ${this.prefix}entries WHERE deleted_at IS NULL`
|
|
786
|
+
);
|
|
787
|
+
}
|
|
788
|
+
async findRowsForSourceRefMigration(tx) {
|
|
789
|
+
const executor = this.getExecutor(tx);
|
|
790
|
+
return executor.getAllAsync(
|
|
791
|
+
`SELECT rowid, source_ref FROM ${this.prefix}entries
|
|
792
|
+
WHERE source_ref IS NOT NULL
|
|
793
|
+
AND (
|
|
794
|
+
TRIM(source_ref) != source_ref
|
|
795
|
+
OR INSTR(source_ref, '/') > 0
|
|
796
|
+
OR INSTR(source_ref, '\\') > 0
|
|
797
|
+
OR INSTR(source_ref, CHAR(0)) > 0
|
|
798
|
+
OR source_ref GLOB '*[^-A-Za-z0-9._ ]*'
|
|
799
|
+
)`
|
|
800
|
+
);
|
|
801
|
+
}
|
|
802
|
+
async updateSourceRefByRowid(rowid, sourceRef, tx) {
|
|
803
|
+
const executor = this.getExecutor(tx);
|
|
804
|
+
await executor.runAsync(
|
|
805
|
+
`UPDATE ${this.prefix}entries SET source_ref = ? WHERE rowid = ?`,
|
|
806
|
+
[sourceRef, rowid]
|
|
807
|
+
);
|
|
808
|
+
}
|
|
809
|
+
async findLatestSourceHash(entityId, sourceRef, tx) {
|
|
810
|
+
const executor = this.getExecutor(tx);
|
|
811
|
+
const row = await executor.getFirstAsync(
|
|
812
|
+
`SELECT source_hash FROM ${this.prefix}entries
|
|
813
|
+
WHERE entity_id = ? AND source_ref = ? AND deleted_at IS NULL
|
|
814
|
+
ORDER BY updated_at DESC
|
|
815
|
+
LIMIT 1`,
|
|
816
|
+
[entityId, sourceRef]
|
|
817
|
+
);
|
|
818
|
+
return row?.source_hash ?? null;
|
|
819
|
+
}
|
|
820
|
+
async findMetadataByIds(ids, tx) {
|
|
821
|
+
if (ids.length === 0) return [];
|
|
822
|
+
const executor = this.getExecutor(tx);
|
|
823
|
+
const rows = [];
|
|
824
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
825
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
826
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
827
|
+
const chunkRows = await executor.getAllAsync(
|
|
828
|
+
`SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
829
|
+
chunk
|
|
830
|
+
);
|
|
831
|
+
rows.push(...chunkRows);
|
|
832
|
+
}
|
|
833
|
+
return rows;
|
|
834
|
+
}
|
|
835
|
+
async findWithEmbeddingsByIds(ids, tx) {
|
|
836
|
+
if (ids.length === 0) return [];
|
|
837
|
+
const executor = this.getExecutor(tx);
|
|
838
|
+
const rows = [];
|
|
839
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
840
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
841
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
842
|
+
const chunkRows = await executor.getAllAsync(
|
|
843
|
+
`SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
844
|
+
chunk
|
|
845
|
+
);
|
|
846
|
+
rows.push(...chunkRows);
|
|
847
|
+
}
|
|
848
|
+
return rows;
|
|
849
|
+
}
|
|
850
|
+
async findMetadataByEntityIds(entityIds, tx) {
|
|
851
|
+
if (entityIds.length === 0) return [];
|
|
852
|
+
const executor = this.getExecutor(tx);
|
|
853
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
854
|
+
return executor.getAllAsync(
|
|
855
|
+
`SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
856
|
+
[...entityIds]
|
|
857
|
+
);
|
|
858
|
+
}
|
|
859
|
+
async findWithEmbeddingsByEntityIds(entityIds, tx) {
|
|
860
|
+
if (entityIds.length === 0) return [];
|
|
861
|
+
const executor = this.getExecutor(tx);
|
|
862
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
863
|
+
return executor.getAllAsync(
|
|
864
|
+
`SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
865
|
+
[...entityIds]
|
|
866
|
+
);
|
|
867
|
+
}
|
|
868
|
+
async findEmbeddingsByIds(ids, tx) {
|
|
869
|
+
if (ids.length === 0) return [];
|
|
870
|
+
const executor = this.getExecutor(tx);
|
|
871
|
+
const rows = [];
|
|
872
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
873
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
874
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
875
|
+
const chunkRows = await executor.getAllAsync(
|
|
876
|
+
`SELECT id, embedding_blob, embedding FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
877
|
+
chunk
|
|
878
|
+
);
|
|
879
|
+
rows.push(...chunkRows);
|
|
880
|
+
}
|
|
881
|
+
return rows;
|
|
882
|
+
}
|
|
883
|
+
async trackAccess(ids, now, tx) {
|
|
884
|
+
if (ids.length === 0) return;
|
|
885
|
+
const executor = this.getExecutor(tx);
|
|
886
|
+
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
887
|
+
const chunk = ids.slice(i, i + this.chunkSize);
|
|
888
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
889
|
+
await executor.runAsync(
|
|
890
|
+
`UPDATE ${this.prefix}entries SET access_count = access_count + 1, last_accessed_at = ? WHERE id IN (${placeholders})`,
|
|
891
|
+
[now, ...chunk]
|
|
892
|
+
);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
getLegacyMigrationSQL() {
|
|
896
|
+
return [
|
|
897
|
+
`-- Migrate legacy source_type values (targets your WikiMemory prefix: ${this.prefix})`,
|
|
898
|
+
`UPDATE ${this.prefix}entries SET source_type = 'immutable_document' WHERE source_type = 'user_document';`,
|
|
899
|
+
`UPDATE ${this.prefix}entries SET source_type = 'librarian_inferred' WHERE source_type = 'agent_inferred';`
|
|
900
|
+
].join("\n");
|
|
901
|
+
}
|
|
902
|
+
async findRecentByEntityIds(entityIds, limit, tx) {
|
|
903
|
+
if (entityIds.length === 0) return [];
|
|
904
|
+
const executor = this.getExecutor(tx);
|
|
905
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
906
|
+
const rows = await executor.getAllAsync(
|
|
907
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL ORDER BY updated_at DESC LIMIT ?`,
|
|
908
|
+
[...entityIds, limit]
|
|
909
|
+
);
|
|
910
|
+
return rows.map(mapRowToFact);
|
|
911
|
+
}
|
|
912
|
+
};
|
|
913
|
+
|
|
914
|
+
// src/utils/ids.ts
|
|
915
|
+
function generateId(prefix = "") {
|
|
916
|
+
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
|
|
917
|
+
return prefix + crypto.randomUUID().replace(/-/g, "").substring(0, 24);
|
|
918
|
+
}
|
|
919
|
+
if (typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function") {
|
|
920
|
+
const bytes = new Uint8Array(16);
|
|
921
|
+
crypto.getRandomValues(bytes);
|
|
922
|
+
return prefix + Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("").substring(0, 24);
|
|
923
|
+
}
|
|
924
|
+
return prefix + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
// src/repositories/OutboxRepository.ts
|
|
928
|
+
var OutboxRepository = class extends BaseRepository {
|
|
929
|
+
/**
|
|
930
|
+
* Insert a new outbox event within the provided transaction.
|
|
931
|
+
* `tx` is required — callers must always pass the active transaction
|
|
932
|
+
* so the write is atomic with the main table mutation.
|
|
933
|
+
*/
|
|
934
|
+
async push(params, tx) {
|
|
935
|
+
const executor = this.getExecutor(tx);
|
|
936
|
+
const id = generateId("out_");
|
|
937
|
+
const now = Date.now();
|
|
938
|
+
await executor.runAsync(
|
|
939
|
+
`INSERT INTO ${this.prefix}outbox (id, entity_id, table_name, record_id, operation, payload, created_at)
|
|
940
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
941
|
+
[id, params.entityId, params.tableName, params.recordId, params.operation, JSON.stringify(params.payload), now]
|
|
942
|
+
);
|
|
943
|
+
}
|
|
944
|
+
/**
|
|
945
|
+
* Fetch pending outbox rows ordered by created_at ASC.
|
|
946
|
+
* Reads directly from `this.db` (not a transaction).
|
|
947
|
+
*/
|
|
948
|
+
async fetchPending(limit = 50) {
|
|
949
|
+
return this.db.getAllAsync(
|
|
950
|
+
`SELECT * FROM ${this.prefix}outbox ORDER BY created_at ASC LIMIT ?`,
|
|
951
|
+
[limit]
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
/**
|
|
955
|
+
* Delete acknowledged outbox rows by their IDs.
|
|
956
|
+
* No-op when `ids` is empty.
|
|
957
|
+
* Deletes directly from `this.db` (not a transaction).
|
|
958
|
+
*/
|
|
959
|
+
async acknowledge(ids) {
|
|
960
|
+
if (ids.length === 0) return;
|
|
961
|
+
const placeholders = ids.map(() => "?").join(", ");
|
|
962
|
+
await this.db.runAsync(
|
|
963
|
+
`DELETE FROM ${this.prefix}outbox WHERE id IN (${placeholders})`,
|
|
964
|
+
ids
|
|
965
|
+
);
|
|
966
|
+
}
|
|
967
|
+
};
|
|
968
|
+
|
|
969
|
+
// src/repositories/TaskRepository.ts
|
|
970
|
+
function mapRowToTask(row) {
|
|
971
|
+
return {
|
|
972
|
+
id: row.id,
|
|
973
|
+
entity_id: row.entity_id,
|
|
974
|
+
description: row.description,
|
|
975
|
+
status: row.status,
|
|
976
|
+
priority: Number(row.priority),
|
|
977
|
+
created_at: Number(row.created_at),
|
|
978
|
+
updated_at: Number(row.updated_at),
|
|
979
|
+
resolved_at: row.resolved_at != null ? Number(row.resolved_at) : null,
|
|
980
|
+
deleted_at: row.deleted_at != null ? Number(row.deleted_at) : null
|
|
981
|
+
};
|
|
982
|
+
}
|
|
983
|
+
var TaskRepository = class extends BaseRepository {
|
|
984
|
+
constructor(db, prefix, outbox) {
|
|
985
|
+
super(db, prefix);
|
|
986
|
+
this.outbox = outbox;
|
|
987
|
+
}
|
|
988
|
+
/**
|
|
989
|
+
* Fetch a single task by ID. Returns null if not found or soft-deleted.
|
|
990
|
+
*/
|
|
991
|
+
async findById(id) {
|
|
992
|
+
const row = await this.db.getFirstAsync(
|
|
993
|
+
`SELECT * FROM ${this.prefix}tasks WHERE id = ? AND deleted_at IS NULL`,
|
|
994
|
+
[id]
|
|
995
|
+
);
|
|
996
|
+
return row ? mapRowToTask(row) : null;
|
|
997
|
+
}
|
|
998
|
+
/**
|
|
999
|
+
* Fetch all pending/in_progress tasks for the given entity IDs.
|
|
1000
|
+
* Returns empty array when entityIds is empty.
|
|
1001
|
+
*/
|
|
1002
|
+
async findAllPending(entityIds, limit) {
|
|
1003
|
+
if (entityIds.length === 0) return [];
|
|
1004
|
+
const placeholders = entityIds.map(() => "?").join(", ");
|
|
1005
|
+
const sql = `SELECT * FROM ${this.prefix}tasks WHERE entity_id IN (${placeholders}) AND status IN ('pending', 'in_progress') AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC` + (limit != null ? ` LIMIT ?` : "");
|
|
1006
|
+
const params = limit != null ? [...entityIds, limit] : [...entityIds];
|
|
1007
|
+
const rows = await this.db.getAllAsync(sql, params);
|
|
1008
|
+
return rows.map(mapRowToTask);
|
|
1009
|
+
}
|
|
1010
|
+
async findExistingMetadataByIds(ids, tx) {
|
|
1011
|
+
const executor = this.getExecutor(tx);
|
|
1012
|
+
const rows = [];
|
|
1013
|
+
const chunkSize = 500;
|
|
1014
|
+
for (let i = 0; i < ids.length; i += chunkSize) {
|
|
1015
|
+
const chunk = ids.slice(i, i + chunkSize);
|
|
1016
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
1017
|
+
const chunkRows = await executor.getAllAsync(
|
|
1018
|
+
`SELECT id, entity_id, updated_at FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
|
|
1019
|
+
chunk
|
|
1020
|
+
);
|
|
1021
|
+
rows.push(...chunkRows.map((row) => ({ id: row.id, entity_id: row.entity_id, updated_at: Number(row.updated_at) })));
|
|
1022
|
+
}
|
|
1023
|
+
return rows;
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* Upsert a WikiTask within the provided transaction.
|
|
1027
|
+
* Uses ON CONFLICT(id) DO UPDATE (not INSERT OR REPLACE).
|
|
1028
|
+
* Stages an outbox entry in the same transaction.
|
|
1029
|
+
* `tx` is REQUIRED.
|
|
1030
|
+
*/
|
|
1031
|
+
async upsert(task, tx, updatedAt) {
|
|
1032
|
+
const executor = this.getExecutor(tx);
|
|
1033
|
+
const now = Number.isFinite(updatedAt) ? updatedAt : Date.now();
|
|
1034
|
+
const existingRow = await executor.getFirstAsync(
|
|
1035
|
+
`SELECT id FROM ${this.prefix}tasks WHERE id = ?`,
|
|
1036
|
+
[task.id]
|
|
1037
|
+
);
|
|
1038
|
+
const operation = task.deleted_at != null ? "DELETE" : existingRow ? "UPDATE" : "INSERT";
|
|
1039
|
+
await executor.runAsync(
|
|
1040
|
+
`INSERT INTO ${this.prefix}tasks (
|
|
1041
|
+
id, entity_id, description, status, priority,
|
|
1042
|
+
created_at, updated_at, resolved_at, deleted_at
|
|
1043
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1044
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1045
|
+
entity_id = excluded.entity_id,
|
|
1046
|
+
description = excluded.description,
|
|
1047
|
+
status = excluded.status,
|
|
1048
|
+
priority = excluded.priority,
|
|
1049
|
+
updated_at = excluded.updated_at,
|
|
1050
|
+
resolved_at = excluded.resolved_at,
|
|
1051
|
+
deleted_at = excluded.deleted_at`,
|
|
1052
|
+
[
|
|
1053
|
+
task.id,
|
|
1054
|
+
task.entity_id,
|
|
1055
|
+
task.description,
|
|
1056
|
+
task.status,
|
|
1057
|
+
task.priority,
|
|
1058
|
+
task.created_at,
|
|
1059
|
+
now,
|
|
1060
|
+
// updated_at set by repo or import override
|
|
1061
|
+
task.resolved_at ?? null,
|
|
1062
|
+
task.deleted_at ?? null
|
|
1063
|
+
]
|
|
1064
|
+
);
|
|
1065
|
+
await this.outbox.push(
|
|
1066
|
+
{
|
|
1067
|
+
entityId: task.entity_id,
|
|
1068
|
+
tableName: "tasks",
|
|
1069
|
+
recordId: task.id,
|
|
1070
|
+
operation,
|
|
1071
|
+
payload: task
|
|
1072
|
+
},
|
|
1073
|
+
tx
|
|
1074
|
+
);
|
|
1075
|
+
}
|
|
1076
|
+
async upsertForImport(task, tx, updatedAt) {
|
|
1077
|
+
const executor = this.getExecutor(tx);
|
|
1078
|
+
const now = Number.isFinite(updatedAt) ? updatedAt : Date.now();
|
|
1079
|
+
await executor.runAsync(
|
|
1080
|
+
`INSERT INTO ${this.prefix}tasks (
|
|
1081
|
+
id, entity_id, description, status, priority,
|
|
1082
|
+
created_at, updated_at, resolved_at, deleted_at
|
|
1083
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1084
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
1085
|
+
entity_id = excluded.entity_id,
|
|
1086
|
+
description = excluded.description,
|
|
1087
|
+
status = excluded.status,
|
|
1088
|
+
priority = excluded.priority,
|
|
1089
|
+
updated_at = excluded.updated_at,
|
|
1090
|
+
resolved_at = excluded.resolved_at,
|
|
1091
|
+
deleted_at = excluded.deleted_at`,
|
|
1092
|
+
[
|
|
1093
|
+
task.id,
|
|
1094
|
+
task.entity_id,
|
|
1095
|
+
task.description,
|
|
1096
|
+
task.status,
|
|
1097
|
+
task.priority,
|
|
1098
|
+
task.created_at,
|
|
1099
|
+
now,
|
|
1100
|
+
task.resolved_at ?? null,
|
|
1101
|
+
task.deleted_at ?? null
|
|
1102
|
+
]
|
|
1103
|
+
);
|
|
1104
|
+
}
|
|
1105
|
+
/**
|
|
1106
|
+
* Soft-delete a task by ID. Sets deleted_at and updated_at.
|
|
1107
|
+
* Stages a DELETE outbox entry in the same transaction.
|
|
1108
|
+
* `tx` is REQUIRED.
|
|
1109
|
+
*/
|
|
1110
|
+
async softDelete(id, entityId, tx) {
|
|
1111
|
+
const executor = this.getExecutor(tx);
|
|
1112
|
+
const now = Date.now();
|
|
1113
|
+
await executor.runAsync(
|
|
1114
|
+
`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
1115
|
+
[now, now, id, entityId]
|
|
1116
|
+
);
|
|
1117
|
+
await this.outbox.push(
|
|
1118
|
+
{
|
|
1119
|
+
entityId,
|
|
1120
|
+
tableName: "tasks",
|
|
1121
|
+
recordId: id,
|
|
1122
|
+
operation: "DELETE",
|
|
1123
|
+
payload: { id, entity_id: entityId, deleted_at: now }
|
|
1124
|
+
},
|
|
1125
|
+
tx
|
|
1126
|
+
);
|
|
1127
|
+
}
|
|
1128
|
+
/**
|
|
1129
|
+
* Fetch all non-deleted tasks for an entity, ordered by priority DESC, created_at ASC.
|
|
1130
|
+
* Used by _getFullBundle().
|
|
1131
|
+
*/
|
|
1132
|
+
async findAllByEntityId(entityId, tx) {
|
|
1133
|
+
const executor = this.getExecutor(tx);
|
|
1134
|
+
const rows = await executor.getAllAsync(
|
|
1135
|
+
`SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC`,
|
|
1136
|
+
[entityId]
|
|
1137
|
+
);
|
|
1138
|
+
return rows.map(mapRowToTask);
|
|
1139
|
+
}
|
|
1140
|
+
/**
|
|
1141
|
+
* Bulk delete pruned tasks (already soft-deleted) by cutoff date.
|
|
1142
|
+
* Used by runPrune(). Returns number of deleted rows.
|
|
1143
|
+
*/
|
|
1144
|
+
async bulkDeletePruned(entityId, cutoff, tx) {
|
|
1145
|
+
const executor = this.getExecutor(tx);
|
|
1146
|
+
const rowsToDelete = await executor.getAllAsync(
|
|
1147
|
+
`SELECT id, deleted_at FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
|
|
1148
|
+
[entityId, cutoff]
|
|
1149
|
+
);
|
|
1150
|
+
if (rowsToDelete.length === 0) return 0;
|
|
1151
|
+
const result = await executor.runAsync(
|
|
1152
|
+
`DELETE FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
|
|
1153
|
+
[entityId, cutoff]
|
|
1154
|
+
);
|
|
1155
|
+
for (const row of rowsToDelete) {
|
|
1156
|
+
await this.outbox.push(
|
|
1157
|
+
{
|
|
1158
|
+
entityId,
|
|
1159
|
+
tableName: "tasks",
|
|
1160
|
+
recordId: row.id,
|
|
1161
|
+
operation: "DELETE",
|
|
1162
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: row.deleted_at }
|
|
1163
|
+
},
|
|
1164
|
+
tx
|
|
1165
|
+
);
|
|
1166
|
+
}
|
|
1167
|
+
return result.changes;
|
|
1168
|
+
}
|
|
1169
|
+
/**
|
|
1170
|
+
* Soft-delete a task by ID within a transaction.
|
|
1171
|
+
* Stages a DELETE outbox entry in the same transaction.
|
|
1172
|
+
* `tx` is REQUIRED.
|
|
1173
|
+
*/
|
|
1174
|
+
async softDeleteById(id, entityId, tx) {
|
|
1175
|
+
const executor = this.getExecutor(tx);
|
|
1176
|
+
const now = Date.now();
|
|
1177
|
+
const result = await executor.runAsync(
|
|
1178
|
+
`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
1179
|
+
[now, now, id, entityId]
|
|
1180
|
+
);
|
|
1181
|
+
if (result.changes > 0) {
|
|
1182
|
+
await this.outbox.push(
|
|
1183
|
+
{
|
|
1184
|
+
entityId,
|
|
1185
|
+
tableName: "tasks",
|
|
1186
|
+
recordId: id,
|
|
1187
|
+
operation: "DELETE",
|
|
1188
|
+
payload: { id, entity_id: entityId, deleted_at: now }
|
|
1189
|
+
},
|
|
1190
|
+
tx
|
|
1191
|
+
);
|
|
1192
|
+
}
|
|
1193
|
+
return result;
|
|
1194
|
+
}
|
|
1195
|
+
/**
|
|
1196
|
+
* Bulk soft-delete all tasks for an entity.
|
|
1197
|
+
* Stages DELETE outbox entries for each row in the same transaction.
|
|
1198
|
+
* `tx` is REQUIRED.
|
|
1199
|
+
*/
|
|
1200
|
+
async bulkSoftDeleteByEntityId(entityId, tx) {
|
|
1201
|
+
const executor = this.getExecutor(tx);
|
|
1202
|
+
const now = Date.now();
|
|
1203
|
+
const idsToDelete = await executor.getAllAsync(
|
|
1204
|
+
`SELECT id FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
1205
|
+
[entityId]
|
|
1206
|
+
);
|
|
1207
|
+
const result = await executor.runAsync(
|
|
1208
|
+
`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
1209
|
+
[now, now, entityId]
|
|
1210
|
+
);
|
|
1211
|
+
for (const row of idsToDelete) {
|
|
1212
|
+
await this.outbox.push({
|
|
1213
|
+
entityId,
|
|
1214
|
+
tableName: "tasks",
|
|
1215
|
+
recordId: row.id,
|
|
1216
|
+
operation: "DELETE",
|
|
1217
|
+
payload: { id: row.id, entity_id: entityId, deleted_at: now }
|
|
1218
|
+
}, tx);
|
|
1219
|
+
}
|
|
1220
|
+
return result.changes;
|
|
216
1221
|
}
|
|
1222
|
+
};
|
|
1223
|
+
|
|
1224
|
+
// src/repositories/EventRepository.ts
|
|
1225
|
+
var EventRepository = class extends BaseRepository {
|
|
217
1226
|
/**
|
|
218
|
-
*
|
|
219
|
-
*
|
|
1227
|
+
* Insert a new event row.
|
|
1228
|
+
* Pass `tx` to participate in a caller-owned transaction; omit to run against the default db.
|
|
220
1229
|
*/
|
|
221
|
-
async
|
|
1230
|
+
async add(event, tx) {
|
|
222
1231
|
const executor = this.getExecutor(tx);
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
const obj = fact.embedding_blob;
|
|
227
|
-
const keys = Object.keys(obj).map(Number).sort((a, b) => a - b);
|
|
228
|
-
const arr = new Uint8Array(keys.length);
|
|
229
|
-
for (let i = 0; i < keys.length; i++) arr[i] = obj[String(keys[i])];
|
|
230
|
-
return arr;
|
|
231
|
-
})() : void 0;
|
|
232
|
-
return executor.runAsync(
|
|
233
|
-
`INSERT INTO ${this.prefix}entries (
|
|
234
|
-
id, entity_id, title, body, tags, confidence, source_type,
|
|
235
|
-
source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count,
|
|
236
|
-
deleted_at, embedding_blob, embedding
|
|
237
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
238
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
239
|
-
entity_id = excluded.entity_id,
|
|
240
|
-
title = excluded.title,
|
|
241
|
-
body = excluded.body,
|
|
242
|
-
tags = excluded.tags,
|
|
243
|
-
confidence = excluded.confidence,
|
|
244
|
-
source_type = excluded.source_type,
|
|
245
|
-
source_hash = excluded.source_hash,
|
|
246
|
-
source_ref = excluded.source_ref,
|
|
247
|
-
updated_at = excluded.updated_at,
|
|
248
|
-
last_accessed_at = excluded.last_accessed_at,
|
|
249
|
-
access_count = excluded.access_count,
|
|
250
|
-
deleted_at = excluded.deleted_at,
|
|
251
|
-
embedding_blob = CASE WHEN excluded.embedding_blob IS NULL THEN embedding_blob ELSE excluded.embedding_blob END,
|
|
252
|
-
embedding = NULL`,
|
|
1232
|
+
await executor.runAsync(
|
|
1233
|
+
`INSERT INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
|
|
1234
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
253
1235
|
[
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
1236
|
+
event.id,
|
|
1237
|
+
event.entity_id,
|
|
1238
|
+
event.event_type,
|
|
1239
|
+
event.summary,
|
|
1240
|
+
event.related_entry_id ?? null,
|
|
1241
|
+
event.created_at
|
|
1242
|
+
]
|
|
1243
|
+
);
|
|
1244
|
+
}
|
|
1245
|
+
async addIgnoreDuplicate(event, tx) {
|
|
1246
|
+
const executor = this.getExecutor(tx);
|
|
1247
|
+
await executor.runAsync(
|
|
1248
|
+
`INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
|
|
1249
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
1250
|
+
[
|
|
1251
|
+
event.id,
|
|
1252
|
+
event.entity_id,
|
|
1253
|
+
event.event_type,
|
|
1254
|
+
event.summary,
|
|
1255
|
+
event.related_entry_id ?? null,
|
|
1256
|
+
event.created_at
|
|
271
1257
|
]
|
|
272
1258
|
);
|
|
273
1259
|
}
|
|
274
1260
|
/**
|
|
275
|
-
*
|
|
1261
|
+
* Return the most recent events for an entity, newest first.
|
|
1262
|
+
* Defaults to a limit of 50.
|
|
276
1263
|
*/
|
|
277
|
-
async
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
282
|
-
[now, now, entryId, entityId]
|
|
1264
|
+
async getRecent(entityId, limit = 50) {
|
|
1265
|
+
return this.db.getAllAsync(
|
|
1266
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?`,
|
|
1267
|
+
[entityId, limit]
|
|
283
1268
|
);
|
|
284
1269
|
}
|
|
285
1270
|
/**
|
|
286
|
-
*
|
|
287
|
-
*
|
|
1271
|
+
* Return the most recent events for the given entity IDs, newest first.
|
|
1272
|
+
* Defaults to a limit of 50.
|
|
288
1273
|
*/
|
|
289
|
-
async
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
1274
|
+
async getRecentForEntities(entityIds, limit = 50) {
|
|
1275
|
+
if (entityIds.length === 0) return [];
|
|
1276
|
+
const placeholders = entityIds.map(() => "?").join(", ");
|
|
1277
|
+
return this.db.getAllAsync(
|
|
1278
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id IN (${placeholders}) ORDER BY created_at DESC LIMIT ?`,
|
|
1279
|
+
[...entityIds, limit]
|
|
1280
|
+
);
|
|
1281
|
+
}
|
|
1282
|
+
/**
|
|
1283
|
+
* Delete events for an entity that were created at or before the given cutoff timestamp.
|
|
1284
|
+
* Returns the number of deleted rows.
|
|
1285
|
+
*/
|
|
1286
|
+
async prune(entityId, cutoff) {
|
|
1287
|
+
return this.db.runAsync(
|
|
1288
|
+
`DELETE FROM ${this.prefix}events WHERE entity_id = ? AND created_at <= ?`,
|
|
294
1289
|
[entityId, cutoff]
|
|
295
1290
|
);
|
|
296
1291
|
}
|
|
1292
|
+
/**
|
|
1293
|
+
* Return the total number of events stored for an entity.
|
|
1294
|
+
* `tx` is optional — pass an active transaction handle for atomic reads.
|
|
1295
|
+
*/
|
|
1296
|
+
async count(entityId, tx) {
|
|
1297
|
+
const executor = tx ?? this.db;
|
|
1298
|
+
const row = await executor.getFirstAsync(
|
|
1299
|
+
`SELECT COUNT(*) as count FROM ${this.prefix}events WHERE entity_id = ?`,
|
|
1300
|
+
[entityId]
|
|
1301
|
+
);
|
|
1302
|
+
return row?.count ?? 0;
|
|
1303
|
+
}
|
|
1304
|
+
/**
|
|
1305
|
+
* Return all events for an entity in chronological (ASC) order.
|
|
1306
|
+
* When limit is provided, fetches newest-first then reverses to preserve chronological order.
|
|
1307
|
+
*/
|
|
1308
|
+
async getByEntityId(entityId, limit) {
|
|
1309
|
+
if (limit != null) {
|
|
1310
|
+
const rows = await this.db.getAllAsync(
|
|
1311
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?`,
|
|
1312
|
+
[entityId, limit]
|
|
1313
|
+
);
|
|
1314
|
+
return rows.slice().reverse();
|
|
1315
|
+
}
|
|
1316
|
+
return this.db.getAllAsync(
|
|
1317
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC`,
|
|
1318
|
+
[entityId]
|
|
1319
|
+
);
|
|
1320
|
+
}
|
|
1321
|
+
};
|
|
1322
|
+
|
|
1323
|
+
// src/repositories/MetadataRepository.ts
|
|
1324
|
+
var MetadataRepository = class extends BaseRepository {
|
|
1325
|
+
// CHECKPOINTS TABLE METHODS
|
|
1326
|
+
async getCheckpoint(entityId, tx) {
|
|
1327
|
+
const executor = this.getExecutor(tx);
|
|
1328
|
+
const row = await executor.getFirstAsync(
|
|
1329
|
+
`SELECT memory_checkpoint, heal_checkpoint FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
1330
|
+
[entityId]
|
|
1331
|
+
);
|
|
1332
|
+
if (!row) return {};
|
|
1333
|
+
return {
|
|
1334
|
+
memory: row.memory_checkpoint ?? void 0,
|
|
1335
|
+
heal: row.heal_checkpoint ?? void 0
|
|
1336
|
+
};
|
|
1337
|
+
}
|
|
1338
|
+
async updateCheckpoint(entityId, updates, tx) {
|
|
1339
|
+
const fields = [];
|
|
1340
|
+
const values = [];
|
|
1341
|
+
if (updates.memory !== void 0) {
|
|
1342
|
+
fields.push("memory_checkpoint = ?");
|
|
1343
|
+
values.push(updates.memory);
|
|
1344
|
+
}
|
|
1345
|
+
if (updates.heal !== void 0) {
|
|
1346
|
+
fields.push("heal_checkpoint = ?");
|
|
1347
|
+
values.push(updates.heal);
|
|
1348
|
+
}
|
|
1349
|
+
if (fields.length === 0) return;
|
|
1350
|
+
const executor = this.getExecutor(tx);
|
|
1351
|
+
await executor.runAsync(
|
|
1352
|
+
`INSERT INTO ${this.prefix}checkpoints (entity_id, memory_checkpoint, heal_checkpoint)
|
|
1353
|
+
VALUES (?, ?, ?)
|
|
1354
|
+
ON CONFLICT(entity_id) DO UPDATE SET ${fields.join(", ")}`,
|
|
1355
|
+
[entityId, updates.memory ?? 0, updates.heal ?? 0, ...values]
|
|
1356
|
+
);
|
|
1357
|
+
}
|
|
1358
|
+
async deleteCheckpoint(entityId, tx) {
|
|
1359
|
+
const executor = this.getExecutor(tx);
|
|
1360
|
+
await executor.runAsync(
|
|
1361
|
+
`DELETE FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
1362
|
+
[entityId]
|
|
1363
|
+
);
|
|
1364
|
+
}
|
|
1365
|
+
// META TABLE METHODS
|
|
1366
|
+
async getMeta(key, tx) {
|
|
1367
|
+
const executor = this.getExecutor(tx);
|
|
1368
|
+
const row = await executor.getFirstAsync(
|
|
1369
|
+
`SELECT value FROM ${this.prefix}meta WHERE key = ?`,
|
|
1370
|
+
[key]
|
|
1371
|
+
);
|
|
1372
|
+
return row ? row.value : null;
|
|
1373
|
+
}
|
|
1374
|
+
async setMeta(key, value, tx) {
|
|
1375
|
+
const executor = this.getExecutor(tx);
|
|
1376
|
+
await executor.runAsync(
|
|
1377
|
+
`INSERT INTO ${this.prefix}meta (key, value) VALUES (?, ?)
|
|
1378
|
+
ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
|
|
1379
|
+
[key, value]
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
async clearDimensionMismatch(tx) {
|
|
1383
|
+
const executor = this.getExecutor(tx);
|
|
1384
|
+
await executor.runAsync(
|
|
1385
|
+
`DELETE FROM ${this.prefix}meta WHERE key = 'embedding_dimension_mismatch'`
|
|
1386
|
+
);
|
|
1387
|
+
}
|
|
1388
|
+
async tableExists(tableName, tx) {
|
|
1389
|
+
const executor = this.getExecutor(tx);
|
|
1390
|
+
const row = await executor.getFirstAsync(
|
|
1391
|
+
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
|
|
1392
|
+
[tableName]
|
|
1393
|
+
);
|
|
1394
|
+
return row != null;
|
|
1395
|
+
}
|
|
1396
|
+
async getTableDdl(tableName, tx) {
|
|
1397
|
+
const executor = this.getExecutor(tx);
|
|
1398
|
+
const row = await executor.getFirstAsync(
|
|
1399
|
+
`SELECT sql FROM sqlite_master WHERE type='table' AND name=?`,
|
|
1400
|
+
[tableName]
|
|
1401
|
+
);
|
|
1402
|
+
return row?.sql ?? null;
|
|
1403
|
+
}
|
|
1404
|
+
async vacuum() {
|
|
1405
|
+
await this.db.execAsync(`PRAGMA wal_checkpoint(TRUNCATE)`);
|
|
1406
|
+
await this.db.execAsync(`VACUUM`);
|
|
1407
|
+
}
|
|
1408
|
+
async getDistinctEntityIds(tx) {
|
|
1409
|
+
const executor = this.getExecutor(tx);
|
|
1410
|
+
const rows = await executor.getAllAsync(
|
|
1411
|
+
`SELECT DISTINCT entity_id FROM (
|
|
1412
|
+
SELECT entity_id FROM ${this.prefix}entries WHERE deleted_at IS NULL
|
|
1413
|
+
UNION
|
|
1414
|
+
SELECT entity_id FROM ${this.prefix}tasks WHERE deleted_at IS NULL
|
|
1415
|
+
UNION
|
|
1416
|
+
SELECT entity_id FROM ${this.prefix}events
|
|
1417
|
+
) ORDER BY entity_id`
|
|
1418
|
+
);
|
|
1419
|
+
return rows.map((r) => r.entity_id);
|
|
1420
|
+
}
|
|
297
1421
|
};
|
|
298
1422
|
|
|
299
1423
|
// src/prompts.ts
|
|
@@ -447,9 +1571,6 @@ function parseJsonResponse(text) {
|
|
|
447
1571
|
if (end === -1) throw new SyntaxError("No JSON object/array found in LLM response");
|
|
448
1572
|
return JSON.parse(text.slice(start, end + 1));
|
|
449
1573
|
}
|
|
450
|
-
function generateId(prefix = "") {
|
|
451
|
-
return prefix + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15);
|
|
452
|
-
}
|
|
453
1574
|
function safeSlice(value, start, end) {
|
|
454
1575
|
const length = value.length;
|
|
455
1576
|
let safeStart = start < 0 ? Math.max(length + start, 0) : Math.min(start, length);
|
|
@@ -618,7 +1739,11 @@ var _WikiMemory = class _WikiMemory {
|
|
|
618
1739
|
this.db = db;
|
|
619
1740
|
this.options = options;
|
|
620
1741
|
this.prefix = options.config?.tablePrefix || "llm_wiki_";
|
|
621
|
-
this.
|
|
1742
|
+
this.outboxRepo = new OutboxRepository(db, this.prefix);
|
|
1743
|
+
this.entryRepo = new EntryRepository(db, this.prefix, this.outboxRepo);
|
|
1744
|
+
this.taskRepo = new TaskRepository(db, this.prefix, this.outboxRepo);
|
|
1745
|
+
this.eventRepo = new EventRepository(db, this.prefix);
|
|
1746
|
+
this.metadataRepo = new MetadataRepository(db, this.prefix);
|
|
622
1747
|
}
|
|
623
1748
|
normalizeMiniSearchRow(row) {
|
|
624
1749
|
return {
|
|
@@ -638,10 +1763,7 @@ var _WikiMemory = class _WikiMemory {
|
|
|
638
1763
|
}
|
|
639
1764
|
async rebuildMiniSearchIndex(entityId) {
|
|
640
1765
|
if (entityId) {
|
|
641
|
-
const rows2 = await this.
|
|
642
|
-
`SELECT id, entity_id, title, body, tags FROM ${this.prefix}entries WHERE deleted_at IS NULL AND entity_id = ?`,
|
|
643
|
-
[entityId]
|
|
644
|
-
);
|
|
1766
|
+
const rows2 = await this.entryRepo.findMiniSearchRows(entityId);
|
|
645
1767
|
const previousIds = this.miniSearchEntryIdsByEntity.get(entityId);
|
|
646
1768
|
if (previousIds) {
|
|
647
1769
|
for (const id of previousIds) {
|
|
@@ -655,7 +1777,7 @@ var _WikiMemory = class _WikiMemory {
|
|
|
655
1777
|
this.miniSearchEntryIdsByEntity.set(entityId, new Set(documents2.map((document) => document.id)));
|
|
656
1778
|
return;
|
|
657
1779
|
}
|
|
658
|
-
const rows = await this.
|
|
1780
|
+
const rows = await this.entryRepo.findMiniSearchRows();
|
|
659
1781
|
this.miniSearch.removeAll();
|
|
660
1782
|
this.miniSearchEntryIdsByEntity.clear();
|
|
661
1783
|
const documents = rows.map((row) => this.normalizeMiniSearchRow(row));
|
|
@@ -669,25 +1791,17 @@ var _WikiMemory = class _WikiMemory {
|
|
|
669
1791
|
}
|
|
670
1792
|
}
|
|
671
1793
|
async storeEmbeddingDimension(dim) {
|
|
672
|
-
const existing = await this.
|
|
673
|
-
`SELECT value FROM ${this.prefix}meta WHERE key = 'embedding_dimension'`
|
|
674
|
-
);
|
|
1794
|
+
const existing = await this.metadataRepo.getMeta("embedding_dimension");
|
|
675
1795
|
if (existing) {
|
|
676
|
-
const storedDim = parseInt(existing
|
|
1796
|
+
const storedDim = parseInt(existing, 10);
|
|
677
1797
|
if (storedDim !== dim) {
|
|
678
1798
|
console.warn(
|
|
679
1799
|
`[WikiMemory] Embedding dimension mismatch: stored ${storedDim}, got ${dim}. Call runReembed() to rebuild embeddings with the new model.`
|
|
680
1800
|
);
|
|
681
|
-
await this.
|
|
682
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
683
|
-
[String(dim)]
|
|
684
|
-
);
|
|
1801
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(dim), this.db);
|
|
685
1802
|
}
|
|
686
1803
|
} else {
|
|
687
|
-
await this.
|
|
688
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension', ?)`,
|
|
689
|
-
[String(dim)]
|
|
690
|
-
);
|
|
1804
|
+
await this.metadataRepo.setMeta("embedding_dimension", String(dim), this.db);
|
|
691
1805
|
}
|
|
692
1806
|
}
|
|
693
1807
|
/**
|
|
@@ -697,28 +1811,13 @@ var _WikiMemory = class _WikiMemory {
|
|
|
697
1811
|
* stuck on the MiniSearch fallback.
|
|
698
1812
|
*/
|
|
699
1813
|
async _reconcileEmbeddingDimension() {
|
|
700
|
-
const
|
|
701
|
-
|
|
702
|
-
);
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
WHERE deleted_at IS NULL
|
|
708
|
-
AND (
|
|
709
|
-
(embedding_blob IS NOT NULL AND (CAST(length(embedding_blob) AS INTEGER) / 4) != ?)
|
|
710
|
-
OR (embedding_blob IS NULL AND embedding IS NOT NULL)
|
|
711
|
-
)`,
|
|
712
|
-
[newDim]
|
|
713
|
-
);
|
|
714
|
-
if (!residual || residual.cnt === 0) {
|
|
715
|
-
await this.db.runAsync(
|
|
716
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension', ?)`,
|
|
717
|
-
[mismatch.value]
|
|
718
|
-
);
|
|
719
|
-
await this.db.runAsync(
|
|
720
|
-
`DELETE FROM ${this.prefix}meta WHERE key = 'embedding_dimension_mismatch'`
|
|
721
|
-
);
|
|
1814
|
+
const mismatchValue = await this.metadataRepo.getMeta("embedding_dimension_mismatch");
|
|
1815
|
+
if (!mismatchValue) return;
|
|
1816
|
+
const newDim = parseInt(mismatchValue, 10);
|
|
1817
|
+
const residualCount = await this.entryRepo.countStaleEmbeddings(newDim);
|
|
1818
|
+
if (residualCount === 0) {
|
|
1819
|
+
await this.metadataRepo.setMeta("embedding_dimension", mismatchValue, this.db);
|
|
1820
|
+
await this.metadataRepo.clearDimensionMismatch(this.db);
|
|
722
1821
|
}
|
|
723
1822
|
}
|
|
724
1823
|
async embedFact(fact) {
|
|
@@ -756,10 +1855,7 @@ var _WikiMemory = class _WikiMemory {
|
|
|
756
1855
|
}
|
|
757
1856
|
await this.storeEmbeddingDimension(float32Vector.length);
|
|
758
1857
|
const blob = new Uint8Array(float32Vector.buffer);
|
|
759
|
-
await this.
|
|
760
|
-
`UPDATE ${this.prefix}entries SET embedding_blob = ?, embedding = NULL WHERE id = ?`,
|
|
761
|
-
[blob, fact.id]
|
|
762
|
-
);
|
|
1858
|
+
await this.entryRepo.updateEmbeddingBlob(fact.id, blob);
|
|
763
1859
|
try {
|
|
764
1860
|
await this._notifyEmbeddingPersisted(fact.entity_id, fact.id, float32Vector);
|
|
765
1861
|
} catch (hookErr) {
|
|
@@ -792,28 +1888,12 @@ var _WikiMemory = class _WikiMemory {
|
|
|
792
1888
|
);
|
|
793
1889
|
}
|
|
794
1890
|
async assertNoLegacySourceTypes() {
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
WHERE source_type IN ('user_document', 'agent_inferred')
|
|
798
|
-
LIMIT 1`,
|
|
799
|
-
[]
|
|
800
|
-
);
|
|
801
|
-
if (!legacyProbe) return;
|
|
802
|
-
const legacyCount = await this.db.getFirstAsync(
|
|
803
|
-
`SELECT COUNT(*) as count FROM ${this.prefix}entries
|
|
804
|
-
WHERE source_type IN ('user_document', 'agent_inferred')`,
|
|
805
|
-
[]
|
|
806
|
-
);
|
|
807
|
-
const count = legacyCount?.count ?? 0;
|
|
808
|
-
const migrationSQL = `
|
|
809
|
-
-- Migrate legacy source_type values (targets your WikiMemory prefix: ${this.prefix})
|
|
810
|
-
UPDATE ${this.prefix}entries SET source_type = 'immutable_document' WHERE source_type = 'user_document';
|
|
811
|
-
UPDATE ${this.prefix}entries SET source_type = 'librarian_inferred' WHERE source_type = 'agent_inferred';
|
|
812
|
-
`.trim();
|
|
1891
|
+
if (!await this.entryRepo.hasLegacySourceTypes()) return;
|
|
1892
|
+
const count = await this.entryRepo.countLegacySourceTypes();
|
|
813
1893
|
throw new Error(
|
|
814
1894
|
`Database contains ${count} entries with legacy source_type values ('user_document' or 'agent_inferred'). These enum values were renamed in this release. Running without migration would allow legacy 'user_document' facts to bypass immutability guards, causing data corruption.
|
|
815
1895
|
|
|
816
|
-
${
|
|
1896
|
+
${this.entryRepo.getLegacyMigrationSQL()}
|
|
817
1897
|
|
|
818
1898
|
After running the migration SQL, restart your application.`
|
|
819
1899
|
);
|
|
@@ -870,77 +1950,45 @@ After running the migration SQL, restart your application.`
|
|
|
870
1950
|
}
|
|
871
1951
|
}
|
|
872
1952
|
async setup() {
|
|
873
|
-
const entriesExistedBeforeSetup = await this.
|
|
874
|
-
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
|
|
875
|
-
[`${this.prefix}entries`]
|
|
876
|
-
);
|
|
1953
|
+
const entriesExistedBeforeSetup = await this.metadataRepo.tableExists(`${this.prefix}entries`);
|
|
877
1954
|
await setupDatabase(this.db, this.prefix);
|
|
878
1955
|
let currentVersion;
|
|
879
1956
|
if (!entriesExistedBeforeSetup) {
|
|
880
|
-
await this.
|
|
881
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
|
|
882
|
-
[String(CURRENT_SCHEMA_VERSION)]
|
|
883
|
-
);
|
|
1957
|
+
await this.metadataRepo.setMeta("schema_version", String(CURRENT_SCHEMA_VERSION), this.db);
|
|
884
1958
|
currentVersion = CURRENT_SCHEMA_VERSION;
|
|
885
1959
|
} else {
|
|
886
|
-
const
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
if (metaRow) {
|
|
890
|
-
currentVersion = parseInt(metaRow.value, 10);
|
|
1960
|
+
const schemaVersionValue = await this.metadataRepo.getMeta("schema_version");
|
|
1961
|
+
if (schemaVersionValue) {
|
|
1962
|
+
currentVersion = parseInt(schemaVersionValue, 10);
|
|
891
1963
|
if (!Number.isFinite(currentVersion)) currentVersion = 0;
|
|
892
1964
|
} else {
|
|
893
|
-
const
|
|
894
|
-
|
|
895
|
-
[`${this.prefix}entries_fts`]
|
|
896
|
-
);
|
|
897
|
-
const hasPorter = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsMeta?.sql ?? "");
|
|
1965
|
+
const ftsDdl = await this.metadataRepo.getTableDdl(`${this.prefix}entries_fts`);
|
|
1966
|
+
const hasPorter = /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl ?? "");
|
|
898
1967
|
currentVersion = hasPorter ? 1 : 0;
|
|
899
1968
|
}
|
|
900
1969
|
}
|
|
901
1970
|
for (const migration of MIGRATIONS) {
|
|
902
1971
|
if (migration.version > currentVersion) {
|
|
903
1972
|
await migration.run(this.db, this.prefix);
|
|
904
|
-
await this.
|
|
905
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
|
|
906
|
-
[String(migration.version)]
|
|
907
|
-
);
|
|
1973
|
+
await this.metadataRepo.setMeta("schema_version", String(migration.version), this.db);
|
|
908
1974
|
currentVersion = migration.version;
|
|
909
1975
|
}
|
|
910
1976
|
}
|
|
911
1977
|
if (entriesExistedBeforeSetup) {
|
|
912
|
-
const
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
if (!metaCheck) {
|
|
916
|
-
await this.db.runAsync(
|
|
917
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('schema_version', ?)`,
|
|
918
|
-
[String(currentVersion)]
|
|
919
|
-
);
|
|
1978
|
+
const schemaVersionCheck = await this.metadataRepo.getMeta("schema_version");
|
|
1979
|
+
if (!schemaVersionCheck) {
|
|
1980
|
+
await this.metadataRepo.setMeta("schema_version", String(currentVersion), this.db);
|
|
920
1981
|
}
|
|
921
1982
|
}
|
|
922
1983
|
if (entriesExistedBeforeSetup) {
|
|
923
1984
|
await this.assertNoLegacySourceTypes();
|
|
924
1985
|
}
|
|
925
|
-
const rows = await this.
|
|
926
|
-
|
|
927
|
-
WHERE source_ref IS NOT NULL
|
|
928
|
-
AND (
|
|
929
|
-
TRIM(source_ref) != source_ref
|
|
930
|
-
OR INSTR(source_ref, '/') > 0
|
|
931
|
-
OR INSTR(source_ref, '\\') > 0
|
|
932
|
-
OR INSTR(source_ref, CHAR(0)) > 0
|
|
933
|
-
OR source_ref GLOB '*[^-A-Za-z0-9._ ]*'
|
|
934
|
-
)
|
|
935
|
-
`);
|
|
936
|
-
await this.db.withTransactionAsync(async () => {
|
|
1986
|
+
const rows = await this.entryRepo.findRowsForSourceRefMigration();
|
|
1987
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
937
1988
|
for (const row of rows) {
|
|
938
1989
|
const normalized = normalizeSourceRef(row.source_ref);
|
|
939
1990
|
if (normalized !== row.source_ref) {
|
|
940
|
-
await this.
|
|
941
|
-
`UPDATE ${this.prefix}entries SET source_ref = ? WHERE rowid = ?`,
|
|
942
|
-
[normalized, row.rowid]
|
|
943
|
-
);
|
|
1991
|
+
await this.entryRepo.updateSourceRefByRowid(row.rowid, normalized, tx);
|
|
944
1992
|
}
|
|
945
1993
|
}
|
|
946
1994
|
});
|
|
@@ -955,15 +2003,9 @@ After running the migration SQL, restart your application.`
|
|
|
955
2003
|
if (!normalizedHash) {
|
|
956
2004
|
throw new Error(`Invalid sourceHash: must be a 64-character hex string (normalized to lowercase)`);
|
|
957
2005
|
}
|
|
958
|
-
const
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
ORDER BY updated_at DESC
|
|
962
|
-
LIMIT 1`,
|
|
963
|
-
[entityId, normalizedRef]
|
|
964
|
-
);
|
|
965
|
-
if (!row) return true;
|
|
966
|
-
const normalizedStoredHash = row.source_hash ? normalizeSourceHash(row.source_hash) : null;
|
|
2006
|
+
const storedHash = await this.entryRepo.findLatestSourceHash(entityId, normalizedRef);
|
|
2007
|
+
if (storedHash === null) return true;
|
|
2008
|
+
const normalizedStoredHash = normalizeSourceHash(storedHash);
|
|
967
2009
|
return normalizedStoredHash !== normalizedHash;
|
|
968
2010
|
}
|
|
969
2011
|
_pruneKey(entityId) {
|
|
@@ -1086,24 +2128,13 @@ After running the migration SQL, restart your application.`
|
|
|
1086
2128
|
break;
|
|
1087
2129
|
}
|
|
1088
2130
|
}
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
1094
|
-
const entryResult = await this.db.runAsync(
|
|
1095
|
-
`DELETE FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ? AND id IN (${placeholders})`,
|
|
1096
|
-
[entityId, cutoff, ...chunk.map((r) => r.id)]
|
|
1097
|
-
);
|
|
1098
|
-
deletedEntries += entryResult.changes;
|
|
2131
|
+
const succeededIds = succeeded.map((r) => r.id);
|
|
2132
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2133
|
+
if (succeededIds.length > 0) {
|
|
2134
|
+
deletedEntries = await this.entryRepo.bulkDeletePruned(entityId, cutoff, succeededIds, tx);
|
|
1099
2135
|
}
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
`DELETE FROM ${this.prefix}tasks
|
|
1103
|
-
WHERE entity_id = ? AND deleted_at IS NOT NULL AND deleted_at <= ?`,
|
|
1104
|
-
[entityId, cutoff]
|
|
1105
|
-
);
|
|
1106
|
-
deletedTasks = taskResult.changes;
|
|
2136
|
+
deletedTasks = await this.taskRepo.bulkDeletePruned(entityId, cutoff, tx);
|
|
2137
|
+
});
|
|
1107
2138
|
if (failure) {
|
|
1108
2139
|
await this.rebuildMiniSearchIndex(entityId);
|
|
1109
2140
|
this.vectorCache.delete(entityId);
|
|
@@ -1136,16 +2167,11 @@ After running the migration SQL, restart your application.`
|
|
|
1136
2167
|
}
|
|
1137
2168
|
if (retainEventsFor !== null) {
|
|
1138
2169
|
const cutoff = now - retainEventsFor * 864e5;
|
|
1139
|
-
const eventResult = await this.
|
|
1140
|
-
`DELETE FROM ${this.prefix}events
|
|
1141
|
-
WHERE entity_id = ? AND created_at <= ?`,
|
|
1142
|
-
[entityId, cutoff]
|
|
1143
|
-
);
|
|
2170
|
+
const eventResult = await this.eventRepo.prune(entityId, cutoff);
|
|
1144
2171
|
deletedEvents = eventResult.changes;
|
|
1145
2172
|
}
|
|
1146
2173
|
if (vacuum) {
|
|
1147
|
-
await this.
|
|
1148
|
-
await this.db.execAsync(`VACUUM`);
|
|
2174
|
+
await this.metadataRepo.vacuum();
|
|
1149
2175
|
}
|
|
1150
2176
|
await this.rebuildMiniSearchIndex(entityId);
|
|
1151
2177
|
this.vectorCache.delete(entityId);
|
|
@@ -1201,27 +2227,17 @@ After running the migration SQL, restart your application.`
|
|
|
1201
2227
|
"embed() returned an empty or non-finite vector. Falling back to keyword search."
|
|
1202
2228
|
);
|
|
1203
2229
|
}
|
|
1204
|
-
const
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
if (storedDimRow) {
|
|
1208
|
-
const storedDim = parseInt(storedDimRow.value, 10);
|
|
2230
|
+
const storedDimValue = await this.metadataRepo.getMeta("embedding_dimension");
|
|
2231
|
+
if (storedDimValue) {
|
|
2232
|
+
const storedDim = parseInt(storedDimValue, 10);
|
|
1209
2233
|
if (storedDim !== queryVec.length) {
|
|
1210
2234
|
throw new Error(
|
|
1211
2235
|
`Embedding dimension mismatch: stored ${storedDim}, query has ${queryVec.length}. Call runReembed() to rebuild embeddings with the new model.`
|
|
1212
2236
|
);
|
|
1213
2237
|
}
|
|
1214
2238
|
}
|
|
1215
|
-
const
|
|
1216
|
-
|
|
1217
|
-
`SELECT COUNT(*) AS cnt FROM ${this.prefix}entries
|
|
1218
|
-
WHERE ${mismatchScope.clause} AND deleted_at IS NULL
|
|
1219
|
-
AND embedding_blob IS NOT NULL
|
|
1220
|
-
AND (CAST(length(embedding_blob) AS INTEGER) % 4 = 0)
|
|
1221
|
-
AND (CAST(length(embedding_blob) AS INTEGER) / 4) != ?`,
|
|
1222
|
-
[...mismatchScope.params, queryVec.length]
|
|
1223
|
-
);
|
|
1224
|
-
if (mismatchedCount && mismatchedCount.cnt > 0) {
|
|
2239
|
+
const mismatchedCount = await this.entryRepo.countDimensionMismatched(scoredEntityIds, queryVec.length);
|
|
2240
|
+
if (mismatchedCount > 0) {
|
|
1225
2241
|
throw new Error(
|
|
1226
2242
|
`Some facts have embeddings that do not match the current model dimension. Call runReembed() to rebuild all embeddings consistently.`
|
|
1227
2243
|
);
|
|
@@ -1245,31 +2261,10 @@ After running the migration SQL, restart your application.`
|
|
|
1245
2261
|
candidateRows = null;
|
|
1246
2262
|
} else {
|
|
1247
2263
|
const topKIds = topKResults.map((r) => r.id);
|
|
1248
|
-
const inClauseChunkSize = 500;
|
|
1249
2264
|
if (useRanker) {
|
|
1250
|
-
|
|
1251
|
-
for (let i = 0; i < topKIds.length; i += inClauseChunkSize) {
|
|
1252
|
-
const idChunk = topKIds.slice(i, i + inClauseChunkSize);
|
|
1253
|
-
const placeholders = idChunk.map(() => "?").join(",");
|
|
1254
|
-
const chunkRows = await this.db.getAllAsync(
|
|
1255
|
-
`SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
1256
|
-
idChunk
|
|
1257
|
-
);
|
|
1258
|
-
rows.push(...chunkRows);
|
|
1259
|
-
}
|
|
1260
|
-
candidateRows = rows;
|
|
2265
|
+
candidateRows = await this.entryRepo.findMetadataByIds(topKIds);
|
|
1261
2266
|
} else {
|
|
1262
|
-
|
|
1263
|
-
for (let i = 0; i < topKIds.length; i += inClauseChunkSize) {
|
|
1264
|
-
const idChunk = topKIds.slice(i, i + inClauseChunkSize);
|
|
1265
|
-
const placeholders = idChunk.map(() => "?").join(",");
|
|
1266
|
-
const chunkRows = await this.db.getAllAsync(
|
|
1267
|
-
`SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
1268
|
-
idChunk
|
|
1269
|
-
);
|
|
1270
|
-
rows.push(...chunkRows);
|
|
1271
|
-
}
|
|
1272
|
-
candidateRows = rows;
|
|
2267
|
+
candidateRows = await this.entryRepo.findWithEmbeddingsByIds(topKIds);
|
|
1273
2268
|
}
|
|
1274
2269
|
if (weight !== void 0 && weight < 1) {
|
|
1275
2270
|
const maxMsScore = Math.max(1, topKResults[0]?.score ?? 1);
|
|
@@ -1279,17 +2274,9 @@ After running the migration SQL, restart your application.`
|
|
|
1279
2274
|
}
|
|
1280
2275
|
} else {
|
|
1281
2276
|
if (useRanker) {
|
|
1282
|
-
|
|
1283
|
-
candidateRows = await this.db.getAllAsync(
|
|
1284
|
-
`SELECT id, entity_id, updated_at, access_count FROM ${this.prefix}entries WHERE ${entityScope.clause} AND deleted_at IS NULL`,
|
|
1285
|
-
entityScope.params
|
|
1286
|
-
);
|
|
2277
|
+
candidateRows = await this.entryRepo.findMetadataByEntityIds(scoredEntityIds);
|
|
1287
2278
|
} else {
|
|
1288
|
-
|
|
1289
|
-
candidateRows = await this.db.getAllAsync(
|
|
1290
|
-
`SELECT id, entity_id, embedding_blob, embedding, updated_at, access_count FROM ${this.prefix}entries WHERE ${entityScope.clause} AND deleted_at IS NULL`,
|
|
1291
|
-
entityScope.params
|
|
1292
|
-
);
|
|
2279
|
+
candidateRows = await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds);
|
|
1293
2280
|
}
|
|
1294
2281
|
if (weight !== void 0 && weight < 1) {
|
|
1295
2282
|
const entityIdSet = new Set(scoredEntityIds);
|
|
@@ -1452,19 +2439,8 @@ After running the migration SQL, restart your application.`
|
|
|
1452
2439
|
let fallbackRows = candidateRows;
|
|
1453
2440
|
if (fallbackRows && fallbackRows.length > 0 && !("embedding_blob" in fallbackRows[0])) {
|
|
1454
2441
|
const rowIds = fallbackRows.map((r) => r.id);
|
|
1455
|
-
const
|
|
1456
|
-
const
|
|
1457
|
-
for (let i = 0; i < rowIds.length; i += chunkSize) {
|
|
1458
|
-
const idChunk = rowIds.slice(i, i + chunkSize);
|
|
1459
|
-
const placeholders = idChunk.map(() => "?").join(",");
|
|
1460
|
-
const embeddingRows = await this.db.getAllAsync(
|
|
1461
|
-
`SELECT id, embedding_blob, embedding FROM ${this.prefix}entries WHERE id IN (${placeholders}) AND deleted_at IS NULL`,
|
|
1462
|
-
idChunk
|
|
1463
|
-
);
|
|
1464
|
-
for (const row of embeddingRows) {
|
|
1465
|
-
embeddingsMap.set(row.id, { embedding_blob: row.embedding_blob, embedding: row.embedding });
|
|
1466
|
-
}
|
|
1467
|
-
}
|
|
2442
|
+
const embeddingRows = await this.entryRepo.findEmbeddingsByIds(rowIds);
|
|
2443
|
+
const embeddingsMap = new Map(embeddingRows.map((row) => [row.id, row]));
|
|
1468
2444
|
fallbackRows = fallbackRows.map((r) => ({
|
|
1469
2445
|
...r,
|
|
1470
2446
|
embedding_blob: embeddingsMap.get(r.id)?.embedding_blob ?? null,
|
|
@@ -1605,65 +2581,15 @@ After running the migration SQL, restart your application.`
|
|
|
1605
2581
|
if (facts.length > 0) {
|
|
1606
2582
|
const ids = facts.map((f) => f.id);
|
|
1607
2583
|
const now = Date.now();
|
|
1608
|
-
|
|
1609
|
-
for (let i = 0; i < ids.length; i += accessChunkSize) {
|
|
1610
|
-
const idChunk = ids.slice(i, i + accessChunkSize);
|
|
1611
|
-
const placeholders = idChunk.map(() => "?").join(",");
|
|
1612
|
-
await this.db.runAsync(
|
|
1613
|
-
`UPDATE ${this.prefix}entries
|
|
1614
|
-
SET access_count = access_count + 1, last_accessed_at = ?
|
|
1615
|
-
WHERE id IN (${placeholders})`,
|
|
1616
|
-
[now, ...idChunk]
|
|
1617
|
-
);
|
|
1618
|
-
}
|
|
2584
|
+
await this.entryRepo.trackAccess(ids, now);
|
|
1619
2585
|
}
|
|
1620
2586
|
} else {
|
|
1621
|
-
|
|
1622
|
-
const rawFacts = await this.db.getAllAsync(
|
|
1623
|
-
`SELECT * FROM ${this.prefix}entries
|
|
1624
|
-
WHERE ${entityScope.clause} AND deleted_at IS NULL
|
|
1625
|
-
ORDER BY updated_at DESC
|
|
1626
|
-
LIMIT ?`,
|
|
1627
|
-
[...entityScope.params, maxResults]
|
|
1628
|
-
);
|
|
1629
|
-
facts = rawFacts.map((f) => {
|
|
1630
|
-
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
1631
|
-
return {
|
|
1632
|
-
...rest,
|
|
1633
|
-
tags: (() => {
|
|
1634
|
-
if (Array.isArray(rest.tags)) return rest.tags;
|
|
1635
|
-
try {
|
|
1636
|
-
const p = JSON.parse(rest.tags);
|
|
1637
|
-
return Array.isArray(p) ? p : [];
|
|
1638
|
-
} catch {
|
|
1639
|
-
return [];
|
|
1640
|
-
}
|
|
1641
|
-
})()
|
|
1642
|
-
};
|
|
1643
|
-
});
|
|
2587
|
+
facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults);
|
|
1644
2588
|
}
|
|
2589
|
+
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
1645
2590
|
const [tasks, events] = await Promise.all([
|
|
1646
|
-
(
|
|
1647
|
-
|
|
1648
|
-
const tasksLimit = entityIds.length === 1 ? void 0 : Math.min(20 * entityIds.length, 200);
|
|
1649
|
-
return this.db.getAllAsync(
|
|
1650
|
-
`SELECT * FROM ${this.prefix}tasks
|
|
1651
|
-
WHERE ${entityScope.clause} AND status IN ('pending', 'in_progress') AND deleted_at IS NULL
|
|
1652
|
-
ORDER BY priority DESC, created_at ASC${tasksLimit !== void 0 ? "\n LIMIT ?" : ""}`,
|
|
1653
|
-
tasksLimit !== void 0 ? [...entityScope.params, tasksLimit] : entityScope.params
|
|
1654
|
-
);
|
|
1655
|
-
})(),
|
|
1656
|
-
(async () => {
|
|
1657
|
-
const entityScope = this._entityInClause(entityIds);
|
|
1658
|
-
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
1659
|
-
return this.db.getAllAsync(
|
|
1660
|
-
`SELECT * FROM ${this.prefix}events
|
|
1661
|
-
WHERE ${entityScope.clause}
|
|
1662
|
-
ORDER BY created_at DESC
|
|
1663
|
-
LIMIT ?`,
|
|
1664
|
-
[...entityScope.params, eventsLimit]
|
|
1665
|
-
);
|
|
1666
|
-
})()
|
|
2591
|
+
this.taskRepo.findAllPending(entityIds, entityIds.length === 1 ? void 0 : Math.min(20 * entityIds.length, 200)),
|
|
2592
|
+
entityIds.length === 1 ? this.eventRepo.getRecent(entityIds[0], eventsLimit) : this.eventRepo.getRecentForEntities(entityIds, eventsLimit)
|
|
1667
2593
|
]);
|
|
1668
2594
|
let factScores;
|
|
1669
2595
|
if (exposeMetadata && trimmedQuery && scoreByFactId) {
|
|
@@ -1706,14 +2632,6 @@ After running the migration SQL, restart your application.`
|
|
|
1706
2632
|
if (updatedAtDiff !== 0) return updatedAtDiff;
|
|
1707
2633
|
return a.id.localeCompare(b.id);
|
|
1708
2634
|
}
|
|
1709
|
-
/**
|
|
1710
|
-
* Build SQL IN clause with placeholders for multiple entity IDs.
|
|
1711
|
-
*/
|
|
1712
|
-
_entityInClause(entityIds) {
|
|
1713
|
-
if (entityIds.length === 0) return { clause: "1=0", params: [] };
|
|
1714
|
-
const placeholders = entityIds.map(() => "?").join(",");
|
|
1715
|
-
return { clause: `entity_id IN (${placeholders})`, params: [...entityIds] };
|
|
1716
|
-
}
|
|
1717
2635
|
/**
|
|
1718
2636
|
* Hydrate full facts by ID. Pass scopedEntityIds to restrict to requested namespaces in SQL
|
|
1719
2637
|
* (defense-in-depth against a rogue VectorRanker returning cross-entity IDs).
|
|
@@ -1851,53 +2769,60 @@ After running the migration SQL, restart your application.`
|
|
|
1851
2769
|
if (!["observation", "decision", "action", "outcome"].includes(eventType)) {
|
|
1852
2770
|
eventType = "observation";
|
|
1853
2771
|
}
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
let
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
const
|
|
1868
|
-
|
|
1869
|
-
this.
|
|
1870
|
-
this.
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
2772
|
+
const newEvent = {
|
|
2773
|
+
id,
|
|
2774
|
+
entity_id: entityId,
|
|
2775
|
+
event_type: eventType,
|
|
2776
|
+
summary: event.summary,
|
|
2777
|
+
related_entry_id: event.related_entry_id || null,
|
|
2778
|
+
created_at: now
|
|
2779
|
+
};
|
|
2780
|
+
let shouldRunLibrarian = false;
|
|
2781
|
+
let librarianCount = 0;
|
|
2782
|
+
let librarianJobKey = null;
|
|
2783
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2784
|
+
await this.eventRepo.add(newEvent, tx);
|
|
2785
|
+
const threshold = this.options.config?.autoLibrarianThreshold || 20;
|
|
2786
|
+
const [count, cp] = await Promise.all([
|
|
2787
|
+
this.eventRepo.count(entityId, tx),
|
|
2788
|
+
this.metadataRepo.getCheckpoint(entityId, tx)
|
|
2789
|
+
]);
|
|
2790
|
+
let memoryCheckpoint = cp.memory ?? 0;
|
|
2791
|
+
if (memoryCheckpoint > count) memoryCheckpoint = 0;
|
|
2792
|
+
if (count - memoryCheckpoint >= threshold) {
|
|
2793
|
+
const jobKey = this._librarianKey(entityId);
|
|
2794
|
+
if (!this.activeMaintenanceJobs.has(jobKey) && !this.activeMaintenanceJobs.has(this._pruneKey(entityId)) && !this._isReembedActive(entityId) && !this._isImportActiveFor(entityId) && !this._isForgetActiveFor(entityId)) {
|
|
2795
|
+
shouldRunLibrarian = true;
|
|
2796
|
+
librarianCount = count;
|
|
2797
|
+
librarianJobKey = jobKey;
|
|
2798
|
+
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
2799
|
+
}
|
|
1875
2800
|
}
|
|
2801
|
+
});
|
|
2802
|
+
if (shouldRunLibrarian && librarianJobKey !== null) {
|
|
2803
|
+
this.activeMaintenanceJobs.add(librarianJobKey);
|
|
2804
|
+
this._notifyStatusSubscribers(entityId);
|
|
2805
|
+
this.runLibrarianThenMaybeHeal(entityId, librarianCount).catch(console.error).finally(() => {
|
|
2806
|
+
this.activeMaintenanceJobs.delete(librarianJobKey);
|
|
2807
|
+
this._notifyStatusSubscribers(entityId);
|
|
2808
|
+
});
|
|
1876
2809
|
}
|
|
1877
2810
|
}
|
|
1878
2811
|
async runLibrarianThenMaybeHeal(entityId, currentEventCount) {
|
|
1879
2812
|
await this._doRunLibrarian(entityId);
|
|
1880
|
-
await this.db.runAsync(`
|
|
1881
|
-
INSERT INTO ${this.prefix}checkpoints (entity_id, memory_checkpoint)
|
|
1882
|
-
VALUES (?, ?)
|
|
1883
|
-
ON CONFLICT(entity_id) DO UPDATE SET memory_checkpoint = ?
|
|
1884
|
-
`, [entityId, currentEventCount, currentEventCount]);
|
|
1885
2813
|
const autoHealThreshold = this.options.config?.autoHealThreshold || 100;
|
|
1886
|
-
const cp = await this.
|
|
1887
|
-
let healCheckpoint = cp
|
|
2814
|
+
const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
|
|
2815
|
+
let healCheckpoint = cp.heal ?? 0;
|
|
1888
2816
|
if (healCheckpoint > currentEventCount) healCheckpoint = 0;
|
|
1889
|
-
|
|
2817
|
+
const shouldRunHeal = currentEventCount - healCheckpoint >= autoHealThreshold;
|
|
2818
|
+
if (shouldRunHeal) {
|
|
1890
2819
|
const healKey = this._healKey(entityId);
|
|
1891
2820
|
if (!this.activeMaintenanceJobs.has(healKey)) {
|
|
1892
2821
|
this.activeMaintenanceJobs.add(healKey);
|
|
1893
2822
|
this._notifyStatusSubscribers(entityId);
|
|
1894
2823
|
try {
|
|
1895
2824
|
await this._doRunHeal(entityId);
|
|
1896
|
-
await this.
|
|
1897
|
-
INSERT INTO ${this.prefix}checkpoints (entity_id, heal_checkpoint)
|
|
1898
|
-
VALUES (?, ?)
|
|
1899
|
-
ON CONFLICT(entity_id) DO UPDATE SET heal_checkpoint = ?
|
|
1900
|
-
`, [entityId, currentEventCount, currentEventCount]);
|
|
2825
|
+
await this.metadataRepo.updateCheckpoint(entityId, { heal: currentEventCount }, this.db);
|
|
1901
2826
|
} finally {
|
|
1902
2827
|
this.activeMaintenanceJobs.delete(healKey);
|
|
1903
2828
|
this._notifyStatusSubscribers(entityId);
|
|
@@ -1906,18 +2831,8 @@ After running the migration SQL, restart your application.`
|
|
|
1906
2831
|
}
|
|
1907
2832
|
}
|
|
1908
2833
|
async _doRunLibrarian(entityId) {
|
|
1909
|
-
const events = await this.
|
|
1910
|
-
|
|
1911
|
-
WHERE entity_id = ?
|
|
1912
|
-
ORDER BY created_at DESC
|
|
1913
|
-
LIMIT 50
|
|
1914
|
-
`, [entityId]);
|
|
1915
|
-
const currentFactsRows = await this.db.getAllAsync(`
|
|
1916
|
-
SELECT * FROM ${this.prefix}entries
|
|
1917
|
-
WHERE entity_id = ? AND deleted_at IS NULL
|
|
1918
|
-
ORDER BY updated_at DESC
|
|
1919
|
-
LIMIT 100
|
|
1920
|
-
`, [entityId]);
|
|
2834
|
+
const events = await this.eventRepo.getRecent(entityId, 50);
|
|
2835
|
+
const currentFactsRows = await this.entryRepo.findRecentByEntityId(entityId, 100);
|
|
1921
2836
|
const currentFacts = currentFactsRows.map((f) => {
|
|
1922
2837
|
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
1923
2838
|
return {
|
|
@@ -1941,12 +2856,13 @@ ${JSON.stringify(currentFacts, null, 2)}`;
|
|
|
1941
2856
|
const validTasks = tasks.map(validateTask).filter((t) => t !== null);
|
|
1942
2857
|
const now = Date.now();
|
|
1943
2858
|
const insertedFacts = [];
|
|
1944
|
-
await this.db.withTransactionAsync(async () => {
|
|
2859
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2860
|
+
const factsForDedupe = await this.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
1945
2861
|
for (const fact of validFacts) {
|
|
1946
2862
|
const newTokens = titleTokens(fact.title);
|
|
1947
2863
|
let skip = false;
|
|
1948
2864
|
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
1949
|
-
for (const existing of
|
|
2865
|
+
for (const existing of factsForDedupe) {
|
|
1950
2866
|
if (existing.source_type !== "librarian_inferred") continue;
|
|
1951
2867
|
const existingTokens = titleTokens(existing.title);
|
|
1952
2868
|
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
@@ -1959,18 +2875,29 @@ ${JSON.stringify(currentFacts, null, 2)}`;
|
|
|
1959
2875
|
}
|
|
1960
2876
|
if (skip) continue;
|
|
1961
2877
|
const id = generateId("fact_");
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
2878
|
+
const factObj = {
|
|
2879
|
+
id,
|
|
2880
|
+
entity_id: entityId,
|
|
2881
|
+
title: fact.title,
|
|
2882
|
+
body: fact.body,
|
|
2883
|
+
tags: fact.tags,
|
|
2884
|
+
confidence: fact.confidence,
|
|
2885
|
+
source_type: "librarian_inferred",
|
|
2886
|
+
source_hash: null,
|
|
2887
|
+
source_ref: null,
|
|
2888
|
+
created_at: now,
|
|
2889
|
+
updated_at: now,
|
|
2890
|
+
last_accessed_at: null,
|
|
2891
|
+
access_count: 0,
|
|
2892
|
+
deleted_at: null
|
|
2893
|
+
};
|
|
2894
|
+
await this.entryRepo.upsert(factObj, tx);
|
|
1966
2895
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
1967
2896
|
}
|
|
1968
2897
|
for (const task of validTasks) {
|
|
1969
2898
|
const id = generateId("task_");
|
|
1970
|
-
|
|
1971
|
-
|
|
1972
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1973
|
-
`, [id, entityId, task.description, "pending", task.priority, now, now]);
|
|
2899
|
+
const taskObj = { id, entity_id: entityId, description: task.description, status: "pending", priority: task.priority, created_at: now, updated_at: now, resolved_at: null, deleted_at: null };
|
|
2900
|
+
await this.taskRepo.upsert(taskObj, tx);
|
|
1974
2901
|
}
|
|
1975
2902
|
});
|
|
1976
2903
|
await this.rebuildMiniSearchIndex(entityId);
|
|
@@ -1991,27 +2918,19 @@ ${JSON.stringify(currentFacts, null, 2)}`;
|
|
|
1991
2918
|
if (staleInferredAfterDays !== null && (typeof staleInferredAfterDays !== "number" || !Number.isFinite(staleInferredAfterDays) || staleInferredAfterDays < 0)) {
|
|
1992
2919
|
throw new Error("Invalid staleInferredAfterDays: must be a finite number >= 0 or null");
|
|
1993
2920
|
}
|
|
1994
|
-
await this.db.withTransactionAsync(async () => {
|
|
2921
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
1995
2922
|
if (orphanAfterDays !== null) {
|
|
1996
2923
|
const orphanThreshold = now - orphanAfterDays * MS_PER_DAY;
|
|
1997
|
-
await this.
|
|
1998
|
-
UPDATE ${this.prefix}entries
|
|
1999
|
-
SET deleted_at = ?, updated_at = ?
|
|
2000
|
-
WHERE entity_id = ? AND access_count = 0 AND created_at <= ? AND source_type != 'immutable_document' AND deleted_at IS NULL
|
|
2001
|
-
`, [now, now, entityId, orphanThreshold]);
|
|
2924
|
+
await this.entryRepo.markOrphaned(entityId, orphanThreshold, tx);
|
|
2002
2925
|
}
|
|
2003
2926
|
if (staleInferredAfterDays !== null) {
|
|
2004
2927
|
const staleThreshold = now - staleInferredAfterDays * MS_PER_DAY;
|
|
2005
|
-
await this.
|
|
2006
|
-
UPDATE ${this.prefix}entries
|
|
2007
|
-
SET confidence = 'tentative', updated_at = ?
|
|
2008
|
-
WHERE entity_id = ? AND confidence = 'inferred' AND (last_accessed_at <= ? OR (last_accessed_at IS NULL AND created_at <= ?)) AND source_type != 'immutable_document' AND deleted_at IS NULL
|
|
2009
|
-
`, [now, entityId, staleThreshold, staleThreshold]);
|
|
2928
|
+
await this.entryRepo.downgradeStaleInferred(entityId, staleThreshold, tx);
|
|
2010
2929
|
}
|
|
2011
2930
|
});
|
|
2012
|
-
const allFactsRows = await this.
|
|
2013
|
-
const allTasks = await this.
|
|
2014
|
-
const recentEvents = await this.
|
|
2931
|
+
const allFactsRows = await this.entryRepo.findAllByEntityId(entityId);
|
|
2932
|
+
const allTasks = await this.taskRepo.findAllPending([entityId]);
|
|
2933
|
+
const recentEvents = await this.eventRepo.getRecent(entityId, 20);
|
|
2015
2934
|
const healCandidates = allFactsRows.filter((f) => f.source_type !== "immutable_document");
|
|
2016
2935
|
const documentAnchors = allFactsRows.filter((f) => f.source_type === "immutable_document").map(({ id, title, source_ref }) => ({ id, title, source_ref }));
|
|
2017
2936
|
const userPrompt = `Heal Candidates:
|
|
@@ -2044,19 +2963,28 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2044
2963
|
const validNewFacts = newFacts.map(validateFact).filter((f) => f !== null);
|
|
2045
2964
|
const insertedFacts = [];
|
|
2046
2965
|
const uniqueDeletedFactIds = Array.from(new Set(safeDeleted));
|
|
2047
|
-
await this.db.withTransactionAsync(async () => {
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
}
|
|
2051
|
-
for (const id of safeDeleted) {
|
|
2052
|
-
await this.db.runAsync(`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE id = ? AND entity_id = ?`, [now, now, id, entityId]);
|
|
2053
|
-
}
|
|
2966
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2967
|
+
await this.entryRepo.downgradeByIds(safeDowngraded, entityId, tx);
|
|
2968
|
+
await this.entryRepo.softDeleteByIds(safeDeleted, entityId, tx);
|
|
2054
2969
|
for (const fact of validNewFacts) {
|
|
2055
2970
|
const id = generateId("fact_");
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2971
|
+
const factObj = {
|
|
2972
|
+
id,
|
|
2973
|
+
entity_id: entityId,
|
|
2974
|
+
title: fact.title,
|
|
2975
|
+
body: fact.body,
|
|
2976
|
+
tags: fact.tags,
|
|
2977
|
+
confidence: fact.confidence,
|
|
2978
|
+
source_type: "librarian_inferred",
|
|
2979
|
+
source_hash: null,
|
|
2980
|
+
source_ref: null,
|
|
2981
|
+
created_at: now,
|
|
2982
|
+
updated_at: now,
|
|
2983
|
+
last_accessed_at: null,
|
|
2984
|
+
access_count: 0,
|
|
2985
|
+
deleted_at: null
|
|
2986
|
+
};
|
|
2987
|
+
await this.entryRepo.upsert(factObj, tx);
|
|
2060
2988
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
2061
2989
|
}
|
|
2062
2990
|
});
|
|
@@ -2180,12 +3108,7 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2180
3108
|
}
|
|
2181
3109
|
this.activeMaintenanceJobs.add(reembedKey);
|
|
2182
3110
|
try {
|
|
2183
|
-
const
|
|
2184
|
-
const params = entityId ? [entityId] : [];
|
|
2185
|
-
const rows = await this.db.getAllAsync(
|
|
2186
|
-
`SELECT * FROM ${this.prefix}entries WHERE ${where}`,
|
|
2187
|
-
params
|
|
2188
|
-
);
|
|
3111
|
+
const rows = await this.entryRepo.findAllForReembed(entityId);
|
|
2189
3112
|
if (entityId) {
|
|
2190
3113
|
this.vectorCache.delete(entityId);
|
|
2191
3114
|
} else {
|
|
@@ -2194,22 +3117,12 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2194
3117
|
const skipExisting = opts?.skipExisting ?? false;
|
|
2195
3118
|
let effectiveSkip = skipExisting;
|
|
2196
3119
|
if (skipExisting) {
|
|
2197
|
-
const
|
|
2198
|
-
|
|
2199
|
-
);
|
|
2200
|
-
if (mismatchRow) {
|
|
3120
|
+
const mismatchValue = await this.metadataRepo.getMeta("embedding_dimension_mismatch");
|
|
3121
|
+
if (mismatchValue) {
|
|
2201
3122
|
if (entityId) {
|
|
2202
|
-
const mismatchDim = parseInt(
|
|
2203
|
-
const
|
|
2204
|
-
|
|
2205
|
-
WHERE entity_id = ? AND deleted_at IS NULL
|
|
2206
|
-
AND (
|
|
2207
|
-
embedding_blob IS NULL
|
|
2208
|
-
OR (CAST(length(embedding_blob) AS INTEGER) / 4) != ?
|
|
2209
|
-
)`,
|
|
2210
|
-
[entityId, mismatchDim]
|
|
2211
|
-
);
|
|
2212
|
-
if (staleForEntity && staleForEntity.cnt > 0) effectiveSkip = false;
|
|
3123
|
+
const mismatchDim = parseInt(mismatchValue, 10);
|
|
3124
|
+
const staleCount = await this.entryRepo.countStaleForEntity(entityId, mismatchDim);
|
|
3125
|
+
if (staleCount > 0) effectiveSkip = false;
|
|
2213
3126
|
} else {
|
|
2214
3127
|
effectiveSkip = false;
|
|
2215
3128
|
}
|
|
@@ -2301,19 +3214,10 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2301
3214
|
this.vectorCache.clear();
|
|
2302
3215
|
}
|
|
2303
3216
|
async _getFullBundle(entityId, opts) {
|
|
2304
|
-
const
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
this.db.getAllAsync(
|
|
2309
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC`,
|
|
2310
|
-
[entityId]
|
|
2311
|
-
),
|
|
2312
|
-
this.db.getAllAsync(
|
|
2313
|
-
`SELECT * FROM ${this.prefix}tasks WHERE entity_id = ? AND deleted_at IS NULL ORDER BY priority DESC, created_at ASC`,
|
|
2314
|
-
[entityId]
|
|
2315
|
-
),
|
|
2316
|
-
this.db.getAllAsync(eventsQuery, eventsParams)
|
|
3217
|
+
const [factsRaw, tasks, events] = await Promise.all([
|
|
3218
|
+
this.entryRepo.findAllByEntityId(entityId),
|
|
3219
|
+
this.taskRepo.findAllByEntityId(entityId),
|
|
3220
|
+
this.eventRepo.getByEntityId(entityId, opts?.maxEvents)
|
|
2317
3221
|
]);
|
|
2318
3222
|
const facts = factsRaw.map((f) => {
|
|
2319
3223
|
const { embedding: _embedding, embedding_blob, ...rest } = f;
|
|
@@ -2328,7 +3232,6 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2328
3232
|
tags: typeof factBase.tags === "string" ? JSON.parse(factBase.tags) : factBase.tags
|
|
2329
3233
|
};
|
|
2330
3234
|
});
|
|
2331
|
-
const events = maxEvents != null ? eventsRaw.slice().reverse() : eventsRaw;
|
|
2332
3235
|
return { facts, tasks, events };
|
|
2333
3236
|
}
|
|
2334
3237
|
async exportDump(entityIds) {
|
|
@@ -2336,16 +3239,7 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2336
3239
|
if (entityIds && entityIds.length > 0) {
|
|
2337
3240
|
ids = Array.from(new Set(entityIds));
|
|
2338
3241
|
} else {
|
|
2339
|
-
|
|
2340
|
-
SELECT DISTINCT entity_id FROM (
|
|
2341
|
-
SELECT entity_id FROM ${this.prefix}entries WHERE deleted_at IS NULL
|
|
2342
|
-
UNION
|
|
2343
|
-
SELECT entity_id FROM ${this.prefix}tasks WHERE deleted_at IS NULL
|
|
2344
|
-
UNION
|
|
2345
|
-
SELECT entity_id FROM ${this.prefix}events
|
|
2346
|
-
) ORDER BY entity_id
|
|
2347
|
-
`);
|
|
2348
|
-
ids = rows.map((r) => r.entity_id);
|
|
3242
|
+
ids = await this.metadataRepo.getDistinctEntityIds();
|
|
2349
3243
|
}
|
|
2350
3244
|
const entities = {};
|
|
2351
3245
|
const BATCH = 3;
|
|
@@ -2411,48 +3305,26 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2411
3305
|
const factsWithPreservedBlob = /* @__PURE__ */ new Map();
|
|
2412
3306
|
const preservedBlobDims = /* @__PURE__ */ new Set();
|
|
2413
3307
|
const softDeletedFactIds = [];
|
|
2414
|
-
await this.db.withTransactionAsync(async () => {
|
|
3308
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
2415
3309
|
if (!merge) {
|
|
2416
|
-
const
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
);
|
|
2420
|
-
|
|
2421
|
-
const now = Date.now();
|
|
2422
|
-
await this.db.runAsync(
|
|
2423
|
-
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
2424
|
-
[now, now, entityId]
|
|
2425
|
-
);
|
|
2426
|
-
await this.db.runAsync(
|
|
2427
|
-
`UPDATE ${this.prefix}tasks SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`,
|
|
2428
|
-
[now, now, entityId]
|
|
2429
|
-
);
|
|
2430
|
-
await this.db.runAsync(
|
|
2431
|
-
`DELETE FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
2432
|
-
[entityId]
|
|
2433
|
-
);
|
|
3310
|
+
const deletedLiveFactIds = await this.entryRepo.findIdsBySource(entityId, null, null, tx, false);
|
|
3311
|
+
softDeletedFactIds.push(...deletedLiveFactIds);
|
|
3312
|
+
await this.entryRepo.bulkSoftDeleteByEntityId(entityId, tx);
|
|
3313
|
+
await this.taskRepo.bulkSoftDeleteByEntityId(entityId, tx);
|
|
3314
|
+
await this.metadataRepo.deleteCheckpoint(entityId, tx);
|
|
2434
3315
|
}
|
|
2435
3316
|
const factIds = bundle.facts.map((fact) => fact.id);
|
|
2436
3317
|
const existingFactsById = /* @__PURE__ */ new Map();
|
|
2437
|
-
const
|
|
2438
|
-
for (
|
|
2439
|
-
|
|
2440
|
-
if (factIdChunk.length === 0) continue;
|
|
2441
|
-
const placeholders = factIdChunk.map(() => "?").join(", ");
|
|
2442
|
-
const existingFacts = await this.db.getAllAsync(
|
|
2443
|
-
`SELECT id, entity_id, updated_at FROM ${this.prefix}entries WHERE id IN (${placeholders})`,
|
|
2444
|
-
factIdChunk
|
|
2445
|
-
);
|
|
2446
|
-
for (const existingFact of existingFacts) {
|
|
2447
|
-
existingFactsById.set(existingFact.id, existingFact);
|
|
2448
|
-
}
|
|
3318
|
+
const existingFacts = await this.entryRepo.findExistingMetadataByIds(factIds, tx);
|
|
3319
|
+
for (const existingFact of existingFacts) {
|
|
3320
|
+
existingFactsById.set(existingFact.id, existingFact);
|
|
2449
3321
|
}
|
|
2450
3322
|
for (const fact of bundle.facts) {
|
|
2451
3323
|
const sourceType = this._normalizeImportedSourceType(String(fact.source_type), {
|
|
2452
3324
|
entityId,
|
|
2453
3325
|
factId: fact.id
|
|
2454
3326
|
});
|
|
2455
|
-
|
|
3327
|
+
JSON.stringify(Array.isArray(fact.tags) ? fact.tags : []);
|
|
2456
3328
|
const safeUpdatedAt = Number.isFinite(fact.updated_at) ? fact.updated_at : 0;
|
|
2457
3329
|
const existing = existingFactsById.get(fact.id);
|
|
2458
3330
|
const rawBlobRaw = fact.embedding_blob;
|
|
@@ -2497,55 +3369,38 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2497
3369
|
if (merge) {
|
|
2498
3370
|
if (safeUpdatedAt <= existing.updated_at) continue;
|
|
2499
3371
|
}
|
|
2500
|
-
if (blobData != null) {
|
|
2501
|
-
await this.db.runAsync(
|
|
2502
|
-
`UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ?, embedding_blob = ?, embedding = NULL WHERE id = ?`,
|
|
2503
|
-
[entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData, fact.id]
|
|
2504
|
-
);
|
|
2505
|
-
factsWithPreservedBlob.set(fact.id, blobData);
|
|
2506
|
-
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
2507
|
-
} else {
|
|
2508
|
-
await this.db.runAsync(
|
|
2509
|
-
`UPDATE ${this.prefix}entries SET entity_id = ?, title = ?, body = ?, tags = ?, confidence = ?, source_type = ?, source_hash = ?, source_ref = ?, created_at = ?, updated_at = ?, last_accessed_at = ?, access_count = ?, deleted_at = ?, embedding_blob = NULL, embedding = NULL WHERE id = ?`,
|
|
2510
|
-
[entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, fact.id]
|
|
2511
|
-
);
|
|
2512
|
-
}
|
|
2513
|
-
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2514
|
-
upsertedFactIds.add(fact.id);
|
|
2515
|
-
if (fact.deleted_at) upsertedDeletedFactIds.add(fact.id);
|
|
2516
|
-
} else {
|
|
2517
|
-
if (blobData != null) {
|
|
2518
|
-
await this.db.runAsync(
|
|
2519
|
-
`INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at, embedding_blob) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2520
|
-
[fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at, blobData]
|
|
2521
|
-
);
|
|
2522
|
-
factsWithPreservedBlob.set(fact.id, blobData);
|
|
2523
|
-
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
2524
|
-
} else {
|
|
2525
|
-
await this.db.runAsync(
|
|
2526
|
-
`INSERT INTO ${this.prefix}entries (id, entity_id, title, body, tags, confidence, source_type, source_hash, source_ref, created_at, updated_at, last_accessed_at, access_count, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2527
|
-
[fact.id, entityId, fact.title, fact.body, tagsJson, fact.confidence, sourceType, fact.source_hash, fact.source_ref, fact.created_at, safeUpdatedAt, fact.last_accessed_at, fact.access_count, fact.deleted_at]
|
|
2528
|
-
);
|
|
2529
|
-
}
|
|
2530
|
-
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2531
|
-
upsertedFactIds.add(fact.id);
|
|
2532
|
-
if (fact.deleted_at) upsertedDeletedFactIds.add(fact.id);
|
|
2533
3372
|
}
|
|
3373
|
+
const factObj = {
|
|
3374
|
+
id: fact.id,
|
|
3375
|
+
entity_id: entityId,
|
|
3376
|
+
title: fact.title,
|
|
3377
|
+
body: fact.body,
|
|
3378
|
+
tags: Array.isArray(fact.tags) ? fact.tags : [],
|
|
3379
|
+
confidence: fact.confidence,
|
|
3380
|
+
source_type: sourceType,
|
|
3381
|
+
source_hash: fact.source_hash,
|
|
3382
|
+
source_ref: fact.source_ref,
|
|
3383
|
+
created_at: fact.created_at,
|
|
3384
|
+
updated_at: safeUpdatedAt,
|
|
3385
|
+
last_accessed_at: fact.last_accessed_at,
|
|
3386
|
+
access_count: fact.access_count,
|
|
3387
|
+
deleted_at: fact.deleted_at,
|
|
3388
|
+
embedding_blob: blobData ?? void 0
|
|
3389
|
+
};
|
|
3390
|
+
await this.entryRepo.upsertForImport(factObj, tx);
|
|
3391
|
+
if (blobData != null) {
|
|
3392
|
+
factsWithPreservedBlob.set(fact.id, blobData);
|
|
3393
|
+
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
3394
|
+
}
|
|
3395
|
+
existingFactsById.set(fact.id, { id: fact.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
3396
|
+
upsertedFactIds.add(fact.id);
|
|
3397
|
+
if (fact.deleted_at) upsertedDeletedFactIds.add(fact.id);
|
|
2534
3398
|
}
|
|
2535
3399
|
const taskIds = bundle.tasks.map((task) => task.id);
|
|
2536
3400
|
const existingTasksById = /* @__PURE__ */ new Map();
|
|
2537
|
-
const
|
|
2538
|
-
for (
|
|
2539
|
-
|
|
2540
|
-
if (taskIdChunk.length === 0) continue;
|
|
2541
|
-
const placeholders = taskIdChunk.map(() => "?").join(", ");
|
|
2542
|
-
const existingTasks = await this.db.getAllAsync(
|
|
2543
|
-
`SELECT id, entity_id, updated_at FROM ${this.prefix}tasks WHERE id IN (${placeholders})`,
|
|
2544
|
-
taskIdChunk
|
|
2545
|
-
);
|
|
2546
|
-
for (const existingTask of existingTasks) {
|
|
2547
|
-
existingTasksById.set(existingTask.id, existingTask);
|
|
2548
|
-
}
|
|
3401
|
+
const existingTasks = await this.taskRepo.findExistingMetadataByIds(taskIds, tx);
|
|
3402
|
+
for (const existingTask of existingTasks) {
|
|
3403
|
+
existingTasksById.set(existingTask.id, existingTask);
|
|
2549
3404
|
}
|
|
2550
3405
|
for (const task of bundle.tasks) {
|
|
2551
3406
|
const safeUpdatedAt = Number.isFinite(task.updated_at) ? task.updated_at : 0;
|
|
@@ -2558,25 +3413,29 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2558
3413
|
if (merge) {
|
|
2559
3414
|
if (safeUpdatedAt <= existing.updated_at) continue;
|
|
2560
3415
|
}
|
|
2561
|
-
await this.db.runAsync(
|
|
2562
|
-
`UPDATE ${this.prefix}tasks SET entity_id = ?, description = ?, status = ?, priority = ?, created_at = ?, updated_at = ?, resolved_at = ?, deleted_at = ? WHERE id = ?`,
|
|
2563
|
-
[entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at, task.id]
|
|
2564
|
-
);
|
|
2565
|
-
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2566
|
-
} else {
|
|
2567
|
-
await this.db.runAsync(
|
|
2568
|
-
`INSERT INTO ${this.prefix}tasks (id, entity_id, description, status, priority, created_at, updated_at, resolved_at, deleted_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2569
|
-
[task.id, entityId, task.description, task.status, task.priority, task.created_at, safeUpdatedAt, task.resolved_at, task.deleted_at]
|
|
2570
|
-
);
|
|
2571
|
-
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2572
3416
|
}
|
|
3417
|
+
await this.taskRepo.upsertForImport({
|
|
3418
|
+
id: task.id,
|
|
3419
|
+
entity_id: entityId,
|
|
3420
|
+
description: task.description,
|
|
3421
|
+
status: task.status,
|
|
3422
|
+
priority: task.priority,
|
|
3423
|
+
created_at: task.created_at,
|
|
3424
|
+
updated_at: safeUpdatedAt,
|
|
3425
|
+
resolved_at: task.resolved_at,
|
|
3426
|
+
deleted_at: task.deleted_at
|
|
3427
|
+
}, tx, safeUpdatedAt);
|
|
3428
|
+
existingTasksById.set(task.id, { id: task.id, entity_id: entityId, updated_at: safeUpdatedAt });
|
|
2573
3429
|
}
|
|
2574
3430
|
for (const event of bundle.events) {
|
|
2575
|
-
await this.
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
3431
|
+
await this.eventRepo.addIgnoreDuplicate({
|
|
3432
|
+
id: event.id,
|
|
3433
|
+
entity_id: entityId,
|
|
3434
|
+
event_type: event.event_type,
|
|
3435
|
+
summary: event.summary,
|
|
3436
|
+
related_entry_id: event.related_entry_id ?? null,
|
|
3437
|
+
created_at: event.created_at
|
|
3438
|
+
}, tx);
|
|
2580
3439
|
}
|
|
2581
3440
|
});
|
|
2582
3441
|
this.vectorCache.delete(entityId);
|
|
@@ -2614,43 +3473,27 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2614
3473
|
}
|
|
2615
3474
|
}
|
|
2616
3475
|
try {
|
|
2617
|
-
const
|
|
2618
|
-
|
|
2619
|
-
);
|
|
2620
|
-
const canonicalDim = canonicalRow ? parseInt(canonicalRow.value, 10) : null;
|
|
3476
|
+
const canonicalDimValue = await this.metadataRepo.getMeta("embedding_dimension");
|
|
3477
|
+
const canonicalDim = canonicalDimValue ? parseInt(canonicalDimValue, 10) : null;
|
|
2621
3478
|
if (preservedBlobDims.size === 1) {
|
|
2622
3479
|
const preservedDim = [...preservedBlobDims][0];
|
|
2623
3480
|
if (canonicalDim === null || canonicalDim === preservedDim) {
|
|
2624
3481
|
await this.storeEmbeddingDimension(preservedDim);
|
|
2625
|
-
const
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
if (staleMismatch && parseInt(staleMismatch.value, 10) !== preservedDim) {
|
|
2629
|
-
await this.db.runAsync(
|
|
2630
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
2631
|
-
[String(preservedDim)]
|
|
2632
|
-
);
|
|
3482
|
+
const staleMismatchValue = await this.metadataRepo.getMeta("embedding_dimension_mismatch");
|
|
3483
|
+
if (staleMismatchValue && parseInt(staleMismatchValue, 10) !== preservedDim) {
|
|
3484
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(preservedDim), this.db);
|
|
2633
3485
|
}
|
|
2634
3486
|
await this._reconcileEmbeddingDimension();
|
|
2635
3487
|
} else {
|
|
2636
|
-
await this.
|
|
2637
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
2638
|
-
[String(canonicalDim)]
|
|
2639
|
-
);
|
|
3488
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(canonicalDim), this.db);
|
|
2640
3489
|
}
|
|
2641
3490
|
} else if (preservedBlobDims.size > 1) {
|
|
2642
3491
|
if (canonicalDim === null) {
|
|
2643
3492
|
const sortedPreservedBlobDims = [...preservedBlobDims].sort((a, b) => a - b);
|
|
2644
3493
|
await this.storeEmbeddingDimension(sortedPreservedBlobDims[0]);
|
|
2645
|
-
await this.
|
|
2646
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
2647
|
-
[String(sortedPreservedBlobDims[0])]
|
|
2648
|
-
);
|
|
3494
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(sortedPreservedBlobDims[0]), this.db);
|
|
2649
3495
|
} else {
|
|
2650
|
-
await this.
|
|
2651
|
-
`INSERT OR REPLACE INTO ${this.prefix}meta (key, value) VALUES ('embedding_dimension_mismatch', ?)`,
|
|
2652
|
-
[String(canonicalDim)]
|
|
2653
|
-
);
|
|
3496
|
+
await this.metadataRepo.setMeta("embedding_dimension_mismatch", String(canonicalDim), this.db);
|
|
2654
3497
|
}
|
|
2655
3498
|
}
|
|
2656
3499
|
} finally {
|
|
@@ -2684,79 +3527,44 @@ The following document anchors are provided for contradiction detection only. Do
|
|
|
2684
3527
|
let deletedEntries = 0;
|
|
2685
3528
|
let deletedTasks = 0;
|
|
2686
3529
|
const deletedEntryIds = [];
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
|
|
2695
|
-
|
|
2696
|
-
|
|
2697
|
-
|
|
2698
|
-
|
|
2699
|
-
|
|
2700
|
-
]);
|
|
2701
|
-
await this.db.runAsync(`UPDATE ${this.prefix}checkpoints SET memory_checkpoint = 0, heal_checkpoint = 0 WHERE entity_id = ?`, [entityId]);
|
|
2702
|
-
deletedEntries = entriesRes.changes;
|
|
2703
|
-
deletedTasks = tasksRes.changes;
|
|
2704
|
-
} else {
|
|
2705
|
-
const hasIdSelectors = params.entryId !== void 0 || params.taskId !== void 0;
|
|
2706
|
-
const hasSourceSelectors = params.sourceRef !== void 0 || params.sourceHash !== void 0;
|
|
2707
|
-
if (hasIdSelectors && hasSourceSelectors) {
|
|
2708
|
-
throw new Error("forget() params are mutually exclusive: use entryId/taskId together, or sourceRef/sourceHash together, but not both in the same call");
|
|
2709
|
-
}
|
|
2710
|
-
const sourceRef = params.sourceRef !== void 0 ? normalizeSourceRef(params.sourceRef) : null;
|
|
2711
|
-
if (params.sourceRef !== void 0 && !sourceRef) throw new Error("Invalid sourceRef");
|
|
2712
|
-
const sourceHash = params.sourceHash !== void 0 ? normalizeSourceHash(params.sourceHash) : null;
|
|
2713
|
-
if (params.sourceHash !== void 0 && !sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
2714
|
-
if (params.entryId) {
|
|
2715
|
-
const entry = await this.db.getFirstAsync(
|
|
2716
|
-
`SELECT id FROM ${this.prefix}entries WHERE id = ? AND entity_id = ?`,
|
|
2717
|
-
[params.entryId, entityId]
|
|
2718
|
-
);
|
|
2719
|
-
if (entry) deletedEntryIds.push(entry.id);
|
|
2720
|
-
}
|
|
2721
|
-
if (sourceRef || sourceHash) {
|
|
2722
|
-
let q = `SELECT id FROM ${this.prefix}entries WHERE entity_id = ?`;
|
|
2723
|
-
const args = [entityId];
|
|
2724
|
-
if (sourceRef) {
|
|
2725
|
-
q += ` AND source_ref = ?`;
|
|
2726
|
-
args.push(sourceRef);
|
|
2727
|
-
}
|
|
2728
|
-
if (sourceHash) {
|
|
2729
|
-
q += ` AND source_hash = ?`;
|
|
2730
|
-
args.push(sourceHash);
|
|
3530
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3531
|
+
if (params.clearAll) {
|
|
3532
|
+
deletedEntryIds.push(...await this.entryRepo.findIdsBySource(entityId, null, null, tx, true));
|
|
3533
|
+
const entriesRes = await this.entryRepo.bulkSoftDeleteByEntityId(entityId, tx);
|
|
3534
|
+
const tasksRes = await this.taskRepo.bulkSoftDeleteByEntityId(entityId, tx);
|
|
3535
|
+
await this.metadataRepo.updateCheckpoint(entityId, { memory: 0, heal: 0 }, tx);
|
|
3536
|
+
deletedEntries = entriesRes;
|
|
3537
|
+
deletedTasks = tasksRes;
|
|
3538
|
+
} else {
|
|
3539
|
+
const hasIdSelectors = params.entryId !== void 0 || params.taskId !== void 0;
|
|
3540
|
+
const hasSourceSelectors = params.sourceRef !== void 0 || params.sourceHash !== void 0;
|
|
3541
|
+
if (hasIdSelectors && hasSourceSelectors) {
|
|
3542
|
+
throw new Error("forget() params are mutually exclusive: use entryId/taskId together, or sourceRef/sourceHash together, but not both in the same call");
|
|
2731
3543
|
}
|
|
2732
|
-
const
|
|
2733
|
-
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
2738
|
-
|
|
2739
|
-
let q = `UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE entity_id = ? AND deleted_at IS NULL`;
|
|
2740
|
-
const args = [now, now, entityId];
|
|
2741
|
-
if (sourceRef) {
|
|
2742
|
-
q += ` AND source_ref = ?`;
|
|
2743
|
-
args.push(sourceRef);
|
|
3544
|
+
const sourceRef = params.sourceRef !== void 0 ? normalizeSourceRef(params.sourceRef) : null;
|
|
3545
|
+
if (params.sourceRef !== void 0 && !sourceRef) throw new Error("Invalid sourceRef");
|
|
3546
|
+
const sourceHash = params.sourceHash !== void 0 ? normalizeSourceHash(params.sourceHash) : null;
|
|
3547
|
+
if (params.sourceHash !== void 0 && !sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
3548
|
+
if (params.entryId) {
|
|
3549
|
+
const entryId = await this.entryRepo.findIdById(params.entryId, entityId, tx);
|
|
3550
|
+
if (entryId) deletedEntryIds.push(entryId);
|
|
2744
3551
|
}
|
|
2745
|
-
if (sourceHash) {
|
|
2746
|
-
|
|
2747
|
-
args.push(sourceHash);
|
|
3552
|
+
if (sourceRef || sourceHash) {
|
|
3553
|
+
deletedEntryIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, sourceHash, tx, true));
|
|
2748
3554
|
}
|
|
2749
|
-
|
|
3555
|
+
const entryPromise = params.entryId ? this.entryRepo.softDelete(params.entryId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
3556
|
+
const taskDeletedPromise = params.taskId ? this.taskRepo.softDeleteById(params.taskId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
3557
|
+
const refPromise = sourceRef || sourceHash ? this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, sourceHash) : null;
|
|
3558
|
+
const [entryResult, taskResult, refResult] = await Promise.all([
|
|
3559
|
+
entryPromise ?? Promise.resolve(false),
|
|
3560
|
+
taskDeletedPromise ?? Promise.resolve(false),
|
|
3561
|
+
refPromise ?? Promise.resolve(0)
|
|
3562
|
+
]);
|
|
3563
|
+
if (entryResult) deletedEntries++;
|
|
3564
|
+
if (taskResult) deletedTasks++;
|
|
3565
|
+
deletedEntries += refResult;
|
|
2750
3566
|
}
|
|
2751
|
-
|
|
2752
|
-
entryPromise ?? Promise.resolve(null),
|
|
2753
|
-
taskPromise ?? Promise.resolve(null),
|
|
2754
|
-
refPromise ?? Promise.resolve(null)
|
|
2755
|
-
]);
|
|
2756
|
-
if (entryResult) deletedEntries += entryResult.changes;
|
|
2757
|
-
if (taskResult) deletedTasks += taskResult.changes;
|
|
2758
|
-
if (refResult) deletedEntries += refResult.changes;
|
|
2759
|
-
}
|
|
3567
|
+
});
|
|
2760
3568
|
await this.rebuildMiniSearchIndex(entityId);
|
|
2761
3569
|
this.vectorCache.delete(entityId);
|
|
2762
3570
|
const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
|
|
@@ -2855,18 +3663,9 @@ ${chunk}`;
|
|
|
2855
3663
|
const now = Date.now();
|
|
2856
3664
|
const insertedFacts = [];
|
|
2857
3665
|
const deletedSourceFactIds = [];
|
|
2858
|
-
await this.db.withTransactionAsync(async () => {
|
|
2859
|
-
|
|
2860
|
-
|
|
2861
|
-
[sourceRef, entityId]
|
|
2862
|
-
);
|
|
2863
|
-
for (const row of existingSourceFacts) {
|
|
2864
|
-
deletedSourceFactIds.push(row.id);
|
|
2865
|
-
}
|
|
2866
|
-
await this.db.runAsync(
|
|
2867
|
-
`UPDATE ${this.prefix}entries SET deleted_at = ?, updated_at = ? WHERE source_ref = ? AND entity_id = ? AND deleted_at IS NULL`,
|
|
2868
|
-
[now, now, sourceRef, entityId]
|
|
2869
|
-
);
|
|
3666
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
3667
|
+
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
3668
|
+
await this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, null);
|
|
2870
3669
|
for (const fact of allValidFacts) {
|
|
2871
3670
|
const id = generateId("fact_");
|
|
2872
3671
|
const wikiFact = {
|
|
@@ -2885,7 +3684,7 @@ ${chunk}`;
|
|
|
2885
3684
|
access_count: 0,
|
|
2886
3685
|
deleted_at: null
|
|
2887
3686
|
};
|
|
2888
|
-
await this.entryRepo.upsert(wikiFact);
|
|
3687
|
+
await this.entryRepo.upsert(wikiFact, tx);
|
|
2889
3688
|
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
2890
3689
|
}
|
|
2891
3690
|
});
|