@equationalapplications/core-llm-wiki 7.7.7 → 7.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +94 -0
- package/dist/{chunk-46X3HO77.mjs → chunk-J2TWKOEE.mjs} +1190 -185
- package/dist/chunk-J2TWKOEE.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1862 -285
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +606 -35
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-BIFNLUva.d.mts → testing-Kb1uVj1E.d.mts} +594 -7
- package/dist/{testing-BIFNLUva.d.ts → testing-Kb1uVj1E.d.ts} +594 -7
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +1139 -197
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-46X3HO77.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -48,7 +48,11 @@ async function setupDatabase(db, prefix) {
|
|
|
48
48
|
okf_usage_window TEXT,
|
|
49
49
|
embedding_failed_at INTEGER,
|
|
50
50
|
embedding_failure_kind TEXT,
|
|
51
|
-
embedding_attempts INTEGER NOT NULL DEFAULT 0
|
|
51
|
+
embedding_attempts INTEGER NOT NULL DEFAULT 0,
|
|
52
|
+
valid_from INTEGER,
|
|
53
|
+
valid_to INTEGER,
|
|
54
|
+
superseded_by TEXT,
|
|
55
|
+
superseded_at INTEGER
|
|
52
56
|
);
|
|
53
57
|
|
|
54
58
|
CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
|
|
@@ -115,7 +119,8 @@ async function setupDatabase(db, prefix) {
|
|
|
115
119
|
event_type TEXT NOT NULL,
|
|
116
120
|
summary TEXT NOT NULL,
|
|
117
121
|
related_entry_id TEXT,
|
|
118
|
-
created_at INTEGER NOT NULL
|
|
122
|
+
created_at INTEGER NOT NULL,
|
|
123
|
+
occurred_at INTEGER
|
|
119
124
|
);
|
|
120
125
|
|
|
121
126
|
CREATE INDEX IF NOT EXISTS ${prefix}events_entity_idx ON ${prefix}events(entity_id, created_at DESC);
|
|
@@ -123,7 +128,9 @@ async function setupDatabase(db, prefix) {
|
|
|
123
128
|
CREATE TABLE IF NOT EXISTS ${prefix}checkpoints (
|
|
124
129
|
entity_id TEXT PRIMARY KEY,
|
|
125
130
|
heal_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
126
|
-
memory_checkpoint INTEGER NOT NULL DEFAULT 0
|
|
131
|
+
memory_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
132
|
+
librarian_watermark_at INTEGER,
|
|
133
|
+
librarian_watermark_id TEXT
|
|
127
134
|
);
|
|
128
135
|
|
|
129
136
|
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
@@ -155,6 +162,7 @@ async function setupDatabase(db, prefix) {
|
|
|
155
162
|
ON ${prefix}outbox (created_at);
|
|
156
163
|
`);
|
|
157
164
|
await createOkfV02IndexesIfColumnsExist(db, prefix);
|
|
165
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
158
166
|
}
|
|
159
167
|
async function createOkfV02IndexesIfColumnsExist(db, prefix) {
|
|
160
168
|
for (const table of ["entries", "tasks"]) {
|
|
@@ -173,6 +181,20 @@ async function createOkfV02IndexesIfColumnsExist(db, prefix) {
|
|
|
173
181
|
}
|
|
174
182
|
}
|
|
175
183
|
}
|
|
184
|
+
async function createTemporalIndexesIfColumnsExist(db, prefix) {
|
|
185
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}entries)`);
|
|
186
|
+
const names = new Set(cols.map((c) => c.name));
|
|
187
|
+
if (names.has("superseded_by")) {
|
|
188
|
+
await db.execAsync(
|
|
189
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_superseded_idx ON ${prefix}entries(entity_id, superseded_by) WHERE superseded_by IS NOT NULL;`
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
if (names.has("valid_from") && names.has("valid_to")) {
|
|
193
|
+
await db.execAsync(
|
|
194
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_temporal_idx ON ${prefix}entries(entity_id) WHERE valid_from IS NOT NULL OR valid_to IS NOT NULL;`
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
176
198
|
|
|
177
199
|
// src/db/sqliteCodes.ts
|
|
178
200
|
var SQLITE_RESULT_CODE_NAMES = {
|
|
@@ -341,6 +363,15 @@ var WikiDraftNotFound = class _WikiDraftNotFound extends Error {
|
|
|
341
363
|
Object.setPrototypeOf(this, _WikiDraftNotFound.prototype);
|
|
342
364
|
}
|
|
343
365
|
};
|
|
366
|
+
var WikiSupersedeError = class _WikiSupersedeError extends Error {
|
|
367
|
+
constructor(reason) {
|
|
368
|
+
super(`supersede() rejected: ${reason}`);
|
|
369
|
+
this.code = "WIKI_SUPERSEDE_REJECTED";
|
|
370
|
+
this.name = "WikiSupersedeError";
|
|
371
|
+
this.reason = reason;
|
|
372
|
+
Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
|
|
373
|
+
}
|
|
374
|
+
};
|
|
344
375
|
|
|
345
376
|
// src/db/serializedAdapter.ts
|
|
346
377
|
function isDriverError(err) {
|
|
@@ -630,6 +661,35 @@ var MIGRATIONS = [
|
|
|
630
661
|
DROP INDEX IF EXISTS ${prefix}edges_entity_idx;
|
|
631
662
|
`);
|
|
632
663
|
}
|
|
664
|
+
},
|
|
665
|
+
{
|
|
666
|
+
version: 13,
|
|
667
|
+
description: "Temporal facts: valid_from/valid_to/superseded_by/superseded_at on entries, occurred_at on events, librarian watermark on checkpoints",
|
|
668
|
+
run: async (db, prefix) => {
|
|
669
|
+
const adds = [
|
|
670
|
+
["entries", "valid_from", "INTEGER"],
|
|
671
|
+
["entries", "valid_to", "INTEGER"],
|
|
672
|
+
["entries", "superseded_by", "TEXT"],
|
|
673
|
+
["entries", "superseded_at", "INTEGER"],
|
|
674
|
+
["events", "occurred_at", "INTEGER"],
|
|
675
|
+
["checkpoints", "librarian_watermark_at", "INTEGER"],
|
|
676
|
+
["checkpoints", "librarian_watermark_id", "TEXT"]
|
|
677
|
+
];
|
|
678
|
+
const existingByTable = /* @__PURE__ */ new Map();
|
|
679
|
+
for (const [table, column, type] of adds) {
|
|
680
|
+
let existing = existingByTable.get(table);
|
|
681
|
+
if (!existing) {
|
|
682
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}${table})`);
|
|
683
|
+
existing = new Set(cols.map((c) => c.name));
|
|
684
|
+
existingByTable.set(table, existing);
|
|
685
|
+
}
|
|
686
|
+
if (!existing.has(column)) {
|
|
687
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN ${column} ${type}`);
|
|
688
|
+
existing.add(column);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
692
|
+
}
|
|
633
693
|
}
|
|
634
694
|
];
|
|
635
695
|
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
@@ -679,6 +739,24 @@ function parseJsonObject(s, fallback = null) {
|
|
|
679
739
|
}
|
|
680
740
|
return fallback;
|
|
681
741
|
}
|
|
742
|
+
|
|
743
|
+
// src/utils/temporal.ts
|
|
744
|
+
function liveAtSql(mode, alias = "") {
|
|
745
|
+
const from = mode === "asOf" ? `COALESCE(${alias}valid_from, ${alias}created_at) <= ?` : `(${alias}valid_from IS NULL OR ${alias}valid_from <= ?)`;
|
|
746
|
+
return `(${from} AND (${alias}valid_to IS NULL OR ${alias}valid_to > ?))`;
|
|
747
|
+
}
|
|
748
|
+
function isLiveAt(f, mode, t) {
|
|
749
|
+
if (f.deleted_at != null) return false;
|
|
750
|
+
const from = mode === "asOf" ? f.valid_from ?? f.created_at : f.valid_from;
|
|
751
|
+
if (from != null && from > t) return false;
|
|
752
|
+
return f.valid_to == null || f.valid_to > t;
|
|
753
|
+
}
|
|
754
|
+
function assertEpochMs(name, v) {
|
|
755
|
+
if (typeof v !== "number" || !Number.isFinite(v) || v < 0) {
|
|
756
|
+
throw new TypeError(`${name} must be a finite epoch-ms number >= 0`);
|
|
757
|
+
}
|
|
758
|
+
return Math.trunc(v);
|
|
759
|
+
}
|
|
682
760
|
function mapRowToFact(row) {
|
|
683
761
|
const tags = (() => {
|
|
684
762
|
if (Array.isArray(row.tags)) return row.tags;
|
|
@@ -719,7 +797,13 @@ function mapRowToFact(row) {
|
|
|
719
797
|
last_verified_by: row.last_verified_by ?? null,
|
|
720
798
|
// Spec §2.7 + §5.3: hydrate so read() consumers don't re-call the helpers.
|
|
721
799
|
isStale: coreOkf.isStaleAfter(staleAfterRaw, now),
|
|
722
|
-
trustTier: coreOkf.deriveTrustTier(okf_verified)
|
|
800
|
+
trustTier: coreOkf.deriveTrustTier(okf_verified),
|
|
801
|
+
// Temporal (spec 2026-09-29 §4, §10.8): keys appear only when set so
|
|
802
|
+
// existing deep-equality assertions and host snapshots are unchanged.
|
|
803
|
+
...row.valid_from != null ? { valid_from: Number(row.valid_from) } : {},
|
|
804
|
+
...row.valid_to != null ? { valid_to: Number(row.valid_to) } : {},
|
|
805
|
+
...row.superseded_by != null ? { superseded_by: String(row.superseded_by) } : {},
|
|
806
|
+
...row.superseded_at != null ? { superseded_at: Number(row.superseded_at) } : {}
|
|
723
807
|
};
|
|
724
808
|
}
|
|
725
809
|
function normalizeEmbeddingBlobValue(blob) {
|
|
@@ -1140,12 +1224,14 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1140
1224
|
*/
|
|
1141
1225
|
async findHealCandidatesByEntityId(entityId, limit, recheckCutoff, tx) {
|
|
1142
1226
|
const executor = this.getExecutor(tx);
|
|
1227
|
+
const now = Date.now();
|
|
1143
1228
|
const rows = await executor.getAllAsync(
|
|
1144
1229
|
`SELECT * FROM ${this.prefix}entries
|
|
1145
1230
|
WHERE entity_id = ? AND deleted_at IS NULL AND source_type != 'immutable_document'
|
|
1231
|
+
AND ${liveAtSql("current")}
|
|
1146
1232
|
AND (heal_checked_at IS NULL OR heal_checked_at <= ?)
|
|
1147
1233
|
ORDER BY updated_at ASC LIMIT ?`,
|
|
1148
|
-
[entityId, recheckCutoff, limit]
|
|
1234
|
+
[entityId, now, now, recheckCutoff, limit]
|
|
1149
1235
|
);
|
|
1150
1236
|
return rows.map(mapRowToFact);
|
|
1151
1237
|
}
|
|
@@ -1159,6 +1245,7 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1159
1245
|
const executor = this.getExecutor(tx);
|
|
1160
1246
|
const columns = opts?.withBody ? "id, title, source_ref, body, lifecycle_status" : "id, title, source_ref";
|
|
1161
1247
|
const rows = [];
|
|
1248
|
+
const now = Date.now();
|
|
1162
1249
|
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
1163
1250
|
const chunk = ids.slice(i, i + this.chunkSize);
|
|
1164
1251
|
const placeholders = chunk.map(() => "?").join(", ");
|
|
@@ -1166,8 +1253,9 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1166
1253
|
`SELECT ${columns} FROM ${this.prefix}entries
|
|
1167
1254
|
WHERE entity_id = ? AND deleted_at IS NULL
|
|
1168
1255
|
AND source_type = 'immutable_document'
|
|
1256
|
+
AND ${liveAtSql("current")}
|
|
1169
1257
|
AND id IN (${placeholders})`,
|
|
1170
|
-
[entityId, ...chunk]
|
|
1258
|
+
[entityId, now, now, ...chunk]
|
|
1171
1259
|
);
|
|
1172
1260
|
rows.push(...chunkRows);
|
|
1173
1261
|
}
|
|
@@ -1187,10 +1275,11 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1187
1275
|
*/
|
|
1188
1276
|
async findRecentByEntityId(entityId, limit, tx, excludeSourceRef) {
|
|
1189
1277
|
const executor = this.getExecutor(tx);
|
|
1278
|
+
const now = Date.now();
|
|
1190
1279
|
const excludeClause = excludeSourceRef ? ` AND (source_ref IS NULL OR source_ref != ?)` : "";
|
|
1191
|
-
const args = excludeSourceRef ? [entityId, excludeSourceRef, limit] : [entityId, limit];
|
|
1280
|
+
const args = excludeSourceRef ? [entityId, now, now, excludeSourceRef, limit] : [entityId, now, now, limit];
|
|
1192
1281
|
const rows = await executor.getAllAsync(
|
|
1193
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL${excludeClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
1282
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL AND ${liveAtSql("current")}${excludeClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
1194
1283
|
args
|
|
1195
1284
|
);
|
|
1196
1285
|
return rows.map(mapRowToFact);
|
|
@@ -1398,6 +1487,97 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1398
1487
|
}, tx);
|
|
1399
1488
|
}
|
|
1400
1489
|
}
|
|
1490
|
+
/** Spec §4: write any subset of the four temporal columns. Absent key = untouched; null = clear. */
|
|
1491
|
+
async setTemporal(id, entityId, fields, tx, opts) {
|
|
1492
|
+
const keys = ["valid_from", "valid_to", "superseded_by", "superseded_at"].filter((k) => k in fields);
|
|
1493
|
+
if (keys.length === 0) return 0;
|
|
1494
|
+
const executor = this.getExecutor(tx);
|
|
1495
|
+
const result = await executor.runAsync(
|
|
1496
|
+
`UPDATE ${this.prefix}entries SET ${keys.map((k) => `${k} = ?`).join(", ")} WHERE id = ? AND entity_id = ?`,
|
|
1497
|
+
[...keys.map((k) => fields[k] ?? null), id, entityId]
|
|
1498
|
+
);
|
|
1499
|
+
if (result.changes > 0 && opts?.outbox !== false) {
|
|
1500
|
+
const payload = { id, entity_id: entityId };
|
|
1501
|
+
for (const k of keys) payload[k] = fields[k] ?? null;
|
|
1502
|
+
await this.outbox.push({ entityId, tableName: "entries", recordId: id, operation: "UPDATE", payload }, tx);
|
|
1503
|
+
}
|
|
1504
|
+
return result.changes;
|
|
1505
|
+
}
|
|
1506
|
+
async findTemporalRow(id, tx) {
|
|
1507
|
+
const executor = this.getExecutor(tx);
|
|
1508
|
+
const row = await executor.getFirstAsync(
|
|
1509
|
+
`SELECT id, entity_id, source_type, deleted_at, created_at, valid_from, valid_to, superseded_by
|
|
1510
|
+
FROM ${this.prefix}entries WHERE id = ?`,
|
|
1511
|
+
[id]
|
|
1512
|
+
);
|
|
1513
|
+
if (!row) return null;
|
|
1514
|
+
return {
|
|
1515
|
+
id: String(row.id),
|
|
1516
|
+
entity_id: String(row.entity_id),
|
|
1517
|
+
source_type: row.source_type,
|
|
1518
|
+
deleted_at: row.deleted_at != null ? Number(row.deleted_at) : null,
|
|
1519
|
+
created_at: Number(row.created_at),
|
|
1520
|
+
valid_from: row.valid_from != null ? Number(row.valid_from) : null,
|
|
1521
|
+
valid_to: row.valid_to != null ? Number(row.valid_to) : null,
|
|
1522
|
+
superseded_by: row.superseded_by != null ? String(row.superseded_by) : null
|
|
1523
|
+
};
|
|
1524
|
+
}
|
|
1525
|
+
/**
|
|
1526
|
+
* Ids of non-deleted facts NOT live at `t` (spec §4.2). In 'current' mode a
|
|
1527
|
+
* row with both validity columns NULL is always live, so the query is
|
|
1528
|
+
* restricted to rows that carry a validity column — served by the partial
|
|
1529
|
+
* entries_temporal_idx.
|
|
1530
|
+
*/
|
|
1531
|
+
async findNonLiveIdsByEntityIds(entityIds, mode, t, tx) {
|
|
1532
|
+
if (entityIds.length === 0) return /* @__PURE__ */ new Set();
|
|
1533
|
+
const executor = this.getExecutor(tx);
|
|
1534
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
1535
|
+
const temporalOnly = mode === "current" ? " AND (valid_from IS NOT NULL OR valid_to IS NOT NULL)" : "";
|
|
1536
|
+
const rows = await executor.getAllAsync(
|
|
1537
|
+
`SELECT id FROM ${this.prefix}entries
|
|
1538
|
+
WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL${temporalOnly}
|
|
1539
|
+
AND NOT ${liveAtSql(mode)}`,
|
|
1540
|
+
[...entityIds, t, t]
|
|
1541
|
+
);
|
|
1542
|
+
return new Set(rows.map((r) => r.id));
|
|
1543
|
+
}
|
|
1544
|
+
async findSupersessionChainIds(entityId, factId, maxDepth, tx) {
|
|
1545
|
+
const executor = this.getExecutor(tx);
|
|
1546
|
+
const successors = await executor.getAllAsync(
|
|
1547
|
+
`WITH RECURSIVE fwd(id, depth) AS (
|
|
1548
|
+
SELECT superseded_by, 1 FROM ${this.prefix}entries WHERE id = ? AND entity_id = ? AND superseded_by IS NOT NULL
|
|
1549
|
+
UNION
|
|
1550
|
+
SELECT e.superseded_by, fwd.depth + 1 FROM fwd
|
|
1551
|
+
JOIN ${this.prefix}entries e ON e.id = fwd.id AND e.entity_id = ?
|
|
1552
|
+
WHERE e.superseded_by IS NOT NULL AND fwd.depth < ?
|
|
1553
|
+
)
|
|
1554
|
+
SELECT id FROM fwd ORDER BY depth ASC`,
|
|
1555
|
+
[factId, entityId, entityId, maxDepth]
|
|
1556
|
+
);
|
|
1557
|
+
const predecessors = await executor.getAllAsync(
|
|
1558
|
+
`WITH RECURSIVE bwd(id, depth) AS (
|
|
1559
|
+
SELECT id, 1 FROM ${this.prefix}entries WHERE entity_id = ? AND superseded_by = ?
|
|
1560
|
+
UNION
|
|
1561
|
+
SELECT e.id, bwd.depth + 1 FROM bwd
|
|
1562
|
+
JOIN ${this.prefix}entries e ON e.entity_id = ? AND e.superseded_by = bwd.id
|
|
1563
|
+
WHERE bwd.depth < ?
|
|
1564
|
+
)
|
|
1565
|
+
SELECT id FROM bwd ORDER BY depth DESC`,
|
|
1566
|
+
[entityId, factId, entityId, maxDepth]
|
|
1567
|
+
);
|
|
1568
|
+
const drop = (ids) => ids.map((r) => r.id).filter((id) => id !== factId);
|
|
1569
|
+
return { predecessors: drop(predecessors), successors: drop(successors) };
|
|
1570
|
+
}
|
|
1571
|
+
async findExpiredIds(entityId, cutoff, tx) {
|
|
1572
|
+
const executor = this.getExecutor(tx);
|
|
1573
|
+
const rows = await executor.getAllAsync(
|
|
1574
|
+
`SELECT id FROM ${this.prefix}entries
|
|
1575
|
+
WHERE entity_id = ? AND deleted_at IS NULL AND valid_to IS NOT NULL AND valid_to <= ?
|
|
1576
|
+
ORDER BY valid_to ASC`,
|
|
1577
|
+
[entityId, cutoff]
|
|
1578
|
+
);
|
|
1579
|
+
return rows.map((r) => r.id);
|
|
1580
|
+
}
|
|
1401
1581
|
/**
|
|
1402
1582
|
* Bulk soft-delete all entries for an entity.
|
|
1403
1583
|
* Stages DELETE outbox entries for each row in the same transaction.
|
|
@@ -1830,9 +2010,11 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1830
2010
|
const executor = this.getExecutor(tx);
|
|
1831
2011
|
const placeholders = entityIds.map(() => "?").join(",");
|
|
1832
2012
|
const draftClause = opts?.excludeDrafts === true ? ` AND lifecycle_status != 'draft'` : "";
|
|
2013
|
+
const liveClause = opts?.live ? ` AND ${liveAtSql(opts.live.mode)}` : "";
|
|
2014
|
+
const liveArgs = opts?.live ? [opts.live.t, opts.live.t] : [];
|
|
1833
2015
|
const rows = await executor.getAllAsync(
|
|
1834
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL${draftClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
1835
|
-
[...entityIds, limit]
|
|
2016
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL${draftClause}${liveClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
2017
|
+
[...entityIds, ...liveArgs, limit]
|
|
1836
2018
|
);
|
|
1837
2019
|
return rows.map(mapRowToFact);
|
|
1838
2020
|
}
|
|
@@ -1947,13 +2129,15 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1947
2129
|
/** Counts live mutable facts: eligible (past cooldown) vs deferred (in cooldown). */
|
|
1948
2130
|
async countHealCandidatesByEntityId(entityId, recheckCutoff, tx) {
|
|
1949
2131
|
const executor = this.getExecutor(tx);
|
|
2132
|
+
const now = Date.now();
|
|
1950
2133
|
const row = await executor.getFirstAsync(
|
|
1951
2134
|
`SELECT
|
|
1952
2135
|
SUM(CASE WHEN heal_checked_at IS NULL OR heal_checked_at <= ? THEN 1 ELSE 0 END) AS eligible,
|
|
1953
2136
|
SUM(CASE WHEN heal_checked_at IS NOT NULL AND heal_checked_at > ? THEN 1 ELSE 0 END) AS deferred
|
|
1954
2137
|
FROM ${this.prefix}entries
|
|
1955
|
-
WHERE entity_id = ? AND deleted_at IS NULL AND source_type != 'immutable_document'
|
|
1956
|
-
|
|
2138
|
+
WHERE entity_id = ? AND deleted_at IS NULL AND source_type != 'immutable_document'
|
|
2139
|
+
AND ${liveAtSql("current")}`,
|
|
2140
|
+
[recheckCutoff, recheckCutoff, entityId, now, now]
|
|
1957
2141
|
);
|
|
1958
2142
|
return { eligible: Number(row?.eligible ?? 0), deferred: Number(row?.deferred ?? 0) };
|
|
1959
2143
|
}
|
|
@@ -1996,6 +2180,21 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1996
2180
|
);
|
|
1997
2181
|
return rows.map((r) => ({ id: r.id, title: r.title }));
|
|
1998
2182
|
}
|
|
2183
|
+
/**
|
|
2184
|
+
* Count of live facts for `entityId` with no stored embedding that are still
|
|
2185
|
+
* retryable (`embedding_attempts < maxAttempts`). Backs the deferred
|
|
2186
|
+
* maintenance pending report's `reembedPending` flag and the reembed job's
|
|
2187
|
+
* entity selection (PR-C 2026-09-29).
|
|
2188
|
+
*/
|
|
2189
|
+
async countReembedPending(entityId, maxAttempts, tx) {
|
|
2190
|
+
const executor = this.getExecutor(tx);
|
|
2191
|
+
const row = await executor.getFirstAsync(
|
|
2192
|
+
`SELECT COUNT(*) AS n FROM ${this.prefix}entries
|
|
2193
|
+
WHERE entity_id = ? AND deleted_at IS NULL AND embedding_blob IS NULL AND embedding IS NULL AND embedding_attempts < ?`,
|
|
2194
|
+
[entityId, maxAttempts]
|
|
2195
|
+
);
|
|
2196
|
+
return Number(row?.n ?? 0);
|
|
2197
|
+
}
|
|
1999
2198
|
};
|
|
2000
2199
|
/**
|
|
2001
2200
|
* Column list and liveness predicate shared by findMiniSearchRows and
|
|
@@ -2468,6 +2667,18 @@ var TaskRepository = class extends BaseRepository {
|
|
|
2468
2667
|
};
|
|
2469
2668
|
|
|
2470
2669
|
// src/repositories/EventRepository.ts
|
|
2670
|
+
var AFTER_SQL = "((created_at > ?) OR (created_at = ? AND id > ?))";
|
|
2671
|
+
function afterArgs(wm) {
|
|
2672
|
+
return [wm.at, wm.at, wm.id];
|
|
2673
|
+
}
|
|
2674
|
+
function mapEvent(row) {
|
|
2675
|
+
if (row.occurred_at == null) {
|
|
2676
|
+
const rest = { ...row };
|
|
2677
|
+
delete rest.occurred_at;
|
|
2678
|
+
return rest;
|
|
2679
|
+
}
|
|
2680
|
+
return row;
|
|
2681
|
+
}
|
|
2471
2682
|
var EventRepository = class extends BaseRepository {
|
|
2472
2683
|
/**
|
|
2473
2684
|
* Insert a new event row.
|
|
@@ -2476,30 +2687,32 @@ var EventRepository = class extends BaseRepository {
|
|
|
2476
2687
|
async add(event, tx) {
|
|
2477
2688
|
const executor = this.getExecutor(tx);
|
|
2478
2689
|
await executor.runAsync(
|
|
2479
|
-
`INSERT INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
|
|
2480
|
-
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
2690
|
+
`INSERT INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at, occurred_at)
|
|
2691
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
2481
2692
|
[
|
|
2482
2693
|
event.id,
|
|
2483
2694
|
event.entity_id,
|
|
2484
2695
|
event.event_type,
|
|
2485
2696
|
event.summary,
|
|
2486
2697
|
event.related_entry_id ?? null,
|
|
2487
|
-
event.created_at
|
|
2698
|
+
event.created_at,
|
|
2699
|
+
event.occurred_at ?? null
|
|
2488
2700
|
]
|
|
2489
2701
|
);
|
|
2490
2702
|
}
|
|
2491
2703
|
async addIgnoreDuplicate(event, tx) {
|
|
2492
2704
|
const executor = this.getExecutor(tx);
|
|
2493
2705
|
await executor.runAsync(
|
|
2494
|
-
`INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
|
|
2495
|
-
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
2706
|
+
`INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at, occurred_at)
|
|
2707
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
2496
2708
|
[
|
|
2497
2709
|
event.id,
|
|
2498
2710
|
event.entity_id,
|
|
2499
2711
|
event.event_type,
|
|
2500
2712
|
event.summary,
|
|
2501
2713
|
event.related_entry_id ?? null,
|
|
2502
|
-
event.created_at
|
|
2714
|
+
event.created_at,
|
|
2715
|
+
event.occurred_at ?? null
|
|
2503
2716
|
]
|
|
2504
2717
|
);
|
|
2505
2718
|
}
|
|
@@ -2508,10 +2721,11 @@ var EventRepository = class extends BaseRepository {
|
|
|
2508
2721
|
* Defaults to a limit of 50.
|
|
2509
2722
|
*/
|
|
2510
2723
|
async getRecent(entityId, limit = 50) {
|
|
2511
|
-
|
|
2724
|
+
const rows = await this.db.getAllAsync(
|
|
2512
2725
|
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?`,
|
|
2513
2726
|
[entityId, limit]
|
|
2514
2727
|
);
|
|
2728
|
+
return rows.map(mapEvent);
|
|
2515
2729
|
}
|
|
2516
2730
|
/**
|
|
2517
2731
|
* Return the most recent events for the given entity IDs, newest first.
|
|
@@ -2520,10 +2734,11 @@ var EventRepository = class extends BaseRepository {
|
|
|
2520
2734
|
async getRecentForEntities(entityIds, limit = 50) {
|
|
2521
2735
|
if (entityIds.length === 0) return [];
|
|
2522
2736
|
const placeholders = entityIds.map(() => "?").join(", ");
|
|
2523
|
-
|
|
2737
|
+
const rows = await this.db.getAllAsync(
|
|
2524
2738
|
`SELECT * FROM ${this.prefix}events WHERE entity_id IN (${placeholders}) ORDER BY created_at DESC LIMIT ?`,
|
|
2525
2739
|
[...entityIds, limit]
|
|
2526
2740
|
);
|
|
2741
|
+
return rows.map(mapEvent);
|
|
2527
2742
|
}
|
|
2528
2743
|
/**
|
|
2529
2744
|
* Delete events for an entity that were created at or before the given cutoff timestamp.
|
|
@@ -2535,6 +2750,33 @@ var EventRepository = class extends BaseRepository {
|
|
|
2535
2750
|
[entityId, cutoff]
|
|
2536
2751
|
);
|
|
2537
2752
|
}
|
|
2753
|
+
/**
|
|
2754
|
+
* Watermark-guarded prune (spec 2026-09-29 competitive-memory §6): delete
|
|
2755
|
+
* events at or before `cutoff` **and** at-or-before the librarian watermark.
|
|
2756
|
+
* Events the librarian has not processed yet are retention-held, never
|
|
2757
|
+
* deleted. `wm = null` (no watermark yet) deletes nothing.
|
|
2758
|
+
* Used by runPrune under the deferred/ops strategies.
|
|
2759
|
+
*/
|
|
2760
|
+
async pruneThrough(entityId, cutoff, wm) {
|
|
2761
|
+
if (!wm) return { changes: 0 };
|
|
2762
|
+
return this.db.runAsync(
|
|
2763
|
+
`DELETE FROM ${this.prefix}events WHERE entity_id = ? AND created_at <= ? AND NOT ${AFTER_SQL}`,
|
|
2764
|
+
[entityId, cutoff, ...afterArgs(wm)]
|
|
2765
|
+
);
|
|
2766
|
+
}
|
|
2767
|
+
/**
|
|
2768
|
+
* Count events at or before `cutoff` that are after the watermark — the
|
|
2769
|
+
* retention-held tail `pruneThrough` left in place. `wm = null` counts all
|
|
2770
|
+
* events at or before `cutoff` (every one of them is held back).
|
|
2771
|
+
*/
|
|
2772
|
+
async countHeldBack(entityId, cutoff, wm) {
|
|
2773
|
+
const where = wm ? ` AND ${AFTER_SQL}` : "";
|
|
2774
|
+
const row = await this.db.getFirstAsync(
|
|
2775
|
+
`SELECT COUNT(*) AS n FROM ${this.prefix}events WHERE entity_id = ? AND created_at <= ?${where}`,
|
|
2776
|
+
[entityId, cutoff, ...wm ? afterArgs(wm) : []]
|
|
2777
|
+
);
|
|
2778
|
+
return Number(row?.n ?? 0);
|
|
2779
|
+
}
|
|
2538
2780
|
/**
|
|
2539
2781
|
* Return the total number of events stored for an entity.
|
|
2540
2782
|
* `tx` is optional — pass an active transaction handle for atomic reads.
|
|
@@ -2553,16 +2795,82 @@ var EventRepository = class extends BaseRepository {
|
|
|
2553
2795
|
*/
|
|
2554
2796
|
async getByEntityId(entityId, limit) {
|
|
2555
2797
|
if (limit != null) {
|
|
2556
|
-
const
|
|
2798
|
+
const rows2 = await this.db.getAllAsync(
|
|
2557
2799
|
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?`,
|
|
2558
2800
|
[entityId, limit]
|
|
2559
2801
|
);
|
|
2560
|
-
return
|
|
2802
|
+
return rows2.map(mapEvent).slice().reverse();
|
|
2561
2803
|
}
|
|
2562
|
-
|
|
2804
|
+
const rows = await this.db.getAllAsync(
|
|
2563
2805
|
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC`,
|
|
2564
2806
|
[entityId]
|
|
2565
2807
|
);
|
|
2808
|
+
return rows.map(mapEvent);
|
|
2809
|
+
}
|
|
2810
|
+
/**
|
|
2811
|
+
* Return events for `entityId` strictly after the watermark `(at, id)` in
|
|
2812
|
+
* chronological (ASC, id-ASC tie-break) order. Pass `wm = null` to read
|
|
2813
|
+
* from the beginning of the entity's event log. `limit` caps the result.
|
|
2814
|
+
*
|
|
2815
|
+
* Used by the op-based librarian (PR-B) to enumerate its unprocessed tail.
|
|
2816
|
+
*/
|
|
2817
|
+
async getAfter(entityId, wm, limit) {
|
|
2818
|
+
const limitClause = limit != null ? ` LIMIT ?` : "";
|
|
2819
|
+
const params = wm ? [entityId, wm.at, wm.at, wm.id] : [entityId];
|
|
2820
|
+
const where = wm ? `entity_id = ? AND ${AFTER_SQL}` : `entity_id = ?`;
|
|
2821
|
+
if (limit != null) params.push(limit);
|
|
2822
|
+
const rows = await this.db.getAllAsync(
|
|
2823
|
+
`SELECT * FROM ${this.prefix}events WHERE ${where} ORDER BY created_at ASC, id ASC${limitClause}`,
|
|
2824
|
+
params
|
|
2825
|
+
);
|
|
2826
|
+
return rows.map(mapEvent);
|
|
2827
|
+
}
|
|
2828
|
+
/**
|
|
2829
|
+
* Count of events for `entityId` strictly after the watermark. Used as
|
|
2830
|
+
* the op-based librarian's bounded batch size and by the deferred-maintenance
|
|
2831
|
+
* pending report. Pass `wm = null` to count the entity's whole event log
|
|
2832
|
+
* (no watermark yet) — mirrors `getAfter` / `sumSummaryCharsAfter`.
|
|
2833
|
+
*/
|
|
2834
|
+
async countAfter(entityId, wm) {
|
|
2835
|
+
const params = wm ? [entityId, wm.at, wm.at, wm.id] : [entityId];
|
|
2836
|
+
const where = wm ? `entity_id = ? AND ${AFTER_SQL}` : `entity_id = ?`;
|
|
2837
|
+
const row = await this.db.getFirstAsync(
|
|
2838
|
+
`SELECT COUNT(*) as count FROM ${this.prefix}events WHERE ${where}`,
|
|
2839
|
+
params
|
|
2840
|
+
);
|
|
2841
|
+
return row?.count ?? 0;
|
|
2842
|
+
}
|
|
2843
|
+
/**
|
|
2844
|
+
* Return the event at a given chronological offset for `entityId` (0-based,
|
|
2845
|
+
* oldest first). Used by MaintenanceService.seedLibrarianWatermark to anchor
|
|
2846
|
+
* the watermark on a real row when a legacy DB first migrates to the
|
|
2847
|
+
* op-based librarian. The offset is clamped and truncated to defend against
|
|
2848
|
+
* hostile or fractional callers.
|
|
2849
|
+
*/
|
|
2850
|
+
async getAtOffset(entityId, offset, tx) {
|
|
2851
|
+
const executor = this.getExecutor(tx);
|
|
2852
|
+
const row = await executor.getFirstAsync(
|
|
2853
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC, id ASC LIMIT 1 OFFSET ?`,
|
|
2854
|
+
[entityId, Math.max(0, Math.trunc(offset))]
|
|
2855
|
+
);
|
|
2856
|
+
return row ? mapEvent(row) : null;
|
|
2857
|
+
}
|
|
2858
|
+
/**
|
|
2859
|
+
* Sum of `summary` lengths (in characters) for events for `entityId`
|
|
2860
|
+
* strictly after the watermark. Pass `wm = null` to sum the entity's whole
|
|
2861
|
+
* event log (no watermark yet). Used by the op-based librarian's token
|
|
2862
|
+
* budget estimator (PR-B) and by the auto-trigger's pending-size check
|
|
2863
|
+
* (PR-C). SQLite's `LENGTH` returns character count for UTF-8 text input.
|
|
2864
|
+
*/
|
|
2865
|
+
async sumSummaryCharsAfter(entityId, wm, tx) {
|
|
2866
|
+
const executor = this.getExecutor(tx);
|
|
2867
|
+
const params = wm ? [entityId, wm.at, wm.at, wm.id] : [entityId];
|
|
2868
|
+
const where = wm ? `entity_id = ? AND ${AFTER_SQL}` : `entity_id = ?`;
|
|
2869
|
+
const row = await executor.getFirstAsync(
|
|
2870
|
+
`SELECT SUM(LENGTH(summary)) as total FROM ${this.prefix}events WHERE ${where}`,
|
|
2871
|
+
params
|
|
2872
|
+
);
|
|
2873
|
+
return row?.total ?? 0;
|
|
2566
2874
|
}
|
|
2567
2875
|
};
|
|
2568
2876
|
|
|
@@ -2691,6 +2999,7 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
2691
2999
|
) >= ?
|
|
2692
3000
|
AND n.source_type NOT IN (${excludeSourceTypesPlaceholders})
|
|
2693
3001
|
AND (? = 0 OR n.lifecycle_status != 'draft')
|
|
3002
|
+
AND ${liveAtSql(opts.live.mode, "n.")}
|
|
2694
3003
|
WHERE w.depth < ?
|
|
2695
3004
|
AND instr(w.visited, ',' || (CASE WHEN e.source_id = w.node_id THEN e.target_id ELSE e.source_id END) || ',') = 0
|
|
2696
3005
|
)
|
|
@@ -2711,6 +3020,8 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
2711
3020
|
minConfidenceRank,
|
|
2712
3021
|
...opts.excludeSourceTypes,
|
|
2713
3022
|
opts.excludeDrafts === true ? 1 : 0,
|
|
3023
|
+
opts.live.t,
|
|
3024
|
+
opts.live.t,
|
|
2714
3025
|
opts.maxDepth,
|
|
2715
3026
|
opts.maxNodes
|
|
2716
3027
|
];
|
|
@@ -2960,6 +3271,37 @@ var MetadataRepository = class extends BaseRepository {
|
|
|
2960
3271
|
[entityId]
|
|
2961
3272
|
);
|
|
2962
3273
|
}
|
|
3274
|
+
/**
|
|
3275
|
+
* Read the op-based librarian's high-water mark. Returns `null` when the
|
|
3276
|
+
* entity has never had its watermark set.
|
|
3277
|
+
*/
|
|
3278
|
+
async getLibrarianWatermark(entityId, tx) {
|
|
3279
|
+
const executor = this.getExecutor(tx);
|
|
3280
|
+
const row = await executor.getFirstAsync(
|
|
3281
|
+
`SELECT librarian_watermark_at, librarian_watermark_id FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
3282
|
+
[entityId]
|
|
3283
|
+
);
|
|
3284
|
+
if (!row || row.librarian_watermark_at == null || row.librarian_watermark_id == null) {
|
|
3285
|
+
return null;
|
|
3286
|
+
}
|
|
3287
|
+
return { at: row.librarian_watermark_at, id: row.librarian_watermark_id };
|
|
3288
|
+
}
|
|
3289
|
+
/**
|
|
3290
|
+
* Write the op-based librarian's high-water mark. Uses `ON CONFLICT`
|
|
3291
|
+
* `DO UPDATE` so existing `memory_checkpoint` / `heal_checkpoint` rows
|
|
3292
|
+
* are preserved verbatim.
|
|
3293
|
+
*/
|
|
3294
|
+
async setLibrarianWatermark(entityId, wm, tx) {
|
|
3295
|
+
const executor = this.getExecutor(tx);
|
|
3296
|
+
await executor.runAsync(
|
|
3297
|
+
`INSERT INTO ${this.prefix}checkpoints (entity_id, memory_checkpoint, heal_checkpoint, librarian_watermark_at, librarian_watermark_id)
|
|
3298
|
+
VALUES (?, 0, 0, ?, ?)
|
|
3299
|
+
ON CONFLICT(entity_id) DO UPDATE SET
|
|
3300
|
+
librarian_watermark_at = excluded.librarian_watermark_at,
|
|
3301
|
+
librarian_watermark_id = excluded.librarian_watermark_id`,
|
|
3302
|
+
[entityId, wm.at, wm.id]
|
|
3303
|
+
);
|
|
3304
|
+
}
|
|
2963
3305
|
// META TABLE METHODS
|
|
2964
3306
|
async getMeta(key, tx) {
|
|
2965
3307
|
const executor = this.getExecutor(tx);
|
|
@@ -3279,6 +3621,16 @@ var _SearchService = class _SearchService {
|
|
|
3279
3621
|
this.staleEntities.add(entityId);
|
|
3280
3622
|
this.staleEpochs.set(entityId, (this.staleEpochs.get(entityId) ?? 0) + 1);
|
|
3281
3623
|
}
|
|
3624
|
+
/**
|
|
3625
|
+
* Runs syncEntries(id, []) for each entity marked stale when this is called,
|
|
3626
|
+
* so only those entities are rebuilt. Each rebuild still takes its own turn
|
|
3627
|
+
* on the sync chain, and like syncEntries this never rejects. With nothing
|
|
3628
|
+
* stale it resolves at once, without waiting on work already on the chain.
|
|
3629
|
+
*/
|
|
3630
|
+
async syncStale() {
|
|
3631
|
+
const ids = [...this.staleEntities];
|
|
3632
|
+
await Promise.all(ids.map((id) => this.syncEntries(id, [])));
|
|
3633
|
+
}
|
|
3282
3634
|
/**
|
|
3283
3635
|
* Clears the parsed vector cache. Useful for mid-loop flush guarantees
|
|
3284
3636
|
* or memory pressure evictions.
|
|
@@ -4482,7 +4834,14 @@ var SEVERITY = {
|
|
|
4482
4834
|
grounding_missing: "warn",
|
|
4483
4835
|
grounding_failed: "warn",
|
|
4484
4836
|
classification_low_confidence: "info",
|
|
4485
|
-
classification_invalid: "warn"
|
|
4837
|
+
classification_invalid: "warn",
|
|
4838
|
+
llm_usage: "info",
|
|
4839
|
+
librarian_gate: "info",
|
|
4840
|
+
librarian_op_rejected: "warn",
|
|
4841
|
+
contradicts_document: "warn",
|
|
4842
|
+
resolve_failed: "warn",
|
|
4843
|
+
read_budget: "info",
|
|
4844
|
+
event_retention_held: "info"
|
|
4486
4845
|
};
|
|
4487
4846
|
var MESSAGE = {
|
|
4488
4847
|
ingest_chunk_failed: "One or more document chunks failed to parse or generate.",
|
|
@@ -4497,7 +4856,14 @@ var MESSAGE = {
|
|
|
4497
4856
|
grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
|
|
4498
4857
|
grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
|
|
4499
4858
|
classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
|
|
4500
|
-
classification_invalid: "A classifier answer was invalid and was not applied."
|
|
4859
|
+
classification_invalid: "A classifier answer was invalid and was not applied.",
|
|
4860
|
+
llm_usage: "An LLM call completed; token usage attached.",
|
|
4861
|
+
librarian_gate: "Librarian gate outcome counts for one pass.",
|
|
4862
|
+
librarian_op_rejected: "A librarian reconciliation op was invalid and the candidate was added instead.",
|
|
4863
|
+
contradicts_document: "A candidate contradicted a document fact and was stored as a draft.",
|
|
4864
|
+
resolve_failed: "Reconciliation failed for a candidate; it was stored as a draft.",
|
|
4865
|
+
read_budget: "A read was packed to a token budget.",
|
|
4866
|
+
event_retention_held: "Event pruning was held back because the librarian has not processed those events yet."
|
|
4501
4867
|
};
|
|
4502
4868
|
var warnedNonFunction = /* @__PURE__ */ new WeakSet();
|
|
4503
4869
|
function copyDetail(detail) {
|
|
@@ -4624,6 +4990,21 @@ function groundingEvidenceBlock(writer, cfg) {
|
|
|
4624
4990
|
return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
|
|
4625
4991
|
"evidence": ["exact substring copied from the SOURCE section"]`;
|
|
4626
4992
|
}
|
|
4993
|
+
var OPS_EXTRACT_SYSTEM_PROMPT = `You are a memory extraction agent. Read the numbered events and extract durable facts about the user or entity, and actionable tasks.
|
|
4994
|
+
Return ONLY a valid JSON object matching this schema:
|
|
4995
|
+
{
|
|
4996
|
+
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative", "source_event": "the event label, e.g. e1", "valid_from": "optional ISO-8601 date the fact became true, only when the events state it" }],
|
|
4997
|
+
"tasks": [{ "description": "string", "priority": "number (0-10)" }]
|
|
4998
|
+
}
|
|
4999
|
+
One fact per claim. Every fact must name the single event it comes from in "source_event". Do not return markdown, just raw JSON.`;
|
|
5000
|
+
var OPS_RESOLVE_SYSTEM_PROMPT = `You are a memory reconciliation agent. Each item has a NEW candidate fact and EXISTING facts labelled n1, n2, ... that look similar to it. Choose exactly one op per item:
|
|
5001
|
+
- "ADD": the candidate is new information; the existing facts are about something else.
|
|
5002
|
+
- "UPDATE": one existing fact states the same thing and the candidate adds detail. Give the merged "title" and "body" for that existing fact.
|
|
5003
|
+
- "SUPERSEDE": the candidate replaces an existing fact that is no longer true (moved, changed jobs, changed preference, reversed decision).
|
|
5004
|
+
- "NOOP": an existing fact already states the candidate.
|
|
5005
|
+
Return ONLY a valid JSON object matching this schema:
|
|
5006
|
+
{ "ops": [{ "item": 0, "op": "ADD|UPDATE|SUPERSEDE|NOOP", "target": "n1", "title": "UPDATE only", "body": "UPDATE only" }] }
|
|
5007
|
+
"target" is required for UPDATE, SUPERSEDE and NOOP and must be one of that item's own labels. Do not return markdown, just raw JSON.`;
|
|
4627
5008
|
|
|
4628
5009
|
// src/utils/healConstants.ts
|
|
4629
5010
|
var HEAL_MAX_ANCHORS = 50;
|
|
@@ -4808,6 +5189,22 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
4808
5189
|
${JSON.stringify(facts, null, 2)}`
|
|
4809
5190
|
};
|
|
4810
5191
|
}
|
|
5192
|
+
/** Ops extract (spec §5.2): events only — no existing facts. */
|
|
5193
|
+
buildOpsExtractPrompt(events, ontologyContext) {
|
|
5194
|
+
const lines = events.map((l) => `${l.label} [${new Date(l.at).toISOString()}] (${l.event.event_type}) ${l.event.summary}`);
|
|
5195
|
+
const grounding = this.groundingFor("librarian");
|
|
5196
|
+
return {
|
|
5197
|
+
systemPrompt: this.appendGrounding(this.appendOntology(OPS_EXTRACT_SYSTEM_PROMPT, ontologyContext), "librarian"),
|
|
5198
|
+
userPrompt: `Events:
|
|
5199
|
+
${lines.join("\n")}`,
|
|
5200
|
+
...grounding ? { groundingCorpus: buildGroundingCorpus(events.map((l) => l.event.summary)) } : {}
|
|
5201
|
+
};
|
|
5202
|
+
}
|
|
5203
|
+
/** Ops resolve (spec §5.4): only each item's own neighbours. */
|
|
5204
|
+
buildOpsResolvePrompt(items) {
|
|
5205
|
+
return { systemPrompt: OPS_RESOLVE_SYSTEM_PROMPT, userPrompt: `Items:
|
|
5206
|
+
${JSON.stringify({ items })}` };
|
|
5207
|
+
}
|
|
4811
5208
|
/**
|
|
4812
5209
|
* The system prompt each writer would send, without hydrating data (spec
|
|
4813
5210
|
* §8.3). `buildSystemPrompt` with no variables hydrates only ontology
|
|
@@ -4872,6 +5269,79 @@ function toGroundingAnchor(anchor) {
|
|
|
4872
5269
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
4873
5270
|
var DEFAULT_CHUNK_OVERLAP = 400;
|
|
4874
5271
|
|
|
5272
|
+
// src/utils/usage.ts
|
|
5273
|
+
function estimateTokens(text) {
|
|
5274
|
+
return Math.ceil(text.length / 4);
|
|
5275
|
+
}
|
|
5276
|
+
var UsageMeter = class {
|
|
5277
|
+
constructor(budget) {
|
|
5278
|
+
this.used = 0;
|
|
5279
|
+
this.estimated = false;
|
|
5280
|
+
if (budget !== void 0 && (typeof budget !== "number" || !Number.isFinite(budget) || budget < 0)) {
|
|
5281
|
+
throw new TypeError("tokenBudget must be a finite number >= 0");
|
|
5282
|
+
}
|
|
5283
|
+
this.budget = budget;
|
|
5284
|
+
}
|
|
5285
|
+
get remaining() {
|
|
5286
|
+
return this.budget === void 0 ? Infinity : Math.max(0, this.budget - this.used);
|
|
5287
|
+
}
|
|
5288
|
+
fits(estimate) {
|
|
5289
|
+
return estimate <= this.remaining;
|
|
5290
|
+
}
|
|
5291
|
+
record(r) {
|
|
5292
|
+
this.used += r.inputTokens + r.outputTokens;
|
|
5293
|
+
if (r.estimated) this.estimated = true;
|
|
5294
|
+
}
|
|
5295
|
+
};
|
|
5296
|
+
var WikiBudgetExhausted = class _WikiBudgetExhausted extends Error {
|
|
5297
|
+
constructor(requiredEstimate, remaining) {
|
|
5298
|
+
super(`LLM call needs ~${requiredEstimate} input tokens; ${remaining} remain in the budget.`);
|
|
5299
|
+
this.name = "WikiBudgetExhausted";
|
|
5300
|
+
this.requiredEstimate = requiredEstimate;
|
|
5301
|
+
this.remaining = remaining;
|
|
5302
|
+
Object.setPrototypeOf(this, _WikiBudgetExhausted.prototype);
|
|
5303
|
+
}
|
|
5304
|
+
};
|
|
5305
|
+
|
|
5306
|
+
// src/utils/llmCall.ts
|
|
5307
|
+
function validUsage(u) {
|
|
5308
|
+
if (!u || typeof u !== "object") return false;
|
|
5309
|
+
const { inputTokens, outputTokens } = u;
|
|
5310
|
+
return [inputTokens, outputTokens].every((n) => typeof n === "number" && Number.isFinite(n) && n >= 0);
|
|
5311
|
+
}
|
|
5312
|
+
async function callLlm(options, ctx, prompts) {
|
|
5313
|
+
const promptEstimate = estimateTokens(prompts.systemPrompt) + estimateTokens(prompts.userPrompt);
|
|
5314
|
+
if (ctx.meter && !ctx.meter.fits(promptEstimate)) {
|
|
5315
|
+
throw new WikiBudgetExhausted(promptEstimate, ctx.meter.remaining);
|
|
5316
|
+
}
|
|
5317
|
+
const provider = options.llmProvider;
|
|
5318
|
+
const params = { systemPrompt: prompts.systemPrompt, userPrompt: prompts.userPrompt };
|
|
5319
|
+
let text;
|
|
5320
|
+
let reported;
|
|
5321
|
+
if (typeof provider.generateTextWithUsage === "function") {
|
|
5322
|
+
const result = await provider.generateTextWithUsage(params);
|
|
5323
|
+
if (!result || typeof result.text !== "string") {
|
|
5324
|
+
throw new TypeError("generateTextWithUsage() must resolve to { text: string, usage? }");
|
|
5325
|
+
}
|
|
5326
|
+
text = result.text;
|
|
5327
|
+
reported = validUsage(result.usage) ? result.usage : void 0;
|
|
5328
|
+
} else {
|
|
5329
|
+
text = await provider.generateText(params);
|
|
5330
|
+
}
|
|
5331
|
+
const record = reported ? { inputTokens: reported.inputTokens, outputTokens: reported.outputTokens, estimated: false } : { inputTokens: promptEstimate, outputTokens: estimateTokens(text), estimated: true };
|
|
5332
|
+
ctx.meter?.record(record);
|
|
5333
|
+
if (options.config?.reportLlmUsage === true) {
|
|
5334
|
+
emitDiagnostic(options, {
|
|
5335
|
+
code: "llm_usage",
|
|
5336
|
+
operation: ctx.operation,
|
|
5337
|
+
trigger: ctx.trigger,
|
|
5338
|
+
entityId: ctx.entityId,
|
|
5339
|
+
detail: { inputTokens: record.inputTokens, outputTokens: record.outputTokens, estimated: record.estimated }
|
|
5340
|
+
});
|
|
5341
|
+
}
|
|
5342
|
+
return text;
|
|
5343
|
+
}
|
|
5344
|
+
|
|
4875
5345
|
// src/services/IngestionService.ts
|
|
4876
5346
|
var LLM_PASS_DIAGNOSTIC_CODES = ["ingest_chunk_failed", "fact_rejected", "fact_deduplicated"];
|
|
4877
5347
|
function dedupDiagnostic(entityId, sourceRef, locator) {
|
|
@@ -4948,7 +5418,7 @@ var IngestionService = class {
|
|
|
4948
5418
|
ontologyContext
|
|
4949
5419
|
);
|
|
4950
5420
|
try {
|
|
4951
|
-
const responseText = await this.options
|
|
5421
|
+
const responseText = await callLlm(this.options, { operation: "ingest", entityId, trigger: "call" }, { systemPrompt, userPrompt });
|
|
4952
5422
|
const result2 = parseJsonResponse(responseText);
|
|
4953
5423
|
const rawFacts = Array.isArray(result2.facts) ? result2.facts : [];
|
|
4954
5424
|
const facts = [];
|
|
@@ -5604,6 +6074,8 @@ async function runBatched(args) {
|
|
|
5604
6074
|
const results = [];
|
|
5605
6075
|
const skipped = [];
|
|
5606
6076
|
let batches = 0;
|
|
6077
|
+
const settled = /* @__PURE__ */ new Set();
|
|
6078
|
+
let budgetStop;
|
|
5607
6079
|
let batchSize = initialBatchSize(maxOutputTokens);
|
|
5608
6080
|
const trim = async (candidate) => {
|
|
5609
6081
|
const whole = await buildPrompt(candidate);
|
|
@@ -5637,6 +6109,7 @@ async function runBatched(args) {
|
|
|
5637
6109
|
} else {
|
|
5638
6110
|
const reason = fromCall && !isTruncationError(err) && !isConfigError(err) ? "call_error" : "non_convergent";
|
|
5639
6111
|
skipped.push({ item: batch[0], reason });
|
|
6112
|
+
settled.add(batch[0]);
|
|
5640
6113
|
onSkip?.(batch[0], err);
|
|
5641
6114
|
}
|
|
5642
6115
|
return;
|
|
@@ -5645,7 +6118,7 @@ async function runBatched(args) {
|
|
|
5645
6118
|
const mid = Math.ceil(batch.length / 2);
|
|
5646
6119
|
if (mid < batchSize) batchSize = mid;
|
|
5647
6120
|
let i = 0;
|
|
5648
|
-
while (i < batch.length) {
|
|
6121
|
+
while (i < batch.length && !budgetStop) {
|
|
5649
6122
|
const size = Math.min(batchSize, batch.length - i);
|
|
5650
6123
|
const trimmed = await trim(batch.slice(i, i + size));
|
|
5651
6124
|
await attempt(trimmed.batch, trimmed.prompts);
|
|
@@ -5653,6 +6126,7 @@ async function runBatched(args) {
|
|
|
5653
6126
|
}
|
|
5654
6127
|
};
|
|
5655
6128
|
const attempt = async (batch, prebuilt, attemptLevel = 0) => {
|
|
6129
|
+
if (budgetStop) return;
|
|
5656
6130
|
if (batch.length === 0) return;
|
|
5657
6131
|
const prompts = prebuilt && attemptLevel === 0 ? prebuilt : await buildPrompt(batch, attemptLevel);
|
|
5658
6132
|
batches++;
|
|
@@ -5660,6 +6134,10 @@ async function runBatched(args) {
|
|
|
5660
6134
|
try {
|
|
5661
6135
|
responseText = await call(prompts);
|
|
5662
6136
|
} catch (err) {
|
|
6137
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
6138
|
+
budgetStop = { requiredEstimate: err.requiredEstimate };
|
|
6139
|
+
return;
|
|
6140
|
+
}
|
|
5663
6141
|
await onFailure(batch, err, attemptLevel, true);
|
|
5664
6142
|
return;
|
|
5665
6143
|
}
|
|
@@ -5671,96 +6149,852 @@ async function runBatched(args) {
|
|
|
5671
6149
|
return;
|
|
5672
6150
|
}
|
|
5673
6151
|
results.push(result);
|
|
6152
|
+
for (const item of batch) settled.add(item);
|
|
5674
6153
|
};
|
|
5675
6154
|
let index = 0;
|
|
5676
|
-
while (index < items.length) {
|
|
6155
|
+
while (index < items.length && !budgetStop) {
|
|
5677
6156
|
const { batch, prompts } = await trim(items.slice(index, index + batchSize));
|
|
5678
6157
|
index += batch.length;
|
|
5679
6158
|
await attempt(batch, prompts, 0);
|
|
5680
6159
|
}
|
|
5681
|
-
|
|
6160
|
+
const unattempted = budgetStop ? items.filter((item) => !settled.has(item)) : [];
|
|
6161
|
+
return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
|
|
5682
6162
|
}
|
|
5683
6163
|
|
|
5684
|
-
// src/services/
|
|
6164
|
+
// src/services/librarian/constants.ts
|
|
5685
6165
|
var FUZZY_THRESHOLD = 0.5;
|
|
5686
6166
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
5687
|
-
|
|
5688
|
-
|
|
5689
|
-
var
|
|
5690
|
-
|
|
5691
|
-
|
|
5692
|
-
|
|
5693
|
-
|
|
5694
|
-
|
|
5695
|
-
|
|
5696
|
-
|
|
5697
|
-
|
|
5698
|
-
|
|
5699
|
-
|
|
5700
|
-
|
|
5701
|
-
|
|
5702
|
-
|
|
5703
|
-
|
|
5704
|
-
const
|
|
5705
|
-
|
|
5706
|
-
|
|
5707
|
-
|
|
5708
|
-
|
|
5709
|
-
|
|
5710
|
-
|
|
5711
|
-
var SKIP_ERROR_LOG_CHARS = 4096;
|
|
5712
|
-
var formatSkipError = (err) => {
|
|
5713
|
-
let base;
|
|
5714
|
-
let isErrorLike = false;
|
|
6167
|
+
|
|
6168
|
+
// src/services/librarian/legacy.ts
|
|
6169
|
+
var runLegacyLibrarianPass = async (deps, ctx) => {
|
|
6170
|
+
const { entityId, trigger } = ctx;
|
|
6171
|
+
const promptOverride = ctx.promptOverride;
|
|
6172
|
+
const events = await deps.eventRepo.getRecent(entityId, 50);
|
|
6173
|
+
const currentFactsRows = await deps.entryRepo.findRecentByEntityId(entityId, 100);
|
|
6174
|
+
const currentFacts = currentFactsRows.map((f) => {
|
|
6175
|
+
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
6176
|
+
return {
|
|
6177
|
+
...rest,
|
|
6178
|
+
tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags
|
|
6179
|
+
};
|
|
6180
|
+
});
|
|
6181
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
6182
|
+
const promptEvents = events.reverse();
|
|
6183
|
+
const librarianGrounding = deps.promptService.groundingFor("librarian");
|
|
6184
|
+
const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = deps.promptService.buildLibrarianPrompt(
|
|
6185
|
+
promptEvents,
|
|
6186
|
+
currentFacts,
|
|
6187
|
+
promptOverride,
|
|
6188
|
+
ontologyContext
|
|
6189
|
+
);
|
|
6190
|
+
let responseText;
|
|
5715
6191
|
try {
|
|
5716
|
-
|
|
5717
|
-
|
|
5718
|
-
|
|
5719
|
-
|
|
5720
|
-
|
|
5721
|
-
}
|
|
5722
|
-
|
|
5723
|
-
|
|
5724
|
-
|
|
5725
|
-
|
|
5726
|
-
|
|
6192
|
+
responseText = await callLlm(
|
|
6193
|
+
deps.options,
|
|
6194
|
+
{ operation: "librarian", entityId, trigger, meter: ctx.meter },
|
|
6195
|
+
{ systemPrompt, userPrompt }
|
|
6196
|
+
);
|
|
6197
|
+
} catch (err) {
|
|
6198
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
6199
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
6200
|
+
}
|
|
6201
|
+
throw err;
|
|
6202
|
+
}
|
|
6203
|
+
const result = parseJsonResponse(responseText);
|
|
6204
|
+
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
6205
|
+
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
6206
|
+
const ontologyUpdates = result.ontology_updates;
|
|
6207
|
+
const diagBuffer = new DiagnosticBuffer();
|
|
6208
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
6209
|
+
const validFacts = [];
|
|
6210
|
+
const validFactItemIndexes = [];
|
|
6211
|
+
facts.forEach((raw, itemIndex) => {
|
|
6212
|
+
const valid = validateFact(raw);
|
|
6213
|
+
if (valid) {
|
|
6214
|
+
validFacts.push(valid);
|
|
6215
|
+
validFactItemIndexes.push(itemIndex);
|
|
6216
|
+
} else {
|
|
6217
|
+
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
5727
6218
|
}
|
|
5728
|
-
}
|
|
5729
|
-
|
|
5730
|
-
|
|
5731
|
-
|
|
5732
|
-
|
|
5733
|
-
|
|
5734
|
-
};
|
|
5735
|
-
|
|
5736
|
-
|
|
5737
|
-
|
|
5738
|
-
|
|
5739
|
-
|
|
5740
|
-
|
|
5741
|
-
|
|
5742
|
-
|
|
5743
|
-
|
|
5744
|
-
|
|
5745
|
-
|
|
5746
|
-
|
|
5747
|
-
|
|
5748
|
-
|
|
5749
|
-
|
|
5750
|
-
|
|
5751
|
-
|
|
5752
|
-
|
|
5753
|
-
|
|
5754
|
-
|
|
5755
|
-
|
|
5756
|
-
|
|
5757
|
-
|
|
5758
|
-
|
|
6219
|
+
});
|
|
6220
|
+
const validTasks = [];
|
|
6221
|
+
tasks.forEach((raw, itemIndex) => {
|
|
6222
|
+
const valid = validateTask(raw);
|
|
6223
|
+
if (valid) validTasks.push(valid);
|
|
6224
|
+
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
6225
|
+
});
|
|
6226
|
+
const now = Date.now();
|
|
6227
|
+
const insertedFacts = [];
|
|
6228
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
6229
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
6230
|
+
if (mode === "emergent" && ontologyUpdates && deps.ontologyService) {
|
|
6231
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
6232
|
+
}
|
|
6233
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
6234
|
+
for (const existing of currentFactsRows) {
|
|
6235
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
6236
|
+
id: existing.id,
|
|
6237
|
+
okf_type: existing.okf_type ?? null
|
|
6238
|
+
});
|
|
6239
|
+
}
|
|
6240
|
+
const factsForDedupe = await deps.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
6241
|
+
const pendingEdges = [];
|
|
6242
|
+
for (const [k, fact] of validFacts.entries()) {
|
|
6243
|
+
const newTokens = titleTokens(fact.title);
|
|
6244
|
+
let skip = false;
|
|
6245
|
+
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
6246
|
+
for (const existing of factsForDedupe) {
|
|
6247
|
+
if (existing.source_type !== "librarian_inferred") continue;
|
|
6248
|
+
const existingTokens = titleTokens(existing.title);
|
|
6249
|
+
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
6250
|
+
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
6251
|
+
skip = true;
|
|
6252
|
+
break;
|
|
6253
|
+
}
|
|
6254
|
+
}
|
|
6255
|
+
}
|
|
6256
|
+
}
|
|
6257
|
+
if (skip) {
|
|
6258
|
+
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
6259
|
+
continue;
|
|
6260
|
+
}
|
|
6261
|
+
const ontologyFact = fact;
|
|
6262
|
+
const id = generateId("fact_");
|
|
6263
|
+
const validationDrops = [];
|
|
6264
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
6265
|
+
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
6266
|
+
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
6267
|
+
if (grounding?.diagnostic) {
|
|
6268
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
6269
|
+
}
|
|
6270
|
+
const factObj = {
|
|
6271
|
+
id,
|
|
6272
|
+
entity_id: entityId,
|
|
6273
|
+
title: fact.title,
|
|
6274
|
+
body: fact.body,
|
|
6275
|
+
tags: fact.tags,
|
|
6276
|
+
confidence: fact.confidence,
|
|
6277
|
+
source_type: "librarian_inferred",
|
|
6278
|
+
source_hash: null,
|
|
6279
|
+
source_ref: null,
|
|
6280
|
+
created_at: now,
|
|
6281
|
+
updated_at: now,
|
|
6282
|
+
last_accessed_at: null,
|
|
6283
|
+
access_count: 0,
|
|
6284
|
+
deleted_at: null,
|
|
6285
|
+
okf_type: normalized.okf_type,
|
|
6286
|
+
...grounding?.trust
|
|
6287
|
+
};
|
|
6288
|
+
await deps.entryRepo.upsert(factObj, tx);
|
|
6289
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
6290
|
+
factsForDedupe.push(factObj);
|
|
6291
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
6292
|
+
if (normalized.edges.length > 0) {
|
|
6293
|
+
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
6294
|
+
}
|
|
6295
|
+
}
|
|
6296
|
+
for (const item of pendingEdges) {
|
|
6297
|
+
const resolveDrops = [];
|
|
6298
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
6299
|
+
entityId,
|
|
6300
|
+
item.sourceId,
|
|
6301
|
+
item.sourceType,
|
|
6302
|
+
item.edges ?? [],
|
|
6303
|
+
manifest,
|
|
6304
|
+
titleIndex,
|
|
6305
|
+
tx,
|
|
6306
|
+
now,
|
|
6307
|
+
resolveDrops
|
|
6308
|
+
);
|
|
6309
|
+
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
6310
|
+
}
|
|
6311
|
+
for (const task of validTasks) {
|
|
6312
|
+
const id = generateId("task_");
|
|
6313
|
+
const taskObj = {
|
|
6314
|
+
id,
|
|
6315
|
+
entity_id: entityId,
|
|
6316
|
+
description: task.description,
|
|
6317
|
+
status: "pending",
|
|
6318
|
+
priority: task.priority,
|
|
6319
|
+
created_at: now,
|
|
6320
|
+
updated_at: now,
|
|
6321
|
+
resolved_at: null,
|
|
6322
|
+
deleted_at: null
|
|
6323
|
+
};
|
|
6324
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
6325
|
+
}
|
|
6326
|
+
});
|
|
6327
|
+
diagBuffer.flush(deps.options);
|
|
6328
|
+
await deps.searchService.syncEntries(entityId, insertedFacts.map((f) => f.id));
|
|
6329
|
+
for (const fact of insertedFacts) {
|
|
6330
|
+
await deps.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
6331
|
+
}
|
|
6332
|
+
deps.searchService.evictCache(entityId);
|
|
6333
|
+
const processedThrough = events.length > 0 ? { at: events[events.length - 1].created_at, id: events[events.length - 1].id } : null;
|
|
6334
|
+
const out = { processedThrough, factsWritten: insertedFacts.length };
|
|
6335
|
+
return out;
|
|
6336
|
+
};
|
|
6337
|
+
|
|
6338
|
+
// src/services/librarian/ops/text.ts
|
|
6339
|
+
var OPS_NEIGHBOUR_BODY_CHARS = 400;
|
|
6340
|
+
function normalizeFactText(title, body) {
|
|
6341
|
+
return `${title} ${body}`.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
6342
|
+
}
|
|
6343
|
+
function labelEvents(events) {
|
|
6344
|
+
return events.map((event, i) => ({ label: `e${i + 1}`, event, at: event.occurred_at ?? event.created_at }));
|
|
6345
|
+
}
|
|
6346
|
+
function parseValidFrom(v) {
|
|
6347
|
+
if (typeof v === "number") return Number.isFinite(v) && v >= 0 ? Math.trunc(v) : void 0;
|
|
6348
|
+
if (typeof v === "string") {
|
|
6349
|
+
const t = Date.parse(v);
|
|
6350
|
+
return Number.isFinite(t) && t >= 0 ? t : void 0;
|
|
6351
|
+
}
|
|
6352
|
+
return void 0;
|
|
6353
|
+
}
|
|
6354
|
+
|
|
6355
|
+
// src/services/librarian/ops/extract.ts
|
|
6356
|
+
var OPS_MAX_EVENTS_PER_PASS = 50;
|
|
6357
|
+
var OPS_MAX_EXTRACT_CHARS = 24e3;
|
|
6358
|
+
var RESOLVE_RESERVE_FACTOR = 2;
|
|
6359
|
+
var estimatePrompt = (p) => estimateTokens(p.systemPrompt) + estimateTokens(p.userPrompt);
|
|
6360
|
+
async function selectBatch(deps, entityId, wm, meter) {
|
|
6361
|
+
const events = await deps.eventRepo.getAfter(entityId, wm, OPS_MAX_EVENTS_PER_PASS);
|
|
6362
|
+
if (events.length === 0) return { batch: [] };
|
|
6363
|
+
let n = 0;
|
|
6364
|
+
let chars = 0;
|
|
6365
|
+
for (const e of events) {
|
|
6366
|
+
if (n > 0 && chars + e.summary.length > OPS_MAX_EXTRACT_CHARS) break;
|
|
6367
|
+
chars += e.summary.length;
|
|
6368
|
+
n++;
|
|
6369
|
+
}
|
|
6370
|
+
let batch = labelEvents(events.slice(0, n));
|
|
6371
|
+
if (meter && meter.budget !== void 0) {
|
|
6372
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
6373
|
+
for (; ; ) {
|
|
6374
|
+
const need = RESOLVE_RESERVE_FACTOR * estimatePrompt(deps.promptService.buildOpsExtractPrompt(batch, ontologyContext));
|
|
6375
|
+
if (meter.fits(need)) break;
|
|
6376
|
+
if (batch.length === 1) return { budgetStop: { requiredEstimate: need } };
|
|
6377
|
+
batch = labelEvents(events.slice(0, Math.max(1, Math.floor(batch.length / 2))));
|
|
6378
|
+
}
|
|
6379
|
+
}
|
|
6380
|
+
return { batch };
|
|
6381
|
+
}
|
|
6382
|
+
async function runExtract(deps, ctx, batch) {
|
|
6383
|
+
const { entityId, trigger } = ctx;
|
|
6384
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
6385
|
+
const prompt = deps.promptService.buildOpsExtractPrompt(batch, ontologyContext);
|
|
6386
|
+
const text = await callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompt);
|
|
6387
|
+
const parsed = parseJsonResponse(text);
|
|
6388
|
+
const base = { entityId, operation: "librarian", trigger };
|
|
6389
|
+
const diagnostics = [];
|
|
6390
|
+
const byLabel = new Map(batch.map((l) => [l.label, l]));
|
|
6391
|
+
const newestAt = batch[batch.length - 1].at;
|
|
6392
|
+
const candidates = [];
|
|
6393
|
+
(Array.isArray(parsed.facts) ? parsed.facts : []).forEach((raw, index) => {
|
|
6394
|
+
const fact = validateFact(raw);
|
|
6395
|
+
if (!fact) {
|
|
6396
|
+
diagnostics.push({ ...base, code: "fact_rejected", detail: { itemIndex: index, reason: factRejectionReason(raw) } });
|
|
6397
|
+
return;
|
|
6398
|
+
}
|
|
6399
|
+
const r = raw;
|
|
6400
|
+
const sourceLabel = typeof r.source_event === "string" && byLabel.has(r.source_event) ? r.source_event : null;
|
|
6401
|
+
const validFrom = parseValidFrom(r.valid_from) ?? (sourceLabel ? byLabel.get(sourceLabel).at : newestAt);
|
|
6402
|
+
candidates.push({ index, fact, sourceLabel, validFrom });
|
|
6403
|
+
});
|
|
6404
|
+
const tasks = [];
|
|
6405
|
+
(Array.isArray(parsed.tasks) ? parsed.tasks : []).forEach((raw, index) => {
|
|
6406
|
+
const task = validateTask(raw);
|
|
6407
|
+
if (task) tasks.push(task);
|
|
6408
|
+
else diagnostics.push({ ...base, code: "task_rejected", detail: { itemIndex: index, reason: taskRejectionReason(raw) } });
|
|
6409
|
+
});
|
|
6410
|
+
return {
|
|
6411
|
+
batch,
|
|
6412
|
+
candidates,
|
|
6413
|
+
tasks,
|
|
6414
|
+
...parsed.ontology_updates ? { ontologyUpdates: parsed.ontology_updates } : {},
|
|
6415
|
+
groundingCorpus: prompt.groundingCorpus ?? null,
|
|
6416
|
+
diagnostics
|
|
6417
|
+
};
|
|
6418
|
+
}
|
|
6419
|
+
|
|
6420
|
+
// src/services/librarian/ops/gate.ts
|
|
6421
|
+
var clamp01 = (v, d) => typeof v === "number" && Number.isFinite(v) ? Math.min(1, Math.max(0, v)) : d;
|
|
6422
|
+
function resolveGateConfig(cfg) {
|
|
6423
|
+
const kRaw = cfg?.k;
|
|
6424
|
+
const k = typeof kRaw === "number" && Number.isFinite(kRaw) ? Math.min(20, Math.max(1, Math.trunc(kRaw))) : 5;
|
|
6425
|
+
const dupThreshold = clamp01(cfg?.dupThreshold, 0.89);
|
|
6426
|
+
const novelThreshold = clamp01(cfg?.novelThreshold, 0.3);
|
|
6427
|
+
if (novelThreshold > dupThreshold) throw new TypeError("librarian.gate: novelThreshold must be <= dupThreshold");
|
|
6428
|
+
return { k, dupThreshold, novelThreshold };
|
|
6429
|
+
}
|
|
6430
|
+
function titlesMatch(a, b) {
|
|
6431
|
+
const ta = titleTokens(a);
|
|
6432
|
+
const tb = titleTokens(b);
|
|
6433
|
+
if (ta.size < MIN_TOKENS_TO_QUALIFY || tb.size < MIN_TOKENS_TO_QUALIFY) return false;
|
|
6434
|
+
return jaccardScore(ta, tb) >= FUZZY_THRESHOLD;
|
|
6435
|
+
}
|
|
6436
|
+
function classifyCandidate(candidate, neighbours, cfg, mode) {
|
|
6437
|
+
const key = normalizeFactText(candidate.title, candidate.body);
|
|
6438
|
+
const same = neighbours.find((n) => normalizeFactText(n.title, n.body) === key);
|
|
6439
|
+
if (same) return { kind: "noop", target: same.id };
|
|
6440
|
+
if (neighbours.length === 0) return { kind: "add" };
|
|
6441
|
+
if (mode === "keyword") return { kind: "ambiguous" };
|
|
6442
|
+
const best = neighbours[0];
|
|
6443
|
+
const score = best.score ?? 0;
|
|
6444
|
+
if (score >= cfg.dupThreshold && titlesMatch(candidate.title, best.title)) return { kind: "noop", target: best.id };
|
|
6445
|
+
if (score < cfg.novelThreshold) return { kind: "add" };
|
|
6446
|
+
return { kind: "ambiguous" };
|
|
6447
|
+
}
|
|
6448
|
+
async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
6449
|
+
const { entityId, trigger } = ctx;
|
|
6450
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6451
|
+
const unique = [];
|
|
6452
|
+
const inBatchDuplicates = [];
|
|
6453
|
+
for (const c of candidates) {
|
|
6454
|
+
const key = normalizeFactText(c.fact.title, c.fact.body);
|
|
6455
|
+
if (seen.has(key)) inBatchDuplicates.push(c);
|
|
6456
|
+
else {
|
|
6457
|
+
seen.add(key);
|
|
6458
|
+
unique.push(c);
|
|
6459
|
+
}
|
|
6460
|
+
}
|
|
6461
|
+
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
6462
|
+
const canEmbed = typeof deps.options.llmProvider.embed === "function";
|
|
6463
|
+
const stored = [];
|
|
6464
|
+
if (canEmbed && unique.length > 0) {
|
|
6465
|
+
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
6466
|
+
if (nonLive.has(row.id)) continue;
|
|
6467
|
+
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
6468
|
+
if (vec) stored.push({ id: row.id, vec });
|
|
6469
|
+
}
|
|
6470
|
+
}
|
|
6471
|
+
const gated = [];
|
|
6472
|
+
for (const c of unique) {
|
|
6473
|
+
let ranked = null;
|
|
6474
|
+
let vector = null;
|
|
6475
|
+
let mode = "keyword";
|
|
6476
|
+
if (canEmbed) {
|
|
6477
|
+
const r = await deps.embeddingService.embedTextForFact(
|
|
6478
|
+
{ id: "candidate", entity_id: entityId, title: c.fact.title, body: c.fact.body, tags: c.fact.tags },
|
|
6479
|
+
void 0,
|
|
6480
|
+
{ markFailures: false }
|
|
6481
|
+
);
|
|
6482
|
+
if (r.ok) {
|
|
6483
|
+
vector = r.vector;
|
|
6484
|
+
if (stored.length > 0 && stored.every((s) => s.vec.length === r.vector.length)) {
|
|
6485
|
+
ranked = stored.map((s) => ({ id: s.id, score: cosineSimilarity(r.vector, s.vec) })).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)).slice(0, cfg.k);
|
|
6486
|
+
mode = "vector";
|
|
6487
|
+
}
|
|
6488
|
+
}
|
|
6489
|
+
}
|
|
6490
|
+
if (!ranked) {
|
|
6491
|
+
ranked = deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
|
|
6492
|
+
}
|
|
6493
|
+
const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
|
|
6494
|
+
const byId = new Map(facts.map((f) => [f.id, f]));
|
|
6495
|
+
const neighbours = ranked.filter((r) => byId.has(r.id)).map((r, i) => {
|
|
6496
|
+
const f = byId.get(r.id);
|
|
6497
|
+
return { id: f.id, ref: `n${i + 1}`, title: f.title, body: f.body, source_type: f.source_type, score: r.score };
|
|
6498
|
+
});
|
|
6499
|
+
gated.push({ candidate: c, neighbours, decision: classifyCandidate(c.fact, neighbours, cfg, mode), vector });
|
|
6500
|
+
}
|
|
6501
|
+
const count = (k) => gated.filter((g) => g.decision.kind === k).length;
|
|
6502
|
+
const gateDiagnostic = {
|
|
6503
|
+
code: "librarian_gate",
|
|
6504
|
+
operation: "librarian",
|
|
6505
|
+
trigger,
|
|
6506
|
+
entityId,
|
|
6507
|
+
detail: { gateNoop: count("noop"), gateAdd: count("add"), gateAmbiguous: count("ambiguous") }
|
|
6508
|
+
};
|
|
6509
|
+
return { gated, inBatchDuplicates, gateDiagnostic };
|
|
6510
|
+
}
|
|
6511
|
+
|
|
6512
|
+
// src/services/librarian/ops/resolve.ts
|
|
6513
|
+
var OPS_RESOLVE_MAX_PROMPT_CHARS = 4e4;
|
|
6514
|
+
function validateOp(raw, aliases) {
|
|
6515
|
+
if (!raw || typeof raw !== "object") return null;
|
|
6516
|
+
const r = raw;
|
|
6517
|
+
if (r.op === "ADD") return { op: "ADD" };
|
|
6518
|
+
if (r.op !== "UPDATE" && r.op !== "SUPERSEDE" && r.op !== "NOOP") return null;
|
|
6519
|
+
const targetId = typeof r.target === "string" ? aliases.get(r.target) : void 0;
|
|
6520
|
+
if (!targetId) return null;
|
|
6521
|
+
if (r.op === "NOOP") return { op: "NOOP", targetId };
|
|
6522
|
+
if (r.op === "SUPERSEDE") {
|
|
6523
|
+
const validFrom = parseValidFrom(r.valid_from);
|
|
6524
|
+
return validFrom === void 0 ? { op: "SUPERSEDE", targetId } : { op: "SUPERSEDE", targetId, validFrom };
|
|
6525
|
+
}
|
|
6526
|
+
const update = { op: "UPDATE", targetId };
|
|
6527
|
+
if (typeof r.title === "string" && r.title.trim() !== "") update.title = clip(r.title, 80);
|
|
6528
|
+
if (typeof r.body === "string" && r.body.trim() !== "") update.body = clip(r.body, 800);
|
|
6529
|
+
return update;
|
|
6530
|
+
}
|
|
6531
|
+
var aliasMapOf = (g) => new Map(g.neighbours.map((n) => [n.ref, n.id]));
|
|
6532
|
+
async function runResolve(deps, ctx, ambiguous) {
|
|
6533
|
+
const out = { ops: /* @__PURE__ */ new Map(), rejected: [], failed: [] };
|
|
6534
|
+
if (ambiguous.length === 0) return out;
|
|
6535
|
+
const { entityId, trigger } = ctx;
|
|
6536
|
+
const outcome = await runBatched({
|
|
6537
|
+
items: ambiguous,
|
|
6538
|
+
buildPrompt: (batch) => deps.promptService.buildOpsResolvePrompt(
|
|
6539
|
+
batch.map((g, i) => ({
|
|
6540
|
+
item: i,
|
|
6541
|
+
candidate: { title: g.candidate.fact.title, body: g.candidate.fact.body },
|
|
6542
|
+
existing: g.neighbours.map((n) => ({
|
|
6543
|
+
ref: n.ref,
|
|
6544
|
+
title: n.title,
|
|
6545
|
+
body: safeSlice(n.body, 0, OPS_NEIGHBOUR_BODY_CHARS)
|
|
6546
|
+
}))
|
|
6547
|
+
}))
|
|
6548
|
+
),
|
|
6549
|
+
call: (prompts) => callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompts),
|
|
6550
|
+
parse: (text, batch) => {
|
|
6551
|
+
const parsed = parseJsonResponse(text);
|
|
6552
|
+
if (!Array.isArray(parsed.ops)) throw new Error("resolve: ops missing");
|
|
6553
|
+
return { batch, ops: parsed.ops };
|
|
6554
|
+
},
|
|
6555
|
+
maxPromptChars: OPS_RESOLVE_MAX_PROMPT_CHARS,
|
|
6556
|
+
maxOutputTokens: deps.options.llmProvider.maxOutputTokens
|
|
6557
|
+
});
|
|
6558
|
+
if (outcome.budgetStop) return { ...out, budgetStop: outcome.budgetStop };
|
|
6559
|
+
for (const { batch, ops } of outcome.results) {
|
|
6560
|
+
const byItem = /* @__PURE__ */ new Map();
|
|
6561
|
+
for (const raw of ops) {
|
|
6562
|
+
const item = raw?.item;
|
|
6563
|
+
if (typeof item === "number" && Number.isInteger(item) && item >= 0 && item < batch.length && !byItem.has(item)) {
|
|
6564
|
+
byItem.set(item, raw);
|
|
6565
|
+
}
|
|
6566
|
+
}
|
|
6567
|
+
batch.forEach((g, i) => {
|
|
6568
|
+
const op = validateOp(byItem.get(i), aliasMapOf(g));
|
|
6569
|
+
if (op) out.ops.set(g, op);
|
|
6570
|
+
else out.rejected.push(g);
|
|
6571
|
+
});
|
|
6572
|
+
}
|
|
6573
|
+
if (outcome.skipped.some((s) => s.reason === "call_error")) {
|
|
6574
|
+
throw new Error("resolve: transient provider error; retry the pass");
|
|
6575
|
+
}
|
|
6576
|
+
for (const s of outcome.skipped) out.failed.push(s.item);
|
|
6577
|
+
return out;
|
|
6578
|
+
}
|
|
6579
|
+
|
|
6580
|
+
// src/services/TemporalService.ts
|
|
6581
|
+
var HISTORY_MAX_DEPTH = 100;
|
|
6582
|
+
var TemporalService = class {
|
|
6583
|
+
constructor(db, options, entryRepo, searchService, embeddingService) {
|
|
6584
|
+
this.db = db;
|
|
6585
|
+
this.options = options;
|
|
6586
|
+
this.entryRepo = entryRepo;
|
|
6587
|
+
this.searchService = searchService;
|
|
6588
|
+
this.embeddingService = embeddingService;
|
|
6589
|
+
}
|
|
6590
|
+
/**
|
|
6591
|
+
* Link `oldId` → `newId` inside the caller's transaction. `newId` must
|
|
6592
|
+
* already exist. Throws WikiSupersedeError on any rule violation; the
|
|
6593
|
+
* caller's transaction then rolls back.
|
|
6594
|
+
*/
|
|
6595
|
+
async supersedeInTx(tx, entityId, oldId, newId, t, now) {
|
|
6596
|
+
const old = await this.entryRepo.findTemporalRow(oldId, tx);
|
|
6597
|
+
if (!old || old.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
6598
|
+
if (old.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
6599
|
+
if (old.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
6600
|
+
if (old.source_type === "immutable_document") throw new WikiSupersedeError("immutable_target");
|
|
6601
|
+
if (newId === oldId) throw new WikiSupersedeError("cycle");
|
|
6602
|
+
const repl = await this.entryRepo.findTemporalRow(newId, tx);
|
|
6603
|
+
if (!repl || repl.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
6604
|
+
if (repl.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
6605
|
+
if (repl.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
6606
|
+
const { predecessors } = await this.entryRepo.findSupersessionChainIds(entityId, oldId, HISTORY_MAX_DEPTH, tx);
|
|
6607
|
+
if (predecessors.includes(newId)) throw new WikiSupersedeError("cycle");
|
|
6608
|
+
await this.entryRepo.setTemporal(oldId, entityId, { valid_to: t, superseded_by: newId, superseded_at: now }, tx);
|
|
6609
|
+
if (repl.valid_from == null) {
|
|
6610
|
+
await this.entryRepo.setTemporal(newId, entityId, { valid_from: t }, tx);
|
|
6611
|
+
}
|
|
6612
|
+
}
|
|
6613
|
+
async supersede(entityId, oldId, replacement, options) {
|
|
6614
|
+
const now = Date.now();
|
|
6615
|
+
const t = options?.validFrom === void 0 ? now : assertEpochMs("options.validFrom", options.validFrom);
|
|
6616
|
+
let created = null;
|
|
6617
|
+
if (typeof replacement !== "string") {
|
|
6618
|
+
if (typeof replacement?.title !== "string" || replacement.title.trim() === "" || typeof replacement.body !== "string") {
|
|
6619
|
+
throw new TypeError("supersede(): replacement must be a fact id or { title, body }");
|
|
6620
|
+
}
|
|
6621
|
+
created = {
|
|
6622
|
+
id: generateId("fact_"),
|
|
6623
|
+
entity_id: entityId,
|
|
6624
|
+
title: replacement.title,
|
|
6625
|
+
body: replacement.body,
|
|
6626
|
+
tags: Array.isArray(replacement.tags) ? replacement.tags : [],
|
|
6627
|
+
confidence: replacement.confidence ?? "certain",
|
|
6628
|
+
source_type: replacement.source_type ?? "user_stated",
|
|
6629
|
+
source_hash: null,
|
|
6630
|
+
source_ref: null,
|
|
6631
|
+
created_at: now,
|
|
6632
|
+
updated_at: now,
|
|
6633
|
+
last_accessed_at: null,
|
|
6634
|
+
access_count: 0,
|
|
6635
|
+
deleted_at: null
|
|
6636
|
+
};
|
|
6637
|
+
}
|
|
6638
|
+
const newId = typeof replacement === "string" ? replacement : created.id;
|
|
6639
|
+
const newValidFrom = typeof replacement !== "string" && replacement.valid_from !== void 0 ? assertEpochMs("replacement.valid_from", replacement.valid_from) : void 0;
|
|
6640
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
6641
|
+
if (created) {
|
|
6642
|
+
await this.entryRepo.upsert(created, tx);
|
|
6643
|
+
if (newValidFrom !== void 0) {
|
|
6644
|
+
await this.entryRepo.setTemporal(created.id, entityId, { valid_from: newValidFrom }, tx);
|
|
6645
|
+
}
|
|
6646
|
+
}
|
|
6647
|
+
await this.supersedeInTx(tx, entityId, oldId, newId, t, now);
|
|
6648
|
+
});
|
|
6649
|
+
await this.searchService.syncEntries(entityId, [oldId, newId]);
|
|
6650
|
+
if (created) {
|
|
6651
|
+
await this.embeddingService.embedFact(
|
|
6652
|
+
{ id: created.id, entity_id: entityId, title: created.title, body: created.body, tags: created.tags },
|
|
6653
|
+
{ operation: "supersede", trigger: "call" }
|
|
6654
|
+
);
|
|
6655
|
+
}
|
|
6656
|
+
this.searchService.evictCache(entityId);
|
|
6657
|
+
return { newId };
|
|
6658
|
+
}
|
|
6659
|
+
async history(entityId, factId) {
|
|
6660
|
+
const start = await this.entryRepo.findTemporalRow(factId);
|
|
6661
|
+
if (!start || start.entity_id !== entityId || start.deleted_at != null) return [];
|
|
6662
|
+
const { predecessors, successors } = await this.entryRepo.findSupersessionChainIds(entityId, factId, HISTORY_MAX_DEPTH);
|
|
6663
|
+
const facts = await this.entryRepo.findByIds([...predecessors, factId, ...successors], [entityId]);
|
|
6664
|
+
return facts.sort((a, b) => (a.valid_from ?? a.created_at) - (b.valid_from ?? b.created_at) || a.created_at - b.created_at || a.id.localeCompare(b.id));
|
|
6665
|
+
}
|
|
6666
|
+
};
|
|
6667
|
+
|
|
6668
|
+
// src/services/librarian/ops/apply.ts
|
|
6669
|
+
async function applyOps(deps, ctx, input) {
|
|
6670
|
+
const { entityId, trigger } = ctx;
|
|
6671
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
6672
|
+
const grounding = deps.promptService.groundingFor("librarian");
|
|
6673
|
+
const temporal = new TemporalService(deps.db, deps.options, deps.entryRepo, deps.searchService, deps.embeddingService);
|
|
6674
|
+
const rejected = new Set(input.rejected);
|
|
6675
|
+
const failed = new Set(input.failed);
|
|
6676
|
+
const written = [];
|
|
6677
|
+
const updatedIds = [];
|
|
6678
|
+
const now = Date.now();
|
|
6679
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
6680
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
6681
|
+
if (mode === "emergent" && input.ontologyUpdates && deps.ontologyService) {
|
|
6682
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, input.ontologyUpdates, tx);
|
|
6683
|
+
}
|
|
6684
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
6685
|
+
for (const f of await deps.entryRepo.findRecentByEntityId(entityId, 100, tx)) {
|
|
6686
|
+
titleIndex.set(normalizeTitleKey(f.title), { id: f.id, okf_type: f.okf_type ?? null });
|
|
6687
|
+
}
|
|
6688
|
+
const pendingEdges = [];
|
|
6689
|
+
const insert = async (g, draft, validFrom, overrides) => {
|
|
6690
|
+
const fact = g.candidate.fact;
|
|
6691
|
+
const title = overrides?.title ?? fact.title;
|
|
6692
|
+
const body = overrides?.body ?? fact.body;
|
|
6693
|
+
const id = generateId("fact_");
|
|
6694
|
+
const drops = [];
|
|
6695
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(fact, manifest, { strict: false, drops }) ?? { okf_type: null, edges: [] };
|
|
6696
|
+
for (const d of drops) input.diagnostics.push(edgeDropDiagnostic(d, { ...diagBase, factId: id }));
|
|
6697
|
+
const g2 = grounding ? groundingOutcome(checkGrounding(fact.evidence, input.groundingCorpus ?? [], grounding), now) : null;
|
|
6698
|
+
if (g2?.diagnostic) {
|
|
6699
|
+
input.diagnostics.push({
|
|
6700
|
+
...diagBase,
|
|
6701
|
+
code: g2.diagnostic.code,
|
|
6702
|
+
detail: { factId: id, itemIndex: g.candidate.index, reason: g2.diagnostic.reason }
|
|
6703
|
+
});
|
|
6704
|
+
}
|
|
6705
|
+
const row = {
|
|
6706
|
+
id,
|
|
6707
|
+
entity_id: entityId,
|
|
6708
|
+
title,
|
|
6709
|
+
body,
|
|
6710
|
+
tags: fact.tags,
|
|
6711
|
+
confidence: fact.confidence,
|
|
6712
|
+
source_type: "librarian_inferred",
|
|
6713
|
+
source_hash: null,
|
|
6714
|
+
source_ref: null,
|
|
6715
|
+
created_at: now,
|
|
6716
|
+
updated_at: now,
|
|
6717
|
+
last_accessed_at: null,
|
|
6718
|
+
access_count: 0,
|
|
6719
|
+
deleted_at: null,
|
|
6720
|
+
okf_type: normalized.okf_type,
|
|
6721
|
+
...g2?.trust,
|
|
6722
|
+
...draft ? { lifecycle_status: "draft" } : {}
|
|
6723
|
+
};
|
|
6724
|
+
await deps.entryRepo.upsert(row, tx);
|
|
6725
|
+
await deps.entryRepo.setTemporal(id, entityId, { valid_from: validFrom }, tx);
|
|
6726
|
+
titleIndex.set(normalizeTitleKey(title), { id, okf_type: normalized.okf_type });
|
|
6727
|
+
if (normalized.edges.length > 0) pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
6728
|
+
const textChanged = title !== fact.title || body !== fact.body;
|
|
6729
|
+
written.push({ id, vector: textChanged ? null : g.vector, fact: { id, entity_id: entityId, title, body, tags: fact.tags } });
|
|
6730
|
+
return id;
|
|
6731
|
+
};
|
|
6732
|
+
for (const g of input.gated) {
|
|
6733
|
+
const idx = g.candidate.index;
|
|
6734
|
+
const vf = g.candidate.validFrom;
|
|
6735
|
+
if (g.decision.kind === "noop") continue;
|
|
6736
|
+
if (g.decision.kind === "add") {
|
|
6737
|
+
await insert(g, false, vf);
|
|
6738
|
+
continue;
|
|
6739
|
+
}
|
|
6740
|
+
if (failed.has(g)) {
|
|
6741
|
+
const id = await insert(g, true, vf);
|
|
6742
|
+
input.diagnostics.push({ ...diagBase, code: "resolve_failed", detail: { factId: id, itemIndex: idx } });
|
|
6743
|
+
continue;
|
|
6744
|
+
}
|
|
6745
|
+
const op = input.ops.get(g);
|
|
6746
|
+
if (rejected.has(g) || !op) {
|
|
6747
|
+
await insert(g, false, vf);
|
|
6748
|
+
input.diagnostics.push({ ...diagBase, code: "librarian_op_rejected", detail: { itemIndex: idx, reason: "invalid_op" } });
|
|
6749
|
+
continue;
|
|
6750
|
+
}
|
|
6751
|
+
if (op.op === "NOOP") continue;
|
|
6752
|
+
if (op.op === "ADD") {
|
|
6753
|
+
await insert(g, false, vf);
|
|
6754
|
+
continue;
|
|
6755
|
+
}
|
|
6756
|
+
const target = g.neighbours.find((n) => n.id === op.targetId);
|
|
6757
|
+
if (op.op === "UPDATE" && target.source_type === "librarian_inferred") {
|
|
6758
|
+
const [existing] = await deps.entryRepo.findByIds([op.targetId], [entityId], tx);
|
|
6759
|
+
if (existing) {
|
|
6760
|
+
const newTitle = op.title ?? g.candidate.fact.title;
|
|
6761
|
+
await deps.entryRepo.upsert({
|
|
6762
|
+
...existing,
|
|
6763
|
+
title: newTitle,
|
|
6764
|
+
body: op.body ?? g.candidate.fact.body,
|
|
6765
|
+
updated_at: now
|
|
6766
|
+
}, tx);
|
|
6767
|
+
titleIndex.delete(normalizeTitleKey(existing.title));
|
|
6768
|
+
titleIndex.set(normalizeTitleKey(newTitle), { id: existing.id, okf_type: existing.okf_type ?? null });
|
|
6769
|
+
updatedIds.push(existing.id);
|
|
6770
|
+
continue;
|
|
6771
|
+
}
|
|
6772
|
+
}
|
|
6773
|
+
const supersedeFrom = op.op === "SUPERSEDE" && op.validFrom !== void 0 ? op.validFrom : vf;
|
|
6774
|
+
const overrides = op.op === "UPDATE" ? { title: op.title, body: op.body } : void 0;
|
|
6775
|
+
if (target.source_type === "immutable_document") {
|
|
6776
|
+
const id = await insert(g, true, supersedeFrom, overrides);
|
|
6777
|
+
input.diagnostics.push({ ...diagBase, code: "contradicts_document", detail: { factId: id, itemIndex: idx } });
|
|
6778
|
+
continue;
|
|
6779
|
+
}
|
|
6780
|
+
const newId = await insert(g, false, supersedeFrom, overrides);
|
|
6781
|
+
try {
|
|
6782
|
+
await temporal.supersedeInTx(tx, entityId, op.targetId, newId, supersedeFrom, now);
|
|
6783
|
+
} catch (err) {
|
|
6784
|
+
if (!(err instanceof WikiSupersedeError)) throw err;
|
|
6785
|
+
input.diagnostics.push({
|
|
6786
|
+
...diagBase,
|
|
6787
|
+
code: "librarian_op_rejected",
|
|
6788
|
+
detail: { factId: newId, itemIndex: idx, reason: err.reason }
|
|
6789
|
+
});
|
|
6790
|
+
}
|
|
6791
|
+
}
|
|
6792
|
+
for (const item of pendingEdges) {
|
|
6793
|
+
const resolveDrops = [];
|
|
6794
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
6795
|
+
entityId,
|
|
6796
|
+
item.sourceId,
|
|
6797
|
+
item.sourceType,
|
|
6798
|
+
item.edges,
|
|
6799
|
+
manifest,
|
|
6800
|
+
titleIndex,
|
|
6801
|
+
tx,
|
|
6802
|
+
now,
|
|
6803
|
+
resolveDrops
|
|
6804
|
+
);
|
|
6805
|
+
for (const d of resolveDrops) input.diagnostics.push(edgeDropDiagnostic(d, diagBase));
|
|
6806
|
+
}
|
|
6807
|
+
for (const task of input.tasks) {
|
|
6808
|
+
const taskObj = {
|
|
6809
|
+
id: generateId("task_"),
|
|
6810
|
+
entity_id: entityId,
|
|
6811
|
+
description: task.description,
|
|
6812
|
+
status: "pending",
|
|
6813
|
+
priority: task.priority,
|
|
6814
|
+
created_at: now,
|
|
6815
|
+
updated_at: now,
|
|
6816
|
+
resolved_at: null,
|
|
6817
|
+
deleted_at: null
|
|
6818
|
+
};
|
|
6819
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
6820
|
+
}
|
|
6821
|
+
});
|
|
6822
|
+
return { written, updatedIds };
|
|
6823
|
+
}
|
|
6824
|
+
|
|
6825
|
+
// src/services/librarian/ops/index.ts
|
|
6826
|
+
var runOpsLibrarianPass = async (deps, ctx) => {
|
|
6827
|
+
const { entityId, trigger } = ctx;
|
|
6828
|
+
const cfg = resolveGateConfig(deps.options.config?.librarian?.gate);
|
|
6829
|
+
await deps.seedWatermark(entityId);
|
|
6830
|
+
const wm = await deps.getWatermark(entityId);
|
|
6831
|
+
const sel = await selectBatch(deps, entityId, wm, ctx.meter);
|
|
6832
|
+
if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
|
|
6833
|
+
if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
|
|
6834
|
+
let ex;
|
|
6835
|
+
try {
|
|
6836
|
+
ex = await runExtract(deps, ctx, sel.batch);
|
|
6837
|
+
} catch (err) {
|
|
6838
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
6839
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
6840
|
+
}
|
|
6841
|
+
throw err;
|
|
6842
|
+
}
|
|
6843
|
+
const diagnostics = new DiagnosticBuffer();
|
|
6844
|
+
for (const d of ex.diagnostics) diagnostics.push(d);
|
|
6845
|
+
const { gated, inBatchDuplicates, gateDiagnostic } = await gateCandidates(deps, ctx, ex.candidates, cfg);
|
|
6846
|
+
for (const dup of inBatchDuplicates) {
|
|
6847
|
+
diagnostics.push({
|
|
6848
|
+
code: "fact_deduplicated",
|
|
6849
|
+
operation: "librarian",
|
|
6850
|
+
trigger,
|
|
6851
|
+
entityId,
|
|
6852
|
+
detail: { itemIndex: dup.index, reason: "in_batch" }
|
|
6853
|
+
});
|
|
6854
|
+
}
|
|
6855
|
+
diagnostics.push(gateDiagnostic);
|
|
6856
|
+
const res = await runResolve(deps, ctx, gated.filter((g) => g.decision.kind === "ambiguous"));
|
|
6857
|
+
if (res.budgetStop) {
|
|
6858
|
+
diagnostics.discard();
|
|
6859
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: res.budgetStop };
|
|
6860
|
+
}
|
|
6861
|
+
const { written, updatedIds } = await applyOps(deps, ctx, {
|
|
6862
|
+
gated,
|
|
6863
|
+
ops: res.ops,
|
|
6864
|
+
rejected: res.rejected,
|
|
6865
|
+
failed: res.failed,
|
|
6866
|
+
tasks: ex.tasks,
|
|
6867
|
+
ontologyUpdates: ex.ontologyUpdates,
|
|
6868
|
+
groundingCorpus: ex.groundingCorpus,
|
|
6869
|
+
diagnostics
|
|
6870
|
+
});
|
|
6871
|
+
diagnostics.flush(deps.options);
|
|
6872
|
+
await deps.searchService.syncEntries(entityId, [...written.map((w) => w.id), ...updatedIds]);
|
|
6873
|
+
const embedCtx = { operation: "librarian", trigger };
|
|
6874
|
+
for (const w of written) {
|
|
6875
|
+
if (w.vector) await deps.embeddingService.storeFactVector(w.fact, w.vector, embedCtx);
|
|
6876
|
+
else await deps.embeddingService.embedFact(w.fact, embedCtx);
|
|
6877
|
+
}
|
|
6878
|
+
if (updatedIds.length > 0) {
|
|
6879
|
+
for (const f of await deps.entryRepo.findByIds(updatedIds, [entityId])) {
|
|
6880
|
+
await deps.embeddingService.embedFact(
|
|
6881
|
+
{ id: f.id, entity_id: f.entity_id, title: f.title, body: f.body, tags: f.tags },
|
|
6882
|
+
embedCtx
|
|
6883
|
+
);
|
|
6884
|
+
}
|
|
6885
|
+
}
|
|
6886
|
+
deps.searchService.evictCache(entityId);
|
|
6887
|
+
const last = sel.batch[sel.batch.length - 1].event;
|
|
6888
|
+
return {
|
|
6889
|
+
processedThrough: { at: last.created_at, id: last.id },
|
|
6890
|
+
factsWritten: written.length + updatedIds.length
|
|
6891
|
+
};
|
|
6892
|
+
};
|
|
6893
|
+
|
|
6894
|
+
// src/services/librarian/index.ts
|
|
6895
|
+
var STRATEGIES = /* @__PURE__ */ new Map([
|
|
6896
|
+
["legacy", runLegacyLibrarianPass],
|
|
6897
|
+
["ops", runOpsLibrarianPass]
|
|
6898
|
+
]);
|
|
6899
|
+
async function runLibrarianStrategy(deps, ctx) {
|
|
6900
|
+
const strategy = deps.options.config?.librarian?.strategy ?? "legacy";
|
|
6901
|
+
const fn = STRATEGIES.get(strategy);
|
|
6902
|
+
if (!fn) {
|
|
6903
|
+
throw new Error(`Unknown librarian strategy: ${strategy}`);
|
|
6904
|
+
}
|
|
6905
|
+
return fn(deps, ctx);
|
|
6906
|
+
}
|
|
6907
|
+
|
|
6908
|
+
// src/services/MaintenanceService.ts
|
|
6909
|
+
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
6910
|
+
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
6911
|
+
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
6912
|
+
var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
|
|
6913
|
+
var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
6914
|
+
var HEAL_BATCH_SIZE = 25;
|
|
6915
|
+
var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
6916
|
+
var MAX_EMBED_ATTEMPTS = 5;
|
|
6917
|
+
var EMBED_RETRY_BASE_MS = 6e4;
|
|
6918
|
+
var EMBED_RETRY_CAP_MS = 24 * 60 * 60 * 1e3;
|
|
6919
|
+
function embedRetryDelayMs(attempts) {
|
|
6920
|
+
const n = Math.max(1, Math.trunc(attempts || 0));
|
|
6921
|
+
const delay = EMBED_RETRY_BASE_MS * Math.pow(2, n - 1);
|
|
6922
|
+
return Math.min(delay, EMBED_RETRY_CAP_MS);
|
|
6923
|
+
}
|
|
6924
|
+
function classifyReembedRow(row, now, force) {
|
|
6925
|
+
if (force) return "attempt";
|
|
6926
|
+
const failedAt = row.embedding_failed_at;
|
|
6927
|
+
if (failedAt === null || failedAt === void 0) return "attempt";
|
|
6928
|
+
if (row.embedding_failure_kind === "float32_overflow") return "permanent";
|
|
6929
|
+
const attempts = row.embedding_attempts ?? 0;
|
|
6930
|
+
if (attempts >= MAX_EMBED_ATTEMPTS) return "permanent";
|
|
6931
|
+
return now - failedAt >= embedRetryDelayMs(attempts) ? "attempt" : "defer";
|
|
6932
|
+
}
|
|
6933
|
+
var SKIP_ERROR_LOG_CHARS = 4096;
|
|
6934
|
+
var formatSkipError = (err) => {
|
|
6935
|
+
let base;
|
|
6936
|
+
let isErrorLike = false;
|
|
6937
|
+
try {
|
|
6938
|
+
isErrorLike = err instanceof Error;
|
|
6939
|
+
} catch {
|
|
6940
|
+
}
|
|
6941
|
+
if (isErrorLike || typeof err !== "object" && typeof err !== "function") {
|
|
6942
|
+
base = safeErrorToString(err);
|
|
6943
|
+
} else {
|
|
6944
|
+
try {
|
|
6945
|
+
const json = JSON.stringify(err);
|
|
6946
|
+
base = typeof json === "string" ? json : safeErrorToString(err);
|
|
6947
|
+
} catch {
|
|
6948
|
+
base = safeErrorToString(err);
|
|
6949
|
+
}
|
|
6950
|
+
}
|
|
6951
|
+
if (base.length > SKIP_ERROR_LOG_CHARS) {
|
|
6952
|
+
const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
|
|
6953
|
+
return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
|
|
6954
|
+
}
|
|
6955
|
+
return base;
|
|
6956
|
+
};
|
|
6957
|
+
var MaintenanceService = class {
|
|
6958
|
+
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
6959
|
+
this.db = db;
|
|
6960
|
+
this.prefix = prefix;
|
|
6961
|
+
this.options = options;
|
|
6962
|
+
this.entryRepo = entryRepo;
|
|
6963
|
+
this.sourceRefIndexRepo = sourceRefIndexRepo;
|
|
6964
|
+
this.taskRepo = taskRepo;
|
|
6965
|
+
this.eventRepo = eventRepo;
|
|
6966
|
+
this.metadataRepo = metadataRepo;
|
|
6967
|
+
this.searchService = searchService;
|
|
6968
|
+
this.jobManager = jobManager;
|
|
6969
|
+
this.embeddingService = embeddingService;
|
|
6970
|
+
this.ontologyService = ontologyService;
|
|
6971
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
6972
|
+
}
|
|
6973
|
+
async runPrune(entityId, options) {
|
|
6974
|
+
this.jobManager.acquireLock("prune", entityId);
|
|
6975
|
+
try {
|
|
6976
|
+
const retainSoftDeletedFor = options?.retainSoftDeletedFor !== void 0 ? options.retainSoftDeletedFor : this.options.config?.pruneRetainSoftDeletedFor ?? 7;
|
|
6977
|
+
const retainEventsFor = options?.retainEventsFor !== void 0 ? options.retainEventsFor : this.options.config?.pruneEventsAfter ?? 30;
|
|
6978
|
+
const vacuum = options?.vacuum ?? false;
|
|
6979
|
+
this._validatePruneDuration(retainSoftDeletedFor, "retainSoftDeletedFor");
|
|
6980
|
+
this._validatePruneDuration(retainEventsFor, "retainEventsFor");
|
|
6981
|
+
const pruneSupersededAfter = this.options.config?.pruneSupersededAfter ?? null;
|
|
6982
|
+
this._validatePruneDuration(pruneSupersededAfter, "pruneSupersededAfter");
|
|
5759
6983
|
const now = Date.now();
|
|
5760
6984
|
let syncedIds = [];
|
|
5761
6985
|
let deletedEntries = 0;
|
|
5762
6986
|
let deletedTasks = 0;
|
|
5763
6987
|
let deletedEvents = 0;
|
|
6988
|
+
if (pruneSupersededAfter !== null) {
|
|
6989
|
+
const expiredCutoff = now - pruneSupersededAfter * 864e5;
|
|
6990
|
+
const expiredIds = await this.entryRepo.findExpiredIds(entityId, expiredCutoff);
|
|
6991
|
+
if (expiredIds.length > 0) {
|
|
6992
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
6993
|
+
await this.entryRepo.softDeleteByIds(expiredIds, entityId, tx);
|
|
6994
|
+
});
|
|
6995
|
+
syncedIds = [...expiredIds];
|
|
6996
|
+
}
|
|
6997
|
+
}
|
|
5764
6998
|
if (retainSoftDeletedFor !== null) {
|
|
5765
6999
|
const cutoff = now - retainSoftDeletedFor * 864e5;
|
|
5766
7000
|
const entriesToDelete = await this.entryRepo.getPrunableMetadata(entityId, cutoff);
|
|
@@ -5776,7 +7010,7 @@ var MaintenanceService = class {
|
|
|
5776
7010
|
}
|
|
5777
7011
|
}
|
|
5778
7012
|
const succeededIds = succeeded.map((r) => r.id);
|
|
5779
|
-
syncedIds = succeededIds;
|
|
7013
|
+
syncedIds = [...syncedIds, ...succeededIds];
|
|
5780
7014
|
await this.db.withTransactionAsync(async (tx) => {
|
|
5781
7015
|
if (succeededIds.length > 0) {
|
|
5782
7016
|
deletedEntries = await this.entryRepo.bulkDeletePruned(entityId, cutoff, succeededIds, tx);
|
|
@@ -5812,8 +7046,18 @@ var MaintenanceService = class {
|
|
|
5812
7046
|
}
|
|
5813
7047
|
if (retainEventsFor !== null) {
|
|
5814
7048
|
const cutoff = now - retainEventsFor * 864e5;
|
|
5815
|
-
const
|
|
5816
|
-
|
|
7049
|
+
const guarded = this.options.config?.maintenance === "deferred" || this.options.config?.librarian?.strategy === "ops";
|
|
7050
|
+
if (guarded) {
|
|
7051
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
7052
|
+
deletedEvents = (await this.eventRepo.pruneThrough(entityId, cutoff, wm)).changes;
|
|
7053
|
+
const held = await this.eventRepo.countHeldBack(entityId, cutoff, wm);
|
|
7054
|
+
if (held > 0) {
|
|
7055
|
+
emitDiagnostic(this.options, { code: "event_retention_held", operation: "prune", trigger: "call", entityId, detail: { count: held } });
|
|
7056
|
+
}
|
|
7057
|
+
} else {
|
|
7058
|
+
const eventResult = await this.eventRepo.prune(entityId, cutoff);
|
|
7059
|
+
deletedEvents = eventResult.changes;
|
|
7060
|
+
}
|
|
5817
7061
|
}
|
|
5818
7062
|
if (vacuum) {
|
|
5819
7063
|
await this.metadataRepo.vacuum();
|
|
@@ -6043,155 +7287,72 @@ var MaintenanceService = class {
|
|
|
6043
7287
|
}
|
|
6044
7288
|
/** Core librarian pass (locks handled by {@link runLibrarian}). Package-internal orchestration hook. */
|
|
6045
7289
|
async doRunLibrarian(entityId, promptOverride, trigger = "call") {
|
|
6046
|
-
|
|
6047
|
-
|
|
6048
|
-
|
|
6049
|
-
|
|
6050
|
-
|
|
6051
|
-
|
|
6052
|
-
|
|
6053
|
-
|
|
6054
|
-
|
|
6055
|
-
|
|
6056
|
-
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
6060
|
-
|
|
6061
|
-
|
|
6062
|
-
|
|
6063
|
-
|
|
6064
|
-
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
6065
|
-
const result = parseJsonResponse(responseText);
|
|
6066
|
-
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
6067
|
-
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
6068
|
-
const ontologyUpdates = result.ontology_updates;
|
|
6069
|
-
const diagBuffer = new DiagnosticBuffer();
|
|
6070
|
-
const diagBase = { entityId, operation: "librarian", trigger };
|
|
6071
|
-
const validFacts = [];
|
|
6072
|
-
const validFactItemIndexes = [];
|
|
6073
|
-
facts.forEach((raw, itemIndex) => {
|
|
6074
|
-
const valid = validateFact(raw);
|
|
6075
|
-
if (valid) {
|
|
6076
|
-
validFacts.push(valid);
|
|
6077
|
-
validFactItemIndexes.push(itemIndex);
|
|
6078
|
-
} else {
|
|
6079
|
-
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
6080
|
-
}
|
|
6081
|
-
});
|
|
6082
|
-
const validTasks = [];
|
|
6083
|
-
tasks.forEach((raw, itemIndex) => {
|
|
6084
|
-
const valid = validateTask(raw);
|
|
6085
|
-
if (valid) validTasks.push(valid);
|
|
6086
|
-
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
6087
|
-
});
|
|
6088
|
-
const now = Date.now();
|
|
6089
|
-
const insertedFacts = [];
|
|
6090
|
-
await this.db.withTransactionAsync(async (tx) => {
|
|
6091
|
-
let { mode, manifest } = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
6092
|
-
if (mode === "emergent" && ontologyUpdates && this.ontologyService) {
|
|
6093
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
6094
|
-
}
|
|
6095
|
-
const titleIndex = /* @__PURE__ */ new Map();
|
|
6096
|
-
for (const existing of currentFactsRows) {
|
|
6097
|
-
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
6098
|
-
id: existing.id,
|
|
6099
|
-
okf_type: existing.okf_type ?? null
|
|
6100
|
-
});
|
|
6101
|
-
}
|
|
6102
|
-
const factsForDedupe = await this.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
6103
|
-
const pendingEdges = [];
|
|
6104
|
-
for (const [k, fact] of validFacts.entries()) {
|
|
6105
|
-
const newTokens = titleTokens(fact.title);
|
|
6106
|
-
let skip = false;
|
|
6107
|
-
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
6108
|
-
for (const existing of factsForDedupe) {
|
|
6109
|
-
if (existing.source_type !== "librarian_inferred") continue;
|
|
6110
|
-
const existingTokens = titleTokens(existing.title);
|
|
6111
|
-
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
6112
|
-
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
6113
|
-
skip = true;
|
|
6114
|
-
break;
|
|
6115
|
-
}
|
|
6116
|
-
}
|
|
6117
|
-
}
|
|
6118
|
-
}
|
|
6119
|
-
if (skip) {
|
|
6120
|
-
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
6121
|
-
continue;
|
|
6122
|
-
}
|
|
6123
|
-
const ontologyFact = fact;
|
|
6124
|
-
const id = generateId("fact_");
|
|
6125
|
-
const validationDrops = [];
|
|
6126
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
6127
|
-
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
6128
|
-
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
6129
|
-
if (grounding?.diagnostic) {
|
|
6130
|
-
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
6131
|
-
}
|
|
6132
|
-
const factObj = {
|
|
6133
|
-
id,
|
|
6134
|
-
entity_id: entityId,
|
|
6135
|
-
title: fact.title,
|
|
6136
|
-
body: fact.body,
|
|
6137
|
-
tags: fact.tags,
|
|
6138
|
-
confidence: fact.confidence,
|
|
6139
|
-
source_type: "librarian_inferred",
|
|
6140
|
-
source_hash: null,
|
|
6141
|
-
source_ref: null,
|
|
6142
|
-
created_at: now,
|
|
6143
|
-
updated_at: now,
|
|
6144
|
-
last_accessed_at: null,
|
|
6145
|
-
access_count: 0,
|
|
6146
|
-
deleted_at: null,
|
|
6147
|
-
okf_type: normalized.okf_type,
|
|
6148
|
-
...grounding?.trust
|
|
6149
|
-
};
|
|
6150
|
-
await this.entryRepo.upsert(factObj, tx);
|
|
6151
|
-
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
6152
|
-
factsForDedupe.push(factObj);
|
|
6153
|
-
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
6154
|
-
if (normalized.edges.length > 0) {
|
|
6155
|
-
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
6156
|
-
}
|
|
6157
|
-
}
|
|
6158
|
-
for (const item of pendingEdges) {
|
|
6159
|
-
const resolveDrops = [];
|
|
6160
|
-
await this.ontologyService?.resolveAndPersistEdges(
|
|
6161
|
-
entityId,
|
|
6162
|
-
item.sourceId,
|
|
6163
|
-
item.sourceType,
|
|
6164
|
-
item.edges ?? [],
|
|
6165
|
-
manifest,
|
|
6166
|
-
titleIndex,
|
|
6167
|
-
tx,
|
|
6168
|
-
now,
|
|
6169
|
-
resolveDrops
|
|
6170
|
-
);
|
|
6171
|
-
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
6172
|
-
}
|
|
6173
|
-
for (const task of validTasks) {
|
|
6174
|
-
const id = generateId("task_");
|
|
6175
|
-
const taskObj = {
|
|
6176
|
-
id,
|
|
6177
|
-
entity_id: entityId,
|
|
6178
|
-
description: task.description,
|
|
6179
|
-
status: "pending",
|
|
6180
|
-
priority: task.priority,
|
|
6181
|
-
created_at: now,
|
|
6182
|
-
updated_at: now,
|
|
6183
|
-
resolved_at: null,
|
|
6184
|
-
deleted_at: null
|
|
6185
|
-
};
|
|
6186
|
-
await this.taskRepo.upsert(taskObj, tx);
|
|
6187
|
-
}
|
|
6188
|
-
});
|
|
6189
|
-
diagBuffer.flush(this.options);
|
|
6190
|
-
await this.searchService.syncEntries(entityId, insertedFacts.map((f) => f.id));
|
|
6191
|
-
for (const fact of insertedFacts) {
|
|
6192
|
-
await this.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
7290
|
+
await this.runLibrarianPass({ entityId, trigger, ...promptOverride !== void 0 ? { promptOverride } : {} });
|
|
7291
|
+
}
|
|
7292
|
+
/**
|
|
7293
|
+
* Seam between the lock wrapper and the strategy dispatcher (spec 2026-09-29
|
|
7294
|
+
* §5.1). Builds the strategy deps from `this` and dispatches via
|
|
7295
|
+
* `runLibrarianStrategy` so PR-B can plug in `ops/` without touching this
|
|
7296
|
+
* class. Returns the strategy's `LibrarianResult` so deferred/budgeted
|
|
7297
|
+
* schedulers (PR-C) can advance watermarks; the existing void-returning
|
|
7298
|
+
* `doRunLibrarian` discards it.
|
|
7299
|
+
*
|
|
7300
|
+
* Spec §10.3: one place advances the watermark, for every strategy, only
|
|
7301
|
+
* after the pass resolved (its transaction committed) and never backwards.
|
|
7302
|
+
* A budget-stopped pass wrote nothing and must not move the watermark.
|
|
7303
|
+
*/
|
|
7304
|
+
async runLibrarianPass(ctx) {
|
|
7305
|
+
const result = await runLibrarianStrategy(this.librarianDeps(), ctx);
|
|
7306
|
+
if (result.processedThrough && !result.budgetStop) {
|
|
7307
|
+
await this.advanceLibrarianWatermark(ctx.entityId, result.processedThrough);
|
|
6193
7308
|
}
|
|
6194
|
-
|
|
7309
|
+
return result;
|
|
7310
|
+
}
|
|
7311
|
+
/**
|
|
7312
|
+
* Strategy deps shared between legacy and ops librarians (spec §5.1). The
|
|
7313
|
+
* ops strategy (PR-B Task 8) reads/anchors the watermark through these two
|
|
7314
|
+
* hooks so the dispatcher itself never reaches into the metadata table —
|
|
7315
|
+
* the watermark invariant (one place advances, never backwards) lives here.
|
|
7316
|
+
*/
|
|
7317
|
+
librarianDeps() {
|
|
7318
|
+
return {
|
|
7319
|
+
db: this.db,
|
|
7320
|
+
options: this.options,
|
|
7321
|
+
entryRepo: this.entryRepo,
|
|
7322
|
+
taskRepo: this.taskRepo,
|
|
7323
|
+
eventRepo: this.eventRepo,
|
|
7324
|
+
searchService: this.searchService,
|
|
7325
|
+
embeddingService: this.embeddingService,
|
|
7326
|
+
promptService: this.promptService,
|
|
7327
|
+
ontologyService: this.ontologyService,
|
|
7328
|
+
seedWatermark: (entityId) => this.seedLibrarianWatermark(entityId),
|
|
7329
|
+
getWatermark: (entityId) => this.metadataRepo.getLibrarianWatermark(entityId, this.db)
|
|
7330
|
+
};
|
|
7331
|
+
}
|
|
7332
|
+
/**
|
|
7333
|
+
* Persist `next` as the entity's librarian watermark when it is strictly
|
|
7334
|
+
* newer than the stored value, under the (created_at, id) ordering used by
|
|
7335
|
+
* the op-based librarian (spec §10.3). Never moves backwards.
|
|
7336
|
+
*/
|
|
7337
|
+
async advanceLibrarianWatermark(entityId, next) {
|
|
7338
|
+
const current = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
7339
|
+
const newer = !current || next.at > current.at || next.at === current.at && next.id > current.id;
|
|
7340
|
+
if (newer) await this.metadataRepo.setLibrarianWatermark(entityId, next, this.db);
|
|
7341
|
+
}
|
|
7342
|
+
/**
|
|
7343
|
+
* Spec §10.3: first ops pass on a legacy DB starts after what the count
|
|
7344
|
+
* checkpoint already covered. No-op when a watermark already exists or the
|
|
7345
|
+
* memory checkpoint is 0; otherwise anchor the watermark to the event at
|
|
7346
|
+
* offset `min(memory_checkpoint, totalEvents) - 1`.
|
|
7347
|
+
*/
|
|
7348
|
+
async seedLibrarianWatermark(entityId) {
|
|
7349
|
+
if (await this.metadataRepo.getLibrarianWatermark(entityId, this.db)) return;
|
|
7350
|
+
const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
|
|
7351
|
+
const memory = cp.memory ?? 0;
|
|
7352
|
+
if (memory <= 0) return;
|
|
7353
|
+
const total = await this.eventRepo.count(entityId);
|
|
7354
|
+
const ev = await this.eventRepo.getAtOffset(entityId, Math.min(memory, total) - 1);
|
|
7355
|
+
if (ev) await this.metadataRepo.setLibrarianWatermark(entityId, { at: ev.created_at, id: ev.id }, this.db);
|
|
6195
7356
|
}
|
|
6196
7357
|
/**
|
|
6197
7358
|
* Core heal pass (locks handled by {@link runHeal}). Package-internal orchestration hook.
|
|
@@ -6297,7 +7458,7 @@ var MaintenanceService = class {
|
|
|
6297
7458
|
if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
|
|
6298
7459
|
return prompts;
|
|
6299
7460
|
},
|
|
6300
|
-
call: (prompts) => this.options
|
|
7461
|
+
call: (prompts) => callLlm(this.options, { operation: "heal", entityId, trigger, meter: options?.meter }, prompts),
|
|
6301
7462
|
parse: (responseText, batch, prompts) => {
|
|
6302
7463
|
const result = parseJsonResponse(responseText);
|
|
6303
7464
|
return {
|
|
@@ -6401,9 +7562,10 @@ var MaintenanceService = class {
|
|
|
6401
7562
|
const callErrorIds = new Set(
|
|
6402
7563
|
outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
|
|
6403
7564
|
);
|
|
7565
|
+
const unattemptedIds = new Set(outcome.unattempted.map((f) => f.id));
|
|
6404
7566
|
await this.entryRepo.markHealChecked(
|
|
6405
7567
|
[
|
|
6406
|
-
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id)),
|
|
7568
|
+
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id) && !unattemptedIds.has(id)),
|
|
6407
7569
|
...insertedFacts.map((f) => f.id)
|
|
6408
7570
|
],
|
|
6409
7571
|
entityId,
|
|
@@ -6455,7 +7617,8 @@ var MaintenanceService = class {
|
|
|
6455
7617
|
skipped: outcome.skipped.map(({ item, reason }) => ({ id: item.id, reason })),
|
|
6456
7618
|
degraded: healedDegraded,
|
|
6457
7619
|
remaining: counts.eligible,
|
|
6458
|
-
deferred: counts.deferred
|
|
7620
|
+
deferred: counts.deferred,
|
|
7621
|
+
...outcome.budgetStop ? { budgetStop: outcome.budgetStop } : {}
|
|
6459
7622
|
};
|
|
6460
7623
|
}
|
|
6461
7624
|
/** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
|
|
@@ -6499,7 +7662,7 @@ var MaintenanceService = class {
|
|
|
6499
7662
|
const outcome = await runBatched({
|
|
6500
7663
|
items: candidates,
|
|
6501
7664
|
buildPrompt,
|
|
6502
|
-
call: (prompts) => this.options
|
|
7665
|
+
call: (prompts) => callLlm(this.options, { operation: "ontologyBackfill", entityId, trigger: "call" }, prompts),
|
|
6503
7666
|
parse: (responseText, batch) => {
|
|
6504
7667
|
const parsed = parseJsonResponse(responseText);
|
|
6505
7668
|
return {
|
|
@@ -6792,6 +7955,186 @@ var MaintenanceService = class {
|
|
|
6792
7955
|
return sanitizeRankerError(err, this.options.sanitizeRankerErrors);
|
|
6793
7956
|
}
|
|
6794
7957
|
};
|
|
7958
|
+
|
|
7959
|
+
// src/services/MaintenanceScheduler.ts
|
|
7960
|
+
var MaintenanceScheduler = class {
|
|
7961
|
+
constructor(db, options, maintenanceService, metadataRepo, eventRepo, entryRepo, jobManager) {
|
|
7962
|
+
this.db = db;
|
|
7963
|
+
this.options = options;
|
|
7964
|
+
this.maintenanceService = maintenanceService;
|
|
7965
|
+
this.metadataRepo = metadataRepo;
|
|
7966
|
+
this.eventRepo = eventRepo;
|
|
7967
|
+
this.entryRepo = entryRepo;
|
|
7968
|
+
this.jobManager = jobManager;
|
|
7969
|
+
}
|
|
7970
|
+
/**
|
|
7971
|
+
* Entities with **any** pending work only, sorted by `pendingEvents` desc
|
|
7972
|
+
* then `entityId` asc. Token estimate is ceil(pending summary chars / 4),
|
|
7973
|
+
* matching the auto-trigger's estimator (PR-B).
|
|
7974
|
+
*/
|
|
7975
|
+
async getPending(entityIds) {
|
|
7976
|
+
const ids = entityIds ?? await this.metadataRepo.getDistinctEntityIds();
|
|
7977
|
+
const autoHealThreshold = this.options.config?.autoHealThreshold || 100;
|
|
7978
|
+
const canEmbed = typeof this.options.llmProvider.embed === "function";
|
|
7979
|
+
const out = [];
|
|
7980
|
+
for (const entityId of ids) {
|
|
7981
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
7982
|
+
const pendingEvents = await this.eventRepo.countAfter(entityId, wm);
|
|
7983
|
+
const chars = pendingEvents > 0 ? await this.eventRepo.sumSummaryCharsAfter(entityId, wm) : 0;
|
|
7984
|
+
const eventCount = await this.eventRepo.count(entityId);
|
|
7985
|
+
const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
|
|
7986
|
+
let healCheckpoint = cp.heal ?? 0;
|
|
7987
|
+
if (healCheckpoint > eventCount) healCheckpoint = 0;
|
|
7988
|
+
const healDue = eventCount - healCheckpoint >= autoHealThreshold;
|
|
7989
|
+
const reembedPending = canEmbed && await this.entryRepo.countReembedPending(entityId, MAX_EMBED_ATTEMPTS) > 0;
|
|
7990
|
+
if (pendingEvents > 0 || healDue || reembedPending) {
|
|
7991
|
+
out.push({ entityId, pendingEvents, pendingTokensEstimate: Math.ceil(chars / 4), healDue, reembedPending });
|
|
7992
|
+
}
|
|
7993
|
+
}
|
|
7994
|
+
return out.sort((x, y) => y.pendingEvents - x.pendingEvents || (x.entityId < y.entityId ? -1 : x.entityId > y.entityId ? 1 : 0));
|
|
7995
|
+
}
|
|
7996
|
+
/**
|
|
7997
|
+
* Execute pending maintenance (spec §6.2): fair-share rounds of at most one
|
|
7998
|
+
* unit per entity (librarian batch, else heal batch, else reembed), entities
|
|
7999
|
+
* ordered by `getPending` (pendingEvents desc, id asc). Holds the same
|
|
8000
|
+
* per-entity `JobManager` locks a manual run would; a busy entity is skipped
|
|
8001
|
+
* and reported, never thrown. Nothing is still running when this resolves.
|
|
8002
|
+
*
|
|
8003
|
+
* Stop precedence (spec §10 items 3-6): `deadline` > `budget_exhausted` >
|
|
8004
|
+
* `budget_too_small` > `complete`. `budget_too_small` means a job's smallest
|
|
8005
|
+
* unit needs more than the WHOLE budget — that job is skipped this run and
|
|
8006
|
+
* the run continues; `budget_exhausted` means a unit needs more than what
|
|
8007
|
+
* REMAINS — the run stops. The token budget meters `generateText` calls
|
|
8008
|
+
* only; embeddings are never metered.
|
|
8009
|
+
*/
|
|
8010
|
+
async run(opts = {}) {
|
|
8011
|
+
if (opts.deadlineMs !== void 0 && (typeof opts.deadlineMs !== "number" || !Number.isFinite(opts.deadlineMs) || opts.deadlineMs < 0)) {
|
|
8012
|
+
throw new TypeError("deadlineMs must be a finite number >= 0");
|
|
8013
|
+
}
|
|
8014
|
+
const meter = new UsageMeter(opts.tokenBudget);
|
|
8015
|
+
const budget = opts.tokenBudget;
|
|
8016
|
+
const deadline = opts.deadlineMs !== void 0 ? Date.now() + Math.max(0, opts.deadlineMs) : Infinity;
|
|
8017
|
+
const jobs = new Set(opts.jobs ?? ["librarian", "heal", "reembed"]);
|
|
8018
|
+
const pending = await this.getPending(opts.entityIds);
|
|
8019
|
+
const states = pending.map((p) => ({
|
|
8020
|
+
p,
|
|
8021
|
+
report: { entityId: p.entityId, librarianPasses: 0, factsWritten: 0, healPasses: 0, reembedded: false },
|
|
8022
|
+
librarian: jobs.has("librarian") && p.pendingEvents > 0,
|
|
8023
|
+
heal: jobs.has("heal") && p.healDue,
|
|
8024
|
+
reembed: jobs.has("reembed") && p.reembedPending
|
|
8025
|
+
}));
|
|
8026
|
+
let stopped = { reason: "complete" };
|
|
8027
|
+
let tooSmall = null;
|
|
8028
|
+
const classify = (job, entityId, requiredEstimate) => {
|
|
8029
|
+
if (budget !== void 0 && requiredEstimate > budget) {
|
|
8030
|
+
tooSmall ?? (tooSmall = { reason: "budget_too_small", job, entityId, requiredEstimate });
|
|
8031
|
+
return "too_small";
|
|
8032
|
+
}
|
|
8033
|
+
return "exhausted";
|
|
8034
|
+
};
|
|
8035
|
+
const exhaustedStop = () => Date.now() >= deadline ? { reason: "deadline" } : { reason: "budget_exhausted" };
|
|
8036
|
+
const markBusy = (st, job) => {
|
|
8037
|
+
var _a;
|
|
8038
|
+
((_a = st.report).busy ?? (_a.busy = [])).push(job);
|
|
8039
|
+
};
|
|
8040
|
+
rounds: for (; ; ) {
|
|
8041
|
+
let progressed = false;
|
|
8042
|
+
for (const st of states) {
|
|
8043
|
+
if (!st.librarian && !st.heal && !st.reembed) continue;
|
|
8044
|
+
if (Date.now() >= deadline) {
|
|
8045
|
+
stopped = { reason: "deadline" };
|
|
8046
|
+
break rounds;
|
|
8047
|
+
}
|
|
8048
|
+
const entityId = st.p.entityId;
|
|
8049
|
+
if (st.librarian) {
|
|
8050
|
+
if (this.jobManager.isBlocked("librarian", entityId)) {
|
|
8051
|
+
st.librarian = false;
|
|
8052
|
+
markBusy(st, "librarian");
|
|
8053
|
+
continue;
|
|
8054
|
+
}
|
|
8055
|
+
this.jobManager.acquireLock("librarian", entityId);
|
|
8056
|
+
let r;
|
|
8057
|
+
try {
|
|
8058
|
+
r = await this.maintenanceService.runLibrarianPass({ entityId, trigger: "call", meter });
|
|
8059
|
+
} finally {
|
|
8060
|
+
this.jobManager.releaseLock("librarian", entityId);
|
|
8061
|
+
}
|
|
8062
|
+
if (r.budgetStop) {
|
|
8063
|
+
if (classify("librarian", entityId, r.budgetStop.requiredEstimate) === "exhausted") {
|
|
8064
|
+
stopped = exhaustedStop();
|
|
8065
|
+
break rounds;
|
|
8066
|
+
}
|
|
8067
|
+
st.librarian = false;
|
|
8068
|
+
continue;
|
|
8069
|
+
}
|
|
8070
|
+
if (!r.processedThrough) {
|
|
8071
|
+
st.librarian = false;
|
|
8072
|
+
continue;
|
|
8073
|
+
}
|
|
8074
|
+
st.report.librarianPasses++;
|
|
8075
|
+
st.report.factsWritten += r.factsWritten;
|
|
8076
|
+
progressed = true;
|
|
8077
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
8078
|
+
if (await this.eventRepo.countAfter(entityId, wm) === 0) st.librarian = false;
|
|
8079
|
+
continue;
|
|
8080
|
+
}
|
|
8081
|
+
if (st.heal) {
|
|
8082
|
+
if (this.jobManager.isBlocked("heal", entityId)) {
|
|
8083
|
+
st.heal = false;
|
|
8084
|
+
markBusy(st, "heal");
|
|
8085
|
+
continue;
|
|
8086
|
+
}
|
|
8087
|
+
this.jobManager.acquireLock("heal", entityId);
|
|
8088
|
+
let h;
|
|
8089
|
+
try {
|
|
8090
|
+
h = await this.maintenanceService.doRunHeal(entityId, { trigger: "call", meter });
|
|
8091
|
+
} finally {
|
|
8092
|
+
this.jobManager.releaseLock("heal", entityId);
|
|
8093
|
+
}
|
|
8094
|
+
if (h.budgetStop && h.scanned === 0) {
|
|
8095
|
+
if (classify("heal", entityId, h.budgetStop.requiredEstimate) === "exhausted") {
|
|
8096
|
+
stopped = exhaustedStop();
|
|
8097
|
+
break rounds;
|
|
8098
|
+
}
|
|
8099
|
+
st.heal = false;
|
|
8100
|
+
continue;
|
|
8101
|
+
}
|
|
8102
|
+
st.report.healPasses++;
|
|
8103
|
+
progressed = true;
|
|
8104
|
+
if (h.remaining === 0) {
|
|
8105
|
+
st.heal = false;
|
|
8106
|
+
await this.metadataRepo.updateCheckpoint(entityId, { heal: await this.eventRepo.count(entityId) }, this.db);
|
|
8107
|
+
}
|
|
8108
|
+
if (h.budgetStop) {
|
|
8109
|
+
stopped = exhaustedStop();
|
|
8110
|
+
break rounds;
|
|
8111
|
+
}
|
|
8112
|
+
continue;
|
|
8113
|
+
}
|
|
8114
|
+
if (st.reembed) {
|
|
8115
|
+
st.reembed = false;
|
|
8116
|
+
try {
|
|
8117
|
+
await this.maintenanceService.runReembed(entityId);
|
|
8118
|
+
st.report.reembedded = true;
|
|
8119
|
+
progressed = true;
|
|
8120
|
+
} catch (err) {
|
|
8121
|
+
if (err instanceof WikiBusyError) markBusy(st, "reembed");
|
|
8122
|
+
else throw err;
|
|
8123
|
+
}
|
|
8124
|
+
}
|
|
8125
|
+
}
|
|
8126
|
+
if (!progressed) break;
|
|
8127
|
+
}
|
|
8128
|
+
if (stopped.reason === "complete" && tooSmall) stopped = tooSmall;
|
|
8129
|
+
return {
|
|
8130
|
+
perEntity: states.map((s) => s.report),
|
|
8131
|
+
tokensUsed: meter.used,
|
|
8132
|
+
estimated: meter.estimated,
|
|
8133
|
+
remaining: budget === void 0 ? null : meter.remaining,
|
|
8134
|
+
stoppedReason: stopped
|
|
8135
|
+
};
|
|
8136
|
+
}
|
|
8137
|
+
};
|
|
6795
8138
|
var MAX_EMBEDDING_BLOB_BYTES = 32 * 1024;
|
|
6796
8139
|
var IMPORT_TITLE_MAX = 500;
|
|
6797
8140
|
var IMPORT_BODY_MAX = 8e3;
|
|
@@ -7030,6 +8373,12 @@ var ImportExportService = class {
|
|
|
7030
8373
|
})()
|
|
7031
8374
|
};
|
|
7032
8375
|
await this.entryRepo.upsertForImport(factObj, tx);
|
|
8376
|
+
await this.entryRepo.setTemporal(fact.id, entityId, {
|
|
8377
|
+
valid_from: fact.valid_from ?? null,
|
|
8378
|
+
valid_to: fact.valid_to ?? null,
|
|
8379
|
+
superseded_by: fact.superseded_by ?? null,
|
|
8380
|
+
superseded_at: fact.superseded_at ?? null
|
|
8381
|
+
}, tx, { outbox: false });
|
|
7033
8382
|
if (blobData != null) {
|
|
7034
8383
|
factsWithPreservedBlob.set(fact.id, blobData);
|
|
7035
8384
|
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
@@ -7108,7 +8457,8 @@ var ImportExportService = class {
|
|
|
7108
8457
|
event_type: event.event_type,
|
|
7109
8458
|
summary: event.summary,
|
|
7110
8459
|
related_entry_id: event.related_entry_id ?? null,
|
|
7111
|
-
created_at: event.created_at
|
|
8460
|
+
created_at: event.created_at,
|
|
8461
|
+
occurred_at: event.occurred_at ?? null
|
|
7112
8462
|
},
|
|
7113
8463
|
tx
|
|
7114
8464
|
);
|
|
@@ -7355,6 +8705,21 @@ var EmbeddingService = class {
|
|
|
7355
8705
|
}
|
|
7356
8706
|
}
|
|
7357
8707
|
async tryEmbedFact(fact, ctx) {
|
|
8708
|
+
const r = await this.embedTextForFact(fact, ctx);
|
|
8709
|
+
return r.ok ? this.storeFactVector(fact, r.vector, ctx) : r;
|
|
8710
|
+
}
|
|
8711
|
+
/**
|
|
8712
|
+
* Compute half: build the embed text, call embed(), validate the result,
|
|
8713
|
+
* mark any embed-time failure, and return the validated Float32 vector.
|
|
8714
|
+
*
|
|
8715
|
+
* Failure kinds: `no_provider`, `invalid_vector`, `float32_overflow`,
|
|
8716
|
+
* `provider_error`. All embed-time failures are marked via `markFailure`;
|
|
8717
|
+
* `no_provider` deliberately does not mark (spec §2.4).
|
|
8718
|
+
*
|
|
8719
|
+
* Storage errors are NOT possible here — that is `storeFactVector`'s domain.
|
|
8720
|
+
*/
|
|
8721
|
+
async embedTextForFact(fact, ctx, opts) {
|
|
8722
|
+
const markFailures = opts?.markFailures !== false;
|
|
7358
8723
|
if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
|
|
7359
8724
|
let tagsStr;
|
|
7360
8725
|
if (Array.isArray(fact.tags)) {
|
|
@@ -7375,8 +8740,8 @@ var EmbeddingService = class {
|
|
|
7375
8740
|
const vector = await this.options.llmProvider.embed(text);
|
|
7376
8741
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
7377
8742
|
console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
|
|
7378
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
7379
|
-
await this.markFailure(fact.id, "invalid_vector");
|
|
8743
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
8744
|
+
if (markFailures) await this.markFailure(fact.id, "invalid_vector");
|
|
7380
8745
|
return { ok: false, kind: "invalid_vector" };
|
|
7381
8746
|
}
|
|
7382
8747
|
float32Vector = new Float32Array(vector);
|
|
@@ -7389,16 +8754,24 @@ var EmbeddingService = class {
|
|
|
7389
8754
|
}
|
|
7390
8755
|
if (hasNonFinite) {
|
|
7391
8756
|
console.warn(`[WikiMemory] embedFact: embed() returned values that overflow float32 for ${fact.id}; skipping.`);
|
|
7392
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
7393
|
-
await this.markFailure(fact.id, "float32_overflow");
|
|
8757
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
8758
|
+
if (markFailures) await this.markFailure(fact.id, "float32_overflow");
|
|
7394
8759
|
return { ok: false, kind: "float32_overflow" };
|
|
7395
8760
|
}
|
|
7396
8761
|
} catch (err) {
|
|
7397
8762
|
console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
|
|
7398
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
7399
|
-
await this.markFailure(fact.id, "provider_error");
|
|
8763
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
8764
|
+
if (markFailures) await this.markFailure(fact.id, "provider_error");
|
|
7400
8765
|
return { ok: false, kind: "provider_error" };
|
|
7401
8766
|
}
|
|
8767
|
+
return { ok: true, vector: float32Vector };
|
|
8768
|
+
}
|
|
8769
|
+
/**
|
|
8770
|
+
* Storage half: persist the computed vector to the entry row, then fire the
|
|
8771
|
+
* `onEmbeddingPersisted` hook. Failure kinds: `storage_error` only — embed
|
|
8772
|
+
* failures do not originate here, so they cannot be marked as such (spec §3.2, D3).
|
|
8773
|
+
*/
|
|
8774
|
+
async storeFactVector(fact, float32Vector, ctx) {
|
|
7402
8775
|
try {
|
|
7403
8776
|
await this.storeEmbeddingDimension(float32Vector.length);
|
|
7404
8777
|
const blob = new Uint8Array(float32Vector.buffer);
|
|
@@ -7583,6 +8956,40 @@ function selectWithFloors(sortedRows, floors, maxResults) {
|
|
|
7583
8956
|
return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
|
|
7584
8957
|
}
|
|
7585
8958
|
|
|
8959
|
+
// src/utils/budget.ts
|
|
8960
|
+
function factTokenCost(f) {
|
|
8961
|
+
return estimateTokens(`${f.title} ${f.body} ${f.tags.join(" ")}`);
|
|
8962
|
+
}
|
|
8963
|
+
function packFactsByBudget(items, budget, scores) {
|
|
8964
|
+
const candidates = items.length;
|
|
8965
|
+
if (candidates === 0) return { items: [], tokensUsed: 0, candidates: 0, packed: 0 };
|
|
8966
|
+
let top = items[0];
|
|
8967
|
+
let topCost = factTokenCost(top);
|
|
8968
|
+
if (topCost > budget) {
|
|
8969
|
+
const overhead = estimateTokens(`${top.title} ${top.tags.join(" ")}`);
|
|
8970
|
+
top = { ...top, body: safeSlice(top.body, 0, Math.max(0, (budget - overhead) * 4)) };
|
|
8971
|
+
topCost = factTokenCost(top);
|
|
8972
|
+
}
|
|
8973
|
+
const kept = [{ rank: 0, item: top }];
|
|
8974
|
+
let used = topCost;
|
|
8975
|
+
const rest = items.slice(1).map((item, i) => {
|
|
8976
|
+
const rank = i + 1;
|
|
8977
|
+
const cost = factTokenCost(item);
|
|
8978
|
+
const s = scores?.get(item.id);
|
|
8979
|
+
const value = typeof s === "number" && Number.isFinite(s) && s > 0 ? s : 1 / (rank + 1);
|
|
8980
|
+
return { rank, item, cost, density: value / Math.max(1, cost) };
|
|
8981
|
+
});
|
|
8982
|
+
rest.sort((a, b) => b.density - a.density || a.rank - b.rank);
|
|
8983
|
+
for (const r of rest) {
|
|
8984
|
+
if (used + r.cost <= budget) {
|
|
8985
|
+
kept.push({ rank: r.rank, item: r.item });
|
|
8986
|
+
used += r.cost;
|
|
8987
|
+
}
|
|
8988
|
+
}
|
|
8989
|
+
kept.sort((a, b) => a.rank - b.rank);
|
|
8990
|
+
return { items: kept.map((k) => k.item), tokensUsed: used, candidates, packed: kept.length };
|
|
8991
|
+
}
|
|
8992
|
+
|
|
7586
8993
|
// src/services/RetrievalService.ts
|
|
7587
8994
|
var EMPTY_ID_SET = /* @__PURE__ */ new Set();
|
|
7588
8995
|
var RetrievalService = class {
|
|
@@ -7618,6 +9025,28 @@ var RetrievalService = class {
|
|
|
7618
9025
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
7619
9026
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
7620
9027
|
const excludeDrafts = (options?.excludeDrafts ?? config?.excludeDrafts ?? false) === true;
|
|
9028
|
+
const rawAsOf = options?.asOf;
|
|
9029
|
+
if (rawAsOf !== void 0 && !(typeof rawAsOf === "number" && Number.isFinite(rawAsOf) && rawAsOf >= 0)) {
|
|
9030
|
+
throw new WikiInvalidReadOptions("asOf", "must be a finite epoch-ms number >= 0");
|
|
9031
|
+
}
|
|
9032
|
+
const tokenBudget = options?.tokenBudget;
|
|
9033
|
+
if (tokenBudget !== void 0 && !(typeof tokenBudget === "number" && Number.isFinite(tokenBudget) && tokenBudget >= 0)) {
|
|
9034
|
+
throw new WikiInvalidReadOptions("tokenBudget", "must be a finite number >= 0");
|
|
9035
|
+
}
|
|
9036
|
+
const applyBudget = (input) => {
|
|
9037
|
+
if (tokenBudget === void 0) return input;
|
|
9038
|
+
const packed = packFactsByBudget(input, Math.trunc(tokenBudget), scoreByFactId);
|
|
9039
|
+
emitDiagnostic(this.options, {
|
|
9040
|
+
code: "read_budget",
|
|
9041
|
+
operation: "read",
|
|
9042
|
+
trigger: "call",
|
|
9043
|
+
entityId: entityIds.length === 1 ? entityIds[0] : entityIds.join(","),
|
|
9044
|
+
detail: { candidates: packed.candidates, packed: packed.packed, tokensUsed: packed.tokensUsed }
|
|
9045
|
+
});
|
|
9046
|
+
return packed.items;
|
|
9047
|
+
};
|
|
9048
|
+
const liveMode = rawAsOf === void 0 ? "current" : "asOf";
|
|
9049
|
+
const liveAt = rawAsOf === void 0 ? Date.now() : Math.trunc(rawAsOf);
|
|
7621
9050
|
const trimmedQuery = query.trim();
|
|
7622
9051
|
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
7623
9052
|
entityIds,
|
|
@@ -7637,8 +9066,10 @@ var RetrievalService = class {
|
|
|
7637
9066
|
let usedEmbed = false;
|
|
7638
9067
|
const scoredEntityIds = this._filterScoredEntities(entityIds, sanitizedTierWeights, options?.includeZeroWeightEntities);
|
|
7639
9068
|
const draftIds = excludeDrafts && scoredEntityIds.length > 0 ? await this.entryRepo.findDraftIdsByEntityIds(scoredEntityIds) : EMPTY_ID_SET;
|
|
7640
|
-
const
|
|
7641
|
-
const
|
|
9069
|
+
const nonLiveIds = scoredEntityIds.length > 0 ? await this.entryRepo.findNonLiveIdsByEntityIds(scoredEntityIds, liveMode, liveAt) : EMPTY_ID_SET;
|
|
9070
|
+
const excludedIds = draftIds.size === 0 ? nonLiveIds : nonLiveIds.size === 0 ? draftIds : /* @__PURE__ */ new Set([...draftIds, ...nonLiveIds]);
|
|
9071
|
+
const excludedPad = excludedIds.size;
|
|
9072
|
+
const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + excludedPad;
|
|
7642
9073
|
if (scoredEntityIds.length === 0) {
|
|
7643
9074
|
usedEmbed = true;
|
|
7644
9075
|
} else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
|
|
@@ -7672,7 +9103,7 @@ var RetrievalService = class {
|
|
|
7672
9103
|
let miniSearchScores;
|
|
7673
9104
|
if (effectivePreFilterLimit !== void 0) {
|
|
7674
9105
|
populateCache = false;
|
|
7675
|
-
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !
|
|
9106
|
+
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !excludedIds.has(r.id));
|
|
7676
9107
|
if (preResults.length === 0) {
|
|
7677
9108
|
candidateRows = null;
|
|
7678
9109
|
} else {
|
|
@@ -7694,9 +9125,9 @@ var RetrievalService = class {
|
|
|
7694
9125
|
}
|
|
7695
9126
|
} else {
|
|
7696
9127
|
if (useRanker) {
|
|
7697
|
-
candidateRows = this.
|
|
9128
|
+
candidateRows = this._withoutIds(await this.entryRepo.findMetadataByEntityIds(scoredEntityIds), excludedIds);
|
|
7698
9129
|
} else {
|
|
7699
|
-
candidateRows = this.
|
|
9130
|
+
candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
|
|
7700
9131
|
}
|
|
7701
9132
|
if (weight !== void 0 && weight < 1) {
|
|
7702
9133
|
miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
|
|
@@ -7981,7 +9412,7 @@ var RetrievalService = class {
|
|
|
7981
9412
|
// read() re-sorts after applying tier weights
|
|
7982
9413
|
});
|
|
7983
9414
|
}
|
|
7984
|
-
if (
|
|
9415
|
+
if (excludedPad > 0) scored = scored.filter((s) => !excludedIds.has(s.id));
|
|
7985
9416
|
if (scored.length > 0) {
|
|
7986
9417
|
scored = scored.map((row) => ({
|
|
7987
9418
|
...row,
|
|
@@ -8045,7 +9476,7 @@ var RetrievalService = class {
|
|
|
8045
9476
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
8046
9477
|
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
8047
9478
|
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
|
|
8048
|
-
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !
|
|
9479
|
+
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !excludedIds.has(r.id));
|
|
8049
9480
|
const candidates = results.map((r) => ({
|
|
8050
9481
|
id: r.id,
|
|
8051
9482
|
entity_id: r.entity_id,
|
|
@@ -8063,13 +9494,18 @@ var RetrievalService = class {
|
|
|
8063
9494
|
}
|
|
8064
9495
|
}
|
|
8065
9496
|
}
|
|
9497
|
+
facts = applyBudget(facts);
|
|
8066
9498
|
if (facts.length > 0) {
|
|
8067
9499
|
const ids = facts.map((f) => f.id);
|
|
8068
9500
|
const now = Date.now();
|
|
8069
9501
|
await this.entryRepo.trackAccess(ids, now);
|
|
8070
9502
|
}
|
|
8071
9503
|
} else {
|
|
8072
|
-
facts =
|
|
9504
|
+
facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults, void 0, {
|
|
9505
|
+
...excludeDrafts ? { excludeDrafts: true } : {},
|
|
9506
|
+
live: { mode: liveMode, t: liveAt }
|
|
9507
|
+
});
|
|
9508
|
+
facts = applyBudget(facts);
|
|
8073
9509
|
}
|
|
8074
9510
|
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
8075
9511
|
const [tasks, events] = await Promise.all([
|
|
@@ -8105,8 +9541,8 @@ var RetrievalService = class {
|
|
|
8105
9541
|
_tieBreakSort(items) {
|
|
8106
9542
|
items.sort((a, b) => this._compareScoredRows(a, b));
|
|
8107
9543
|
}
|
|
8108
|
-
|
|
8109
|
-
return
|
|
9544
|
+
_withoutIds(rows, excludeIds) {
|
|
9545
|
+
return excludeIds.size === 0 ? rows : rows.filter((row) => !excludeIds.has(row.id));
|
|
8110
9546
|
}
|
|
8111
9547
|
/**
|
|
8112
9548
|
* Comparator for score + deterministic tie-break fields.
|
|
@@ -8188,6 +9624,18 @@ var WriteService = class {
|
|
|
8188
9624
|
this.metadataRepo = metadataRepo;
|
|
8189
9625
|
this.jobManager = jobManager;
|
|
8190
9626
|
this.maintenanceService = maintenanceService;
|
|
9627
|
+
this.inflight = /* @__PURE__ */ new Set();
|
|
9628
|
+
}
|
|
9629
|
+
track(p) {
|
|
9630
|
+
this.inflight.add(p);
|
|
9631
|
+
p.finally(() => this.inflight.delete(p)).catch(() => {
|
|
9632
|
+
});
|
|
9633
|
+
}
|
|
9634
|
+
/** Await every background job this instance started (spec §6.2). Loops because a finishing job can start another. */
|
|
9635
|
+
async drain() {
|
|
9636
|
+
while (this.inflight.size > 0) {
|
|
9637
|
+
await Promise.allSettled([...this.inflight]);
|
|
9638
|
+
}
|
|
8191
9639
|
}
|
|
8192
9640
|
async write(entityId, event) {
|
|
8193
9641
|
if (typeof entityId !== "string" || entityId.length === 0 || entityId.length > 200 || entityId.includes("\0")) {
|
|
@@ -8214,6 +9662,7 @@ var WriteService = class {
|
|
|
8214
9662
|
}
|
|
8215
9663
|
const id = generateId("evt_");
|
|
8216
9664
|
const now = Date.now();
|
|
9665
|
+
const occurredAt = event.occurred_at == null ? null : assertEpochMs("event.occurred_at", event.occurred_at);
|
|
8217
9666
|
let eventType = event.event_type;
|
|
8218
9667
|
if (!["observation", "decision", "action", "outcome"].includes(eventType)) {
|
|
8219
9668
|
eventType = "observation";
|
|
@@ -8224,8 +9673,14 @@ var WriteService = class {
|
|
|
8224
9673
|
event_type: eventType,
|
|
8225
9674
|
summary,
|
|
8226
9675
|
related_entry_id: relatedEntryId,
|
|
8227
|
-
created_at: now
|
|
9676
|
+
created_at: now,
|
|
9677
|
+
occurred_at: occurredAt
|
|
8228
9678
|
};
|
|
9679
|
+
const deferred = this.options.config?.maintenance === "deferred";
|
|
9680
|
+
if (deferred) {
|
|
9681
|
+
await this.eventRepo.add(newEvent);
|
|
9682
|
+
return;
|
|
9683
|
+
}
|
|
8229
9684
|
let shouldRunLibrarian = false;
|
|
8230
9685
|
let librarianCount = 0;
|
|
8231
9686
|
let prevMemoryCheckpoint = 0;
|
|
@@ -8247,27 +9702,41 @@ var WriteService = class {
|
|
|
8247
9702
|
prevMemoryCheckpoint = memoryCheckpoint;
|
|
8248
9703
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
8249
9704
|
}
|
|
9705
|
+
} else {
|
|
9706
|
+
const tokenThreshold = this.options.config?.autoLibrarianTokenThreshold;
|
|
9707
|
+
if (typeof tokenThreshold === "number" && tokenThreshold > 0 && !this.jobManager.isBlocked("librarian", entityId)) {
|
|
9708
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, tx);
|
|
9709
|
+
const chars = await this.eventRepo.sumSummaryCharsAfter(entityId, wm, tx);
|
|
9710
|
+
if (Math.ceil(chars / 4) >= tokenThreshold) {
|
|
9711
|
+
shouldRunLibrarian = true;
|
|
9712
|
+
librarianCount = count;
|
|
9713
|
+
prevMemoryCheckpoint = memoryCheckpoint;
|
|
9714
|
+
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
9715
|
+
}
|
|
9716
|
+
}
|
|
8250
9717
|
}
|
|
8251
9718
|
});
|
|
8252
9719
|
if (shouldRunLibrarian) {
|
|
8253
9720
|
try {
|
|
8254
9721
|
this.jobManager.acquireLock("librarian", entityId);
|
|
8255
9722
|
const stage = { job: "librarian" };
|
|
8256
|
-
this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
9723
|
+
const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
8257
9724
|
console.error(err);
|
|
8258
9725
|
this.reportBackgroundFailure(entityId, stage.job);
|
|
8259
9726
|
}).finally(() => {
|
|
8260
9727
|
this.jobManager.releaseLock("librarian", entityId);
|
|
8261
9728
|
});
|
|
9729
|
+
this.track(job);
|
|
8262
9730
|
} catch (e) {
|
|
8263
9731
|
if (!(e instanceof WikiBusyError)) throw e;
|
|
8264
9732
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: prevMemoryCheckpoint }, this.db);
|
|
8265
9733
|
}
|
|
8266
9734
|
} else if (!this.jobManager.isBlocked("librarian", entityId)) {
|
|
8267
|
-
this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
9735
|
+
const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
8268
9736
|
console.error(err);
|
|
8269
9737
|
this.reportBackgroundFailure(entityId, "heal");
|
|
8270
9738
|
});
|
|
9739
|
+
this.track(healJob);
|
|
8271
9740
|
}
|
|
8272
9741
|
}
|
|
8273
9742
|
async runLibrarianThenMaybeHeal(entityId, currentEventCount, prevCheckpoint, stage) {
|
|
@@ -8498,6 +9967,15 @@ var GraphTraversalService = class {
|
|
|
8498
9967
|
const defaultMaxNodes = Number.isFinite(rawConfigDefault) && rawConfigDefault >= 1 ? Math.floor(rawConfigDefault) : fallbackMaxNodes;
|
|
8499
9968
|
const rawMaxNodes = options.maxTraversalNodes ?? defaultMaxNodes;
|
|
8500
9969
|
const maxNodes = Number.isFinite(rawMaxNodes) && rawMaxNodes >= 1 ? Math.floor(rawMaxNodes) : defaultMaxNodes;
|
|
9970
|
+
const rawAsOf = options.asOf;
|
|
9971
|
+
if (rawAsOf !== void 0 && !(typeof rawAsOf === "number" && Number.isFinite(rawAsOf) && rawAsOf >= 0)) {
|
|
9972
|
+
throw new WikiInvalidReadOptions("asOf", "must be a finite epoch-ms number >= 0");
|
|
9973
|
+
}
|
|
9974
|
+
const rawTokenBudget = options.tokenBudget;
|
|
9975
|
+
if (rawTokenBudget !== void 0 && !(typeof rawTokenBudget === "number" && Number.isFinite(rawTokenBudget) && rawTokenBudget >= 0)) {
|
|
9976
|
+
throw new WikiInvalidReadOptions("tokenBudget", "must be a finite number >= 0");
|
|
9977
|
+
}
|
|
9978
|
+
const live = rawAsOf === void 0 ? { mode: "current", t: Date.now() } : { mode: "asOf", t: Math.trunc(rawAsOf) };
|
|
8501
9979
|
const opts = {
|
|
8502
9980
|
maxDepth: Math.max(1, Math.min(options.maxDepth ?? 1, 3)),
|
|
8503
9981
|
direction: options.direction ?? this.config.traversalDirection ?? "both",
|
|
@@ -8505,16 +9983,32 @@ var GraphTraversalService = class {
|
|
|
8505
9983
|
minConfidence: options.minTraversalConfidence ?? this.config.minTraversalConfidence ?? "tentative",
|
|
8506
9984
|
excludeSourceTypes: options.excludeSourceTypes ?? this.config.excludeSourceTypes ?? [],
|
|
8507
9985
|
excludeDrafts: options.excludeDrafts ?? this.config.excludeDrafts ?? false,
|
|
9986
|
+
live,
|
|
8508
9987
|
maxNodes
|
|
8509
9988
|
};
|
|
8510
9989
|
const { nodeIds, edges } = await this.edgeRepo.getNeighborhood(entityId, options.sourceId, opts);
|
|
8511
9990
|
if (nodeIds.length === 0) return { nodes: [], edges: [] };
|
|
9991
|
+
const [anchor] = await this.entryRepo.findByIds([options.sourceId], [entityId]);
|
|
9992
|
+
if (anchor && !isLiveAt(anchor, live.mode, live.t)) return { nodes: [anchor], edges: [] };
|
|
8512
9993
|
const nodes = await this.entryRepo.findByIds(nodeIds, [entityId]);
|
|
8513
9994
|
const hydratedIds = new Set(nodes.map((node) => node.id));
|
|
8514
9995
|
const filteredEdges = edges.filter(
|
|
8515
9996
|
(edge) => hydratedIds.has(edge.source_id) && hydratedIds.has(edge.target_id)
|
|
8516
9997
|
);
|
|
8517
|
-
return { nodes, edges: filteredEdges };
|
|
9998
|
+
if (options.tokenBudget === void 0) return { nodes, edges: filteredEdges };
|
|
9999
|
+
const budget = Math.trunc(options.tokenBudget);
|
|
10000
|
+
const [anchorPacked] = packFactsByBudget([nodes[0]], budget).items;
|
|
10001
|
+
const kept = [anchorPacked];
|
|
10002
|
+
let used = factTokenCost(anchorPacked);
|
|
10003
|
+
for (const node of nodes.slice(1)) {
|
|
10004
|
+
const cost = factTokenCost(node);
|
|
10005
|
+
if (used + cost <= budget) {
|
|
10006
|
+
kept.push(node);
|
|
10007
|
+
used += cost;
|
|
10008
|
+
}
|
|
10009
|
+
}
|
|
10010
|
+
const keptIds = new Set(kept.map((n) => n.id));
|
|
10011
|
+
return { nodes: kept, edges: filteredEdges.filter((e) => keptIds.has(e.source_id) && keptIds.has(e.target_id)) };
|
|
8518
10012
|
}
|
|
8519
10013
|
};
|
|
8520
10014
|
|
|
@@ -8815,6 +10309,7 @@ var WikiMemory = class {
|
|
|
8815
10309
|
this.lintService = new LintService(this.lintRepo, this.ontologyService);
|
|
8816
10310
|
this.embeddingService = new EmbeddingService(this.db, this.options, this.entryRepo, this.metadataRepo);
|
|
8817
10311
|
this.searchService = new SearchService(this.entryRepo);
|
|
10312
|
+
this.temporalService = new TemporalService(this.db, this.options, this.entryRepo, this.searchService, this.embeddingService);
|
|
8818
10313
|
this.jobManager = new JobManager(this.prefix);
|
|
8819
10314
|
this.promptService = new PromptService(options.config?.prompts, resolveGrounding(options.config?.grounding));
|
|
8820
10315
|
this.ingestionService = new IngestionService(
|
|
@@ -8846,6 +10341,15 @@ var WikiMemory = class {
|
|
|
8846
10341
|
this.promptService,
|
|
8847
10342
|
this.ontologyService
|
|
8848
10343
|
);
|
|
10344
|
+
this.maintenanceScheduler = new MaintenanceScheduler(
|
|
10345
|
+
this.db,
|
|
10346
|
+
this.options,
|
|
10347
|
+
this.maintenanceService,
|
|
10348
|
+
this.metadataRepo,
|
|
10349
|
+
this.eventRepo,
|
|
10350
|
+
this.entryRepo,
|
|
10351
|
+
this.jobManager
|
|
10352
|
+
);
|
|
8849
10353
|
this.importExportService = new ImportExportService(
|
|
8850
10354
|
this.db,
|
|
8851
10355
|
this.entryRepo,
|
|
@@ -8896,11 +10400,13 @@ var WikiMemory = class {
|
|
|
8896
10400
|
importExportService: this.importExportService,
|
|
8897
10401
|
ingestionService: this.ingestionService,
|
|
8898
10402
|
maintenanceService: this.maintenanceService,
|
|
10403
|
+
maintenanceScheduler: this.maintenanceScheduler,
|
|
8899
10404
|
retrievalService: this.retrievalService,
|
|
8900
10405
|
searchService: this.searchService,
|
|
8901
10406
|
writeService: this.writeService,
|
|
8902
10407
|
promptService: this.promptService,
|
|
8903
10408
|
graphTraversalService: this.graphTraversalService,
|
|
10409
|
+
temporalService: this.temporalService,
|
|
8904
10410
|
entryRepo: this.entryRepo,
|
|
8905
10411
|
sourceRefIndexRepo: this.sourceRefIndexRepo,
|
|
8906
10412
|
metadataRepo: this.metadataRepo,
|
|
@@ -9118,6 +10624,23 @@ var WikiMemory = class {
|
|
|
9118
10624
|
async write(entityId, event) {
|
|
9119
10625
|
return this.writeService.write(entityId, event);
|
|
9120
10626
|
}
|
|
10627
|
+
/** Await auto-mode background librarian/heal jobs started by write(). No-op in 'deferred' mode. */
|
|
10628
|
+
async drain() {
|
|
10629
|
+
return this.writeService.drain();
|
|
10630
|
+
}
|
|
10631
|
+
/**
|
|
10632
|
+
* Report which entities have pending maintenance work (librarian events
|
|
10633
|
+
* behind the watermark, heal backlog, unembedded facts) without running
|
|
10634
|
+
* anything. Entities with no pending work are omitted; sorted by
|
|
10635
|
+
* `pendingEvents` desc then `entityId` asc.
|
|
10636
|
+
*/
|
|
10637
|
+
async getPendingMaintenance(entityIds) {
|
|
10638
|
+
return this.maintenanceScheduler.getPending(entityIds);
|
|
10639
|
+
}
|
|
10640
|
+
/** Run pending librarian/heal/reembed work under an optional token budget and deadline (spec §6.2). Nothing runs after it resolves. */
|
|
10641
|
+
async runPendingMaintenance(options) {
|
|
10642
|
+
return this.maintenanceScheduler.run(options);
|
|
10643
|
+
}
|
|
9121
10644
|
/**
|
|
9122
10645
|
* @param options.promptOverride - Applies only to this manual call. Does NOT affect
|
|
9123
10646
|
* WriteService-triggered auto-runs. For persistent prompt customization across auto-runs,
|
|
@@ -9178,6 +10701,32 @@ var WikiMemory = class {
|
|
|
9178
10701
|
clearVectorCache() {
|
|
9179
10702
|
this.searchService.evictCache();
|
|
9180
10703
|
}
|
|
10704
|
+
/**
|
|
10705
|
+
* Brings the keyword search index up to date with SQLite after a host
|
|
10706
|
+
* transaction that used `upsertGraph` has committed. Core never sees that
|
|
10707
|
+
* commit, so without this the nodes stay out of keyword, hybrid and
|
|
10708
|
+
* fallback retrieval until the entity's next core write.
|
|
10709
|
+
*
|
|
10710
|
+
* With `entityId`, rebuilds that entity if it is stale or has never been
|
|
10711
|
+
* indexed; otherwise it is a no-op that still waits for rebuilds already
|
|
10712
|
+
* queued. Without an argument, rebuilds every entity currently marked
|
|
10713
|
+
* stale; when none is, it resolves immediately and does not wait for rebuilds
|
|
10714
|
+
* already queued.
|
|
10715
|
+
*
|
|
10716
|
+
* Call only after the host's transaction commits: called inside it, the
|
|
10717
|
+
* rebuild may index rows that later roll back, or miss them and clear the
|
|
10718
|
+
* stale flag anyway. Serialized with core's internal index syncs and takes
|
|
10719
|
+
* no maintenance lock. Never rejects for a valid `entityId` — on failure the
|
|
10720
|
+
* entity stays stale and the next call retries it. Does not compute
|
|
10721
|
+
* embeddings; that remains the maintenance sweep's job.
|
|
10722
|
+
*
|
|
10723
|
+
* @throws {TypeError} when `entityId` is given but is not a non-empty string.
|
|
10724
|
+
*/
|
|
10725
|
+
async syncSearchIndex(entityId) {
|
|
10726
|
+
if (entityId === void 0) return this.searchService.syncStale();
|
|
10727
|
+
assertEntityId(entityId);
|
|
10728
|
+
return this.searchService.syncEntries(entityId, []);
|
|
10729
|
+
}
|
|
9181
10730
|
async exportDump(entityIds) {
|
|
9182
10731
|
return this.importExportService.exportDump(entityIds);
|
|
9183
10732
|
}
|
|
@@ -9191,6 +10740,18 @@ var WikiMemory = class {
|
|
|
9191
10740
|
async forget(entityId, params, opts) {
|
|
9192
10741
|
return this.maintenanceService.forget(entityId, params, opts);
|
|
9193
10742
|
}
|
|
10743
|
+
/**
|
|
10744
|
+
* End `oldId`'s validity and link it to its replacement (spec §4.3). One
|
|
10745
|
+
* transaction. Throws `WikiSupersedeError` for missing, foreign, already
|
|
10746
|
+
* superseded, immutable-document, or cyclic targets.
|
|
10747
|
+
*/
|
|
10748
|
+
async supersede(entityId, oldId, replacement, options) {
|
|
10749
|
+
return this.temporalService.supersede(entityId, oldId, replacement, options);
|
|
10750
|
+
}
|
|
10751
|
+
/** The supersession chain containing `factId`, oldest first. [] when unknown. */
|
|
10752
|
+
async history(entityId, factId) {
|
|
10753
|
+
return this.temporalService.history(entityId, factId);
|
|
10754
|
+
}
|
|
9194
10755
|
/**
|
|
9195
10756
|
* Full and partial ingestion both reject foreign-owned generated fact IDs with
|
|
9196
10757
|
* `WikiGraphNodeOwnershipConflict`. Such collisions are unexpected. Ingestion
|
|
@@ -9212,9 +10773,11 @@ var WikiMemory = class {
|
|
|
9212
10773
|
* MUST be called from inside an open `db.withTransactionAsync` callback —
|
|
9213
10774
|
* the third argument is the caller's `tx`. The method does not open a
|
|
9214
10775
|
* nested transaction, does not acquire any lock, and does not perform
|
|
9215
|
-
* post-commit work (search sync, embedding, cache eviction).
|
|
9216
|
-
*
|
|
9217
|
-
* (`
|
|
10776
|
+
* post-commit work (search sync, embedding, cache eviction). After the
|
|
10777
|
+
* transaction commits, call `syncSearchIndex(entityId)` — or
|
|
10778
|
+
* `syncSearchIndex()` after writing several entities in one transaction —
|
|
10779
|
+
* to make the new nodes keyword-searchable. Embeddings still come from the maintenance
|
|
10780
|
+
* sweep (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
|
|
9218
10781
|
* scoped via `listEntityIds`).
|
|
9219
10782
|
*
|
|
9220
10783
|
* Contract (full pre-flight validation lives inside
|
|
@@ -9509,10 +11072,11 @@ function formatContext(bundle, options) {
|
|
|
9509
11072
|
maxFacts: options?.maxFacts ?? 10,
|
|
9510
11073
|
maxTasks: options?.maxTasks ?? 10,
|
|
9511
11074
|
maxEvents: options?.maxEvents ?? 10,
|
|
9512
|
-
includeConfidence: options?.includeConfidence ?? true,
|
|
9513
|
-
includeTags: options?.includeTags ?? true,
|
|
9514
|
-
includeEntityIds: options?.includeEntityIds ?? false,
|
|
9515
|
-
includeFactScores: options?.includeFactScores ?? false,
|
|
11075
|
+
includeConfidence: !(options?.compact ?? false) && (options?.includeConfidence ?? true),
|
|
11076
|
+
includeTags: !(options?.compact ?? false) && (options?.includeTags ?? true),
|
|
11077
|
+
includeEntityIds: !(options?.compact ?? false) && (options?.includeEntityIds ?? false),
|
|
11078
|
+
includeFactScores: !(options?.compact ?? false) && (options?.includeFactScores ?? false),
|
|
11079
|
+
compact: options?.compact ?? false,
|
|
9516
11080
|
factWeights: {
|
|
9517
11081
|
confidence: options?.factWeights?.confidence ?? 1,
|
|
9518
11082
|
accessCount: options?.factWeights?.accessCount ?? 0.3,
|
|
@@ -9803,7 +11367,12 @@ function factFrontmatterV2(f) {
|
|
|
9803
11367
|
// `verified` key", not from "empty `verified` list").
|
|
9804
11368
|
...f.okf_verified && f.okf_verified.length > 0 ? { verified: f.okf_verified } : {},
|
|
9805
11369
|
...f.okf_sources && f.okf_sources.length > 0 ? { sources: f.okf_sources } : {},
|
|
9806
|
-
...f.okf_usage_window ? { usage_window: f.okf_usage_window } : {}
|
|
11370
|
+
...f.okf_usage_window ? { usage_window: f.okf_usage_window } : {},
|
|
11371
|
+
// Temporal (spec 2026-09-29 §10.9): flat epoch-ms keys, like created_at.
|
|
11372
|
+
...f.valid_from != null ? { valid_from: f.valid_from } : {},
|
|
11373
|
+
...f.valid_to != null ? { valid_to: f.valid_to } : {},
|
|
11374
|
+
...f.superseded_by != null ? { superseded_by: f.superseded_by } : {},
|
|
11375
|
+
...f.superseded_at != null ? { superseded_at: f.superseded_at } : {}
|
|
9807
11376
|
};
|
|
9808
11377
|
}
|
|
9809
11378
|
function taskFrontmatterV2(t) {
|
|
@@ -10131,9 +11700,16 @@ function frontmatterToFact(entityId, id, frontmatter, body, now) {
|
|
|
10131
11700
|
okf_verified: verified,
|
|
10132
11701
|
okf_usage_window: usageWindow,
|
|
10133
11702
|
last_verified_at: lastV?.at ?? null,
|
|
10134
|
-
last_verified_by: lastV?.by ?? null
|
|
11703
|
+
last_verified_by: lastV?.by ?? null,
|
|
11704
|
+
...optionalEpoch("valid_from", frontmatter.valid_from),
|
|
11705
|
+
...optionalEpoch("valid_to", frontmatter.valid_to),
|
|
11706
|
+
...typeof frontmatter.superseded_by === "string" ? { superseded_by: frontmatter.superseded_by } : {},
|
|
11707
|
+
...optionalEpoch("superseded_at", frontmatter.superseded_at)
|
|
10135
11708
|
};
|
|
10136
11709
|
}
|
|
11710
|
+
function optionalEpoch(key, v) {
|
|
11711
|
+
return typeof v === "number" && Number.isFinite(v) && v >= 0 ? { [key]: Math.trunc(v) } : {};
|
|
11712
|
+
}
|
|
10137
11713
|
function frontmatterToTask(entityId, id, frontmatter, now, usesV1StatusRule) {
|
|
10138
11714
|
const created_at = parseFrontmatterTimestamp(frontmatter.created_at, now);
|
|
10139
11715
|
const updated_at = parseFrontmatterTimestamp(
|
|
@@ -10390,6 +11966,7 @@ exports.WikiMemory = WikiMemory;
|
|
|
10390
11966
|
exports.WikiParseError = WikiParseError;
|
|
10391
11967
|
exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
|
|
10392
11968
|
exports.WikiStrictOntologyViolation = WikiStrictOntologyViolation;
|
|
11969
|
+
exports.WikiSupersedeError = WikiSupersedeError;
|
|
10393
11970
|
exports.WikiTransactionError = WikiTransactionError;
|
|
10394
11971
|
exports.chunkText = chunkText;
|
|
10395
11972
|
exports.configureRandomSource = configureRandomSource;
|