@equationalapplications/core-llm-wiki 7.7.6 → 7.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -0
- package/dist/{chunk-PBUZLSOM.mjs → chunk-M76CF2Q6.mjs} +1193 -189
- package/dist/chunk-M76CF2Q6.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1837 -289
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +575 -32
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-BIFNLUva.d.mts → testing-CWZaXKLj.d.mts} +560 -4
- package/dist/{testing-BIFNLUva.d.ts → testing-CWZaXKLj.d.ts} +560 -4
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +1142 -201
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-PBUZLSOM.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -48,7 +48,11 @@ async function setupDatabase(db, prefix) {
|
|
|
48
48
|
okf_usage_window TEXT,
|
|
49
49
|
embedding_failed_at INTEGER,
|
|
50
50
|
embedding_failure_kind TEXT,
|
|
51
|
-
embedding_attempts INTEGER NOT NULL DEFAULT 0
|
|
51
|
+
embedding_attempts INTEGER NOT NULL DEFAULT 0,
|
|
52
|
+
valid_from INTEGER,
|
|
53
|
+
valid_to INTEGER,
|
|
54
|
+
superseded_by TEXT,
|
|
55
|
+
superseded_at INTEGER
|
|
52
56
|
);
|
|
53
57
|
|
|
54
58
|
CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
|
|
@@ -115,7 +119,8 @@ async function setupDatabase(db, prefix) {
|
|
|
115
119
|
event_type TEXT NOT NULL,
|
|
116
120
|
summary TEXT NOT NULL,
|
|
117
121
|
related_entry_id TEXT,
|
|
118
|
-
created_at INTEGER NOT NULL
|
|
122
|
+
created_at INTEGER NOT NULL,
|
|
123
|
+
occurred_at INTEGER
|
|
119
124
|
);
|
|
120
125
|
|
|
121
126
|
CREATE INDEX IF NOT EXISTS ${prefix}events_entity_idx ON ${prefix}events(entity_id, created_at DESC);
|
|
@@ -123,7 +128,9 @@ async function setupDatabase(db, prefix) {
|
|
|
123
128
|
CREATE TABLE IF NOT EXISTS ${prefix}checkpoints (
|
|
124
129
|
entity_id TEXT PRIMARY KEY,
|
|
125
130
|
heal_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
126
|
-
memory_checkpoint INTEGER NOT NULL DEFAULT 0
|
|
131
|
+
memory_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
132
|
+
librarian_watermark_at INTEGER,
|
|
133
|
+
librarian_watermark_id TEXT
|
|
127
134
|
);
|
|
128
135
|
|
|
129
136
|
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
@@ -155,6 +162,7 @@ async function setupDatabase(db, prefix) {
|
|
|
155
162
|
ON ${prefix}outbox (created_at);
|
|
156
163
|
`);
|
|
157
164
|
await createOkfV02IndexesIfColumnsExist(db, prefix);
|
|
165
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
158
166
|
}
|
|
159
167
|
async function createOkfV02IndexesIfColumnsExist(db, prefix) {
|
|
160
168
|
for (const table of ["entries", "tasks"]) {
|
|
@@ -173,6 +181,20 @@ async function createOkfV02IndexesIfColumnsExist(db, prefix) {
|
|
|
173
181
|
}
|
|
174
182
|
}
|
|
175
183
|
}
|
|
184
|
+
async function createTemporalIndexesIfColumnsExist(db, prefix) {
|
|
185
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}entries)`);
|
|
186
|
+
const names = new Set(cols.map((c) => c.name));
|
|
187
|
+
if (names.has("superseded_by")) {
|
|
188
|
+
await db.execAsync(
|
|
189
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_superseded_idx ON ${prefix}entries(entity_id, superseded_by) WHERE superseded_by IS NOT NULL;`
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
if (names.has("valid_from") && names.has("valid_to")) {
|
|
193
|
+
await db.execAsync(
|
|
194
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_temporal_idx ON ${prefix}entries(entity_id) WHERE valid_from IS NOT NULL OR valid_to IS NOT NULL;`
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
176
198
|
|
|
177
199
|
// src/db/sqliteCodes.ts
|
|
178
200
|
var SQLITE_RESULT_CODE_NAMES = {
|
|
@@ -341,6 +363,15 @@ var WikiDraftNotFound = class _WikiDraftNotFound extends Error {
|
|
|
341
363
|
Object.setPrototypeOf(this, _WikiDraftNotFound.prototype);
|
|
342
364
|
}
|
|
343
365
|
};
|
|
366
|
+
var WikiSupersedeError = class _WikiSupersedeError extends Error {
|
|
367
|
+
constructor(reason) {
|
|
368
|
+
super(`supersede() rejected: ${reason}`);
|
|
369
|
+
this.code = "WIKI_SUPERSEDE_REJECTED";
|
|
370
|
+
this.name = "WikiSupersedeError";
|
|
371
|
+
this.reason = reason;
|
|
372
|
+
Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
|
|
373
|
+
}
|
|
374
|
+
};
|
|
344
375
|
|
|
345
376
|
// src/db/serializedAdapter.ts
|
|
346
377
|
function isDriverError(err) {
|
|
@@ -630,6 +661,35 @@ var MIGRATIONS = [
|
|
|
630
661
|
DROP INDEX IF EXISTS ${prefix}edges_entity_idx;
|
|
631
662
|
`);
|
|
632
663
|
}
|
|
664
|
+
},
|
|
665
|
+
{
|
|
666
|
+
version: 13,
|
|
667
|
+
description: "Temporal facts: valid_from/valid_to/superseded_by/superseded_at on entries, occurred_at on events, librarian watermark on checkpoints",
|
|
668
|
+
run: async (db, prefix) => {
|
|
669
|
+
const adds = [
|
|
670
|
+
["entries", "valid_from", "INTEGER"],
|
|
671
|
+
["entries", "valid_to", "INTEGER"],
|
|
672
|
+
["entries", "superseded_by", "TEXT"],
|
|
673
|
+
["entries", "superseded_at", "INTEGER"],
|
|
674
|
+
["events", "occurred_at", "INTEGER"],
|
|
675
|
+
["checkpoints", "librarian_watermark_at", "INTEGER"],
|
|
676
|
+
["checkpoints", "librarian_watermark_id", "TEXT"]
|
|
677
|
+
];
|
|
678
|
+
const existingByTable = /* @__PURE__ */ new Map();
|
|
679
|
+
for (const [table, column, type] of adds) {
|
|
680
|
+
let existing = existingByTable.get(table);
|
|
681
|
+
if (!existing) {
|
|
682
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}${table})`);
|
|
683
|
+
existing = new Set(cols.map((c) => c.name));
|
|
684
|
+
existingByTable.set(table, existing);
|
|
685
|
+
}
|
|
686
|
+
if (!existing.has(column)) {
|
|
687
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN ${column} ${type}`);
|
|
688
|
+
existing.add(column);
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
692
|
+
}
|
|
633
693
|
}
|
|
634
694
|
];
|
|
635
695
|
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
@@ -679,6 +739,24 @@ function parseJsonObject(s, fallback = null) {
|
|
|
679
739
|
}
|
|
680
740
|
return fallback;
|
|
681
741
|
}
|
|
742
|
+
|
|
743
|
+
// src/utils/temporal.ts
|
|
744
|
+
function liveAtSql(mode, alias = "") {
|
|
745
|
+
const from = mode === "asOf" ? `COALESCE(${alias}valid_from, ${alias}created_at) <= ?` : `(${alias}valid_from IS NULL OR ${alias}valid_from <= ?)`;
|
|
746
|
+
return `(${from} AND (${alias}valid_to IS NULL OR ${alias}valid_to > ?))`;
|
|
747
|
+
}
|
|
748
|
+
function isLiveAt(f, mode, t) {
|
|
749
|
+
if (f.deleted_at != null) return false;
|
|
750
|
+
const from = mode === "asOf" ? f.valid_from ?? f.created_at : f.valid_from;
|
|
751
|
+
if (from != null && from > t) return false;
|
|
752
|
+
return f.valid_to == null || f.valid_to > t;
|
|
753
|
+
}
|
|
754
|
+
function assertEpochMs(name, v) {
|
|
755
|
+
if (typeof v !== "number" || !Number.isFinite(v) || v < 0) {
|
|
756
|
+
throw new TypeError(`${name} must be a finite epoch-ms number >= 0`);
|
|
757
|
+
}
|
|
758
|
+
return Math.trunc(v);
|
|
759
|
+
}
|
|
682
760
|
function mapRowToFact(row) {
|
|
683
761
|
const tags = (() => {
|
|
684
762
|
if (Array.isArray(row.tags)) return row.tags;
|
|
@@ -719,7 +797,13 @@ function mapRowToFact(row) {
|
|
|
719
797
|
last_verified_by: row.last_verified_by ?? null,
|
|
720
798
|
// Spec §2.7 + §5.3: hydrate so read() consumers don't re-call the helpers.
|
|
721
799
|
isStale: coreOkf.isStaleAfter(staleAfterRaw, now),
|
|
722
|
-
trustTier: coreOkf.deriveTrustTier(okf_verified)
|
|
800
|
+
trustTier: coreOkf.deriveTrustTier(okf_verified),
|
|
801
|
+
// Temporal (spec 2026-09-29 §4, §10.8): keys appear only when set so
|
|
802
|
+
// existing deep-equality assertions and host snapshots are unchanged.
|
|
803
|
+
...row.valid_from != null ? { valid_from: Number(row.valid_from) } : {},
|
|
804
|
+
...row.valid_to != null ? { valid_to: Number(row.valid_to) } : {},
|
|
805
|
+
...row.superseded_by != null ? { superseded_by: String(row.superseded_by) } : {},
|
|
806
|
+
...row.superseded_at != null ? { superseded_at: Number(row.superseded_at) } : {}
|
|
723
807
|
};
|
|
724
808
|
}
|
|
725
809
|
function normalizeEmbeddingBlobValue(blob) {
|
|
@@ -1140,12 +1224,14 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1140
1224
|
*/
|
|
1141
1225
|
async findHealCandidatesByEntityId(entityId, limit, recheckCutoff, tx) {
|
|
1142
1226
|
const executor = this.getExecutor(tx);
|
|
1227
|
+
const now = Date.now();
|
|
1143
1228
|
const rows = await executor.getAllAsync(
|
|
1144
1229
|
`SELECT * FROM ${this.prefix}entries
|
|
1145
1230
|
WHERE entity_id = ? AND deleted_at IS NULL AND source_type != 'immutable_document'
|
|
1231
|
+
AND ${liveAtSql("current")}
|
|
1146
1232
|
AND (heal_checked_at IS NULL OR heal_checked_at <= ?)
|
|
1147
1233
|
ORDER BY updated_at ASC LIMIT ?`,
|
|
1148
|
-
[entityId, recheckCutoff, limit]
|
|
1234
|
+
[entityId, now, now, recheckCutoff, limit]
|
|
1149
1235
|
);
|
|
1150
1236
|
return rows.map(mapRowToFact);
|
|
1151
1237
|
}
|
|
@@ -1159,6 +1245,7 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1159
1245
|
const executor = this.getExecutor(tx);
|
|
1160
1246
|
const columns = opts?.withBody ? "id, title, source_ref, body, lifecycle_status" : "id, title, source_ref";
|
|
1161
1247
|
const rows = [];
|
|
1248
|
+
const now = Date.now();
|
|
1162
1249
|
for (let i = 0; i < ids.length; i += this.chunkSize) {
|
|
1163
1250
|
const chunk = ids.slice(i, i + this.chunkSize);
|
|
1164
1251
|
const placeholders = chunk.map(() => "?").join(", ");
|
|
@@ -1166,8 +1253,9 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1166
1253
|
`SELECT ${columns} FROM ${this.prefix}entries
|
|
1167
1254
|
WHERE entity_id = ? AND deleted_at IS NULL
|
|
1168
1255
|
AND source_type = 'immutable_document'
|
|
1256
|
+
AND ${liveAtSql("current")}
|
|
1169
1257
|
AND id IN (${placeholders})`,
|
|
1170
|
-
[entityId, ...chunk]
|
|
1258
|
+
[entityId, now, now, ...chunk]
|
|
1171
1259
|
);
|
|
1172
1260
|
rows.push(...chunkRows);
|
|
1173
1261
|
}
|
|
@@ -1187,10 +1275,11 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1187
1275
|
*/
|
|
1188
1276
|
async findRecentByEntityId(entityId, limit, tx, excludeSourceRef) {
|
|
1189
1277
|
const executor = this.getExecutor(tx);
|
|
1278
|
+
const now = Date.now();
|
|
1190
1279
|
const excludeClause = excludeSourceRef ? ` AND (source_ref IS NULL OR source_ref != ?)` : "";
|
|
1191
|
-
const args = excludeSourceRef ? [entityId, excludeSourceRef, limit] : [entityId, limit];
|
|
1280
|
+
const args = excludeSourceRef ? [entityId, now, now, excludeSourceRef, limit] : [entityId, now, now, limit];
|
|
1192
1281
|
const rows = await executor.getAllAsync(
|
|
1193
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL${excludeClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
1282
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL AND ${liveAtSql("current")}${excludeClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
1194
1283
|
args
|
|
1195
1284
|
);
|
|
1196
1285
|
return rows.map(mapRowToFact);
|
|
@@ -1398,6 +1487,97 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1398
1487
|
}, tx);
|
|
1399
1488
|
}
|
|
1400
1489
|
}
|
|
1490
|
+
/** Spec §4: write any subset of the four temporal columns. Absent key = untouched; null = clear. */
|
|
1491
|
+
async setTemporal(id, entityId, fields, tx, opts) {
|
|
1492
|
+
const keys = ["valid_from", "valid_to", "superseded_by", "superseded_at"].filter((k) => k in fields);
|
|
1493
|
+
if (keys.length === 0) return 0;
|
|
1494
|
+
const executor = this.getExecutor(tx);
|
|
1495
|
+
const result = await executor.runAsync(
|
|
1496
|
+
`UPDATE ${this.prefix}entries SET ${keys.map((k) => `${k} = ?`).join(", ")} WHERE id = ? AND entity_id = ?`,
|
|
1497
|
+
[...keys.map((k) => fields[k] ?? null), id, entityId]
|
|
1498
|
+
);
|
|
1499
|
+
if (result.changes > 0 && opts?.outbox !== false) {
|
|
1500
|
+
const payload = { id, entity_id: entityId };
|
|
1501
|
+
for (const k of keys) payload[k] = fields[k] ?? null;
|
|
1502
|
+
await this.outbox.push({ entityId, tableName: "entries", recordId: id, operation: "UPDATE", payload }, tx);
|
|
1503
|
+
}
|
|
1504
|
+
return result.changes;
|
|
1505
|
+
}
|
|
1506
|
+
async findTemporalRow(id, tx) {
|
|
1507
|
+
const executor = this.getExecutor(tx);
|
|
1508
|
+
const row = await executor.getFirstAsync(
|
|
1509
|
+
`SELECT id, entity_id, source_type, deleted_at, created_at, valid_from, valid_to, superseded_by
|
|
1510
|
+
FROM ${this.prefix}entries WHERE id = ?`,
|
|
1511
|
+
[id]
|
|
1512
|
+
);
|
|
1513
|
+
if (!row) return null;
|
|
1514
|
+
return {
|
|
1515
|
+
id: String(row.id),
|
|
1516
|
+
entity_id: String(row.entity_id),
|
|
1517
|
+
source_type: row.source_type,
|
|
1518
|
+
deleted_at: row.deleted_at != null ? Number(row.deleted_at) : null,
|
|
1519
|
+
created_at: Number(row.created_at),
|
|
1520
|
+
valid_from: row.valid_from != null ? Number(row.valid_from) : null,
|
|
1521
|
+
valid_to: row.valid_to != null ? Number(row.valid_to) : null,
|
|
1522
|
+
superseded_by: row.superseded_by != null ? String(row.superseded_by) : null
|
|
1523
|
+
};
|
|
1524
|
+
}
|
|
1525
|
+
/**
|
|
1526
|
+
* Ids of non-deleted facts NOT live at `t` (spec §4.2). In 'current' mode a
|
|
1527
|
+
* row with both validity columns NULL is always live, so the query is
|
|
1528
|
+
* restricted to rows that carry a validity column — served by the partial
|
|
1529
|
+
* entries_temporal_idx.
|
|
1530
|
+
*/
|
|
1531
|
+
async findNonLiveIdsByEntityIds(entityIds, mode, t, tx) {
|
|
1532
|
+
if (entityIds.length === 0) return /* @__PURE__ */ new Set();
|
|
1533
|
+
const executor = this.getExecutor(tx);
|
|
1534
|
+
const placeholders = entityIds.map(() => "?").join(",");
|
|
1535
|
+
const temporalOnly = mode === "current" ? " AND (valid_from IS NOT NULL OR valid_to IS NOT NULL)" : "";
|
|
1536
|
+
const rows = await executor.getAllAsync(
|
|
1537
|
+
`SELECT id FROM ${this.prefix}entries
|
|
1538
|
+
WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL${temporalOnly}
|
|
1539
|
+
AND NOT ${liveAtSql(mode)}`,
|
|
1540
|
+
[...entityIds, t, t]
|
|
1541
|
+
);
|
|
1542
|
+
return new Set(rows.map((r) => r.id));
|
|
1543
|
+
}
|
|
1544
|
+
async findSupersessionChainIds(entityId, factId, maxDepth, tx) {
|
|
1545
|
+
const executor = this.getExecutor(tx);
|
|
1546
|
+
const successors = await executor.getAllAsync(
|
|
1547
|
+
`WITH RECURSIVE fwd(id, depth) AS (
|
|
1548
|
+
SELECT superseded_by, 1 FROM ${this.prefix}entries WHERE id = ? AND entity_id = ? AND superseded_by IS NOT NULL
|
|
1549
|
+
UNION
|
|
1550
|
+
SELECT e.superseded_by, fwd.depth + 1 FROM fwd
|
|
1551
|
+
JOIN ${this.prefix}entries e ON e.id = fwd.id AND e.entity_id = ?
|
|
1552
|
+
WHERE e.superseded_by IS NOT NULL AND fwd.depth < ?
|
|
1553
|
+
)
|
|
1554
|
+
SELECT id FROM fwd ORDER BY depth ASC`,
|
|
1555
|
+
[factId, entityId, entityId, maxDepth]
|
|
1556
|
+
);
|
|
1557
|
+
const predecessors = await executor.getAllAsync(
|
|
1558
|
+
`WITH RECURSIVE bwd(id, depth) AS (
|
|
1559
|
+
SELECT id, 1 FROM ${this.prefix}entries WHERE entity_id = ? AND superseded_by = ?
|
|
1560
|
+
UNION
|
|
1561
|
+
SELECT e.id, bwd.depth + 1 FROM bwd
|
|
1562
|
+
JOIN ${this.prefix}entries e ON e.entity_id = ? AND e.superseded_by = bwd.id
|
|
1563
|
+
WHERE bwd.depth < ?
|
|
1564
|
+
)
|
|
1565
|
+
SELECT id FROM bwd ORDER BY depth DESC`,
|
|
1566
|
+
[entityId, factId, entityId, maxDepth]
|
|
1567
|
+
);
|
|
1568
|
+
const drop = (ids) => ids.map((r) => r.id).filter((id) => id !== factId);
|
|
1569
|
+
return { predecessors: drop(predecessors), successors: drop(successors) };
|
|
1570
|
+
}
|
|
1571
|
+
async findExpiredIds(entityId, cutoff, tx) {
|
|
1572
|
+
const executor = this.getExecutor(tx);
|
|
1573
|
+
const rows = await executor.getAllAsync(
|
|
1574
|
+
`SELECT id FROM ${this.prefix}entries
|
|
1575
|
+
WHERE entity_id = ? AND deleted_at IS NULL AND valid_to IS NOT NULL AND valid_to <= ?
|
|
1576
|
+
ORDER BY valid_to ASC`,
|
|
1577
|
+
[entityId, cutoff]
|
|
1578
|
+
);
|
|
1579
|
+
return rows.map((r) => r.id);
|
|
1580
|
+
}
|
|
1401
1581
|
/**
|
|
1402
1582
|
* Bulk soft-delete all entries for an entity.
|
|
1403
1583
|
* Stages DELETE outbox entries for each row in the same transaction.
|
|
@@ -1830,9 +2010,11 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1830
2010
|
const executor = this.getExecutor(tx);
|
|
1831
2011
|
const placeholders = entityIds.map(() => "?").join(",");
|
|
1832
2012
|
const draftClause = opts?.excludeDrafts === true ? ` AND lifecycle_status != 'draft'` : "";
|
|
2013
|
+
const liveClause = opts?.live ? ` AND ${liveAtSql(opts.live.mode)}` : "";
|
|
2014
|
+
const liveArgs = opts?.live ? [opts.live.t, opts.live.t] : [];
|
|
1833
2015
|
const rows = await executor.getAllAsync(
|
|
1834
|
-
`SELECT * FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL${draftClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
1835
|
-
[...entityIds, limit]
|
|
2016
|
+
`SELECT * FROM ${this.prefix}entries WHERE entity_id IN (${placeholders}) AND deleted_at IS NULL${draftClause}${liveClause} ORDER BY updated_at DESC LIMIT ?`,
|
|
2017
|
+
[...entityIds, ...liveArgs, limit]
|
|
1836
2018
|
);
|
|
1837
2019
|
return rows.map(mapRowToFact);
|
|
1838
2020
|
}
|
|
@@ -1947,13 +2129,15 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1947
2129
|
/** Counts live mutable facts: eligible (past cooldown) vs deferred (in cooldown). */
|
|
1948
2130
|
async countHealCandidatesByEntityId(entityId, recheckCutoff, tx) {
|
|
1949
2131
|
const executor = this.getExecutor(tx);
|
|
2132
|
+
const now = Date.now();
|
|
1950
2133
|
const row = await executor.getFirstAsync(
|
|
1951
2134
|
`SELECT
|
|
1952
2135
|
SUM(CASE WHEN heal_checked_at IS NULL OR heal_checked_at <= ? THEN 1 ELSE 0 END) AS eligible,
|
|
1953
2136
|
SUM(CASE WHEN heal_checked_at IS NOT NULL AND heal_checked_at > ? THEN 1 ELSE 0 END) AS deferred
|
|
1954
2137
|
FROM ${this.prefix}entries
|
|
1955
|
-
WHERE entity_id = ? AND deleted_at IS NULL AND source_type != 'immutable_document'
|
|
1956
|
-
|
|
2138
|
+
WHERE entity_id = ? AND deleted_at IS NULL AND source_type != 'immutable_document'
|
|
2139
|
+
AND ${liveAtSql("current")}`,
|
|
2140
|
+
[recheckCutoff, recheckCutoff, entityId, now, now]
|
|
1957
2141
|
);
|
|
1958
2142
|
return { eligible: Number(row?.eligible ?? 0), deferred: Number(row?.deferred ?? 0) };
|
|
1959
2143
|
}
|
|
@@ -1996,6 +2180,21 @@ var _EntryRepository = class _EntryRepository extends BaseRepository {
|
|
|
1996
2180
|
);
|
|
1997
2181
|
return rows.map((r) => ({ id: r.id, title: r.title }));
|
|
1998
2182
|
}
|
|
2183
|
+
/**
|
|
2184
|
+
* Count of live facts for `entityId` with no stored embedding that are still
|
|
2185
|
+
* retryable (`embedding_attempts < maxAttempts`). Backs the deferred
|
|
2186
|
+
* maintenance pending report's `reembedPending` flag and the reembed job's
|
|
2187
|
+
* entity selection (PR-C 2026-09-29).
|
|
2188
|
+
*/
|
|
2189
|
+
async countReembedPending(entityId, maxAttempts, tx) {
|
|
2190
|
+
const executor = this.getExecutor(tx);
|
|
2191
|
+
const row = await executor.getFirstAsync(
|
|
2192
|
+
`SELECT COUNT(*) AS n FROM ${this.prefix}entries
|
|
2193
|
+
WHERE entity_id = ? AND deleted_at IS NULL AND embedding_blob IS NULL AND embedding IS NULL AND embedding_attempts < ?`,
|
|
2194
|
+
[entityId, maxAttempts]
|
|
2195
|
+
);
|
|
2196
|
+
return Number(row?.n ?? 0);
|
|
2197
|
+
}
|
|
1999
2198
|
};
|
|
2000
2199
|
/**
|
|
2001
2200
|
* Column list and liveness predicate shared by findMiniSearchRows and
|
|
@@ -2468,6 +2667,18 @@ var TaskRepository = class extends BaseRepository {
|
|
|
2468
2667
|
};
|
|
2469
2668
|
|
|
2470
2669
|
// src/repositories/EventRepository.ts
|
|
2670
|
+
var AFTER_SQL = "((created_at > ?) OR (created_at = ? AND id > ?))";
|
|
2671
|
+
function afterArgs(wm) {
|
|
2672
|
+
return [wm.at, wm.at, wm.id];
|
|
2673
|
+
}
|
|
2674
|
+
function mapEvent(row) {
|
|
2675
|
+
if (row.occurred_at == null) {
|
|
2676
|
+
const rest = { ...row };
|
|
2677
|
+
delete rest.occurred_at;
|
|
2678
|
+
return rest;
|
|
2679
|
+
}
|
|
2680
|
+
return row;
|
|
2681
|
+
}
|
|
2471
2682
|
var EventRepository = class extends BaseRepository {
|
|
2472
2683
|
/**
|
|
2473
2684
|
* Insert a new event row.
|
|
@@ -2476,30 +2687,32 @@ var EventRepository = class extends BaseRepository {
|
|
|
2476
2687
|
async add(event, tx) {
|
|
2477
2688
|
const executor = this.getExecutor(tx);
|
|
2478
2689
|
await executor.runAsync(
|
|
2479
|
-
`INSERT INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
|
|
2480
|
-
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
2690
|
+
`INSERT INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at, occurred_at)
|
|
2691
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
2481
2692
|
[
|
|
2482
2693
|
event.id,
|
|
2483
2694
|
event.entity_id,
|
|
2484
2695
|
event.event_type,
|
|
2485
2696
|
event.summary,
|
|
2486
2697
|
event.related_entry_id ?? null,
|
|
2487
|
-
event.created_at
|
|
2698
|
+
event.created_at,
|
|
2699
|
+
event.occurred_at ?? null
|
|
2488
2700
|
]
|
|
2489
2701
|
);
|
|
2490
2702
|
}
|
|
2491
2703
|
async addIgnoreDuplicate(event, tx) {
|
|
2492
2704
|
const executor = this.getExecutor(tx);
|
|
2493
2705
|
await executor.runAsync(
|
|
2494
|
-
`INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at)
|
|
2495
|
-
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
2706
|
+
`INSERT OR IGNORE INTO ${this.prefix}events (id, entity_id, event_type, summary, related_entry_id, created_at, occurred_at)
|
|
2707
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`,
|
|
2496
2708
|
[
|
|
2497
2709
|
event.id,
|
|
2498
2710
|
event.entity_id,
|
|
2499
2711
|
event.event_type,
|
|
2500
2712
|
event.summary,
|
|
2501
2713
|
event.related_entry_id ?? null,
|
|
2502
|
-
event.created_at
|
|
2714
|
+
event.created_at,
|
|
2715
|
+
event.occurred_at ?? null
|
|
2503
2716
|
]
|
|
2504
2717
|
);
|
|
2505
2718
|
}
|
|
@@ -2508,10 +2721,11 @@ var EventRepository = class extends BaseRepository {
|
|
|
2508
2721
|
* Defaults to a limit of 50.
|
|
2509
2722
|
*/
|
|
2510
2723
|
async getRecent(entityId, limit = 50) {
|
|
2511
|
-
|
|
2724
|
+
const rows = await this.db.getAllAsync(
|
|
2512
2725
|
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?`,
|
|
2513
2726
|
[entityId, limit]
|
|
2514
2727
|
);
|
|
2728
|
+
return rows.map(mapEvent);
|
|
2515
2729
|
}
|
|
2516
2730
|
/**
|
|
2517
2731
|
* Return the most recent events for the given entity IDs, newest first.
|
|
@@ -2520,10 +2734,11 @@ var EventRepository = class extends BaseRepository {
|
|
|
2520
2734
|
async getRecentForEntities(entityIds, limit = 50) {
|
|
2521
2735
|
if (entityIds.length === 0) return [];
|
|
2522
2736
|
const placeholders = entityIds.map(() => "?").join(", ");
|
|
2523
|
-
|
|
2737
|
+
const rows = await this.db.getAllAsync(
|
|
2524
2738
|
`SELECT * FROM ${this.prefix}events WHERE entity_id IN (${placeholders}) ORDER BY created_at DESC LIMIT ?`,
|
|
2525
2739
|
[...entityIds, limit]
|
|
2526
2740
|
);
|
|
2741
|
+
return rows.map(mapEvent);
|
|
2527
2742
|
}
|
|
2528
2743
|
/**
|
|
2529
2744
|
* Delete events for an entity that were created at or before the given cutoff timestamp.
|
|
@@ -2535,6 +2750,33 @@ var EventRepository = class extends BaseRepository {
|
|
|
2535
2750
|
[entityId, cutoff]
|
|
2536
2751
|
);
|
|
2537
2752
|
}
|
|
2753
|
+
/**
|
|
2754
|
+
* Watermark-guarded prune (spec 2026-09-29 competitive-memory §6): delete
|
|
2755
|
+
* events at or before `cutoff` **and** at-or-before the librarian watermark.
|
|
2756
|
+
* Events the librarian has not processed yet are retention-held, never
|
|
2757
|
+
* deleted. `wm = null` (no watermark yet) deletes nothing.
|
|
2758
|
+
* Used by runPrune under the deferred/ops strategies.
|
|
2759
|
+
*/
|
|
2760
|
+
async pruneThrough(entityId, cutoff, wm) {
|
|
2761
|
+
if (!wm) return { changes: 0 };
|
|
2762
|
+
return this.db.runAsync(
|
|
2763
|
+
`DELETE FROM ${this.prefix}events WHERE entity_id = ? AND created_at <= ? AND NOT ${AFTER_SQL}`,
|
|
2764
|
+
[entityId, cutoff, ...afterArgs(wm)]
|
|
2765
|
+
);
|
|
2766
|
+
}
|
|
2767
|
+
/**
|
|
2768
|
+
* Count events at or before `cutoff` that are after the watermark — the
|
|
2769
|
+
* retention-held tail `pruneThrough` left in place. `wm = null` counts all
|
|
2770
|
+
* events at or before `cutoff` (every one of them is held back).
|
|
2771
|
+
*/
|
|
2772
|
+
async countHeldBack(entityId, cutoff, wm) {
|
|
2773
|
+
const where = wm ? ` AND ${AFTER_SQL}` : "";
|
|
2774
|
+
const row = await this.db.getFirstAsync(
|
|
2775
|
+
`SELECT COUNT(*) AS n FROM ${this.prefix}events WHERE entity_id = ? AND created_at <= ?${where}`,
|
|
2776
|
+
[entityId, cutoff, ...wm ? afterArgs(wm) : []]
|
|
2777
|
+
);
|
|
2778
|
+
return Number(row?.n ?? 0);
|
|
2779
|
+
}
|
|
2538
2780
|
/**
|
|
2539
2781
|
* Return the total number of events stored for an entity.
|
|
2540
2782
|
* `tx` is optional — pass an active transaction handle for atomic reads.
|
|
@@ -2553,16 +2795,82 @@ var EventRepository = class extends BaseRepository {
|
|
|
2553
2795
|
*/
|
|
2554
2796
|
async getByEntityId(entityId, limit) {
|
|
2555
2797
|
if (limit != null) {
|
|
2556
|
-
const
|
|
2798
|
+
const rows2 = await this.db.getAllAsync(
|
|
2557
2799
|
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at DESC LIMIT ?`,
|
|
2558
2800
|
[entityId, limit]
|
|
2559
2801
|
);
|
|
2560
|
-
return
|
|
2802
|
+
return rows2.map(mapEvent).slice().reverse();
|
|
2561
2803
|
}
|
|
2562
|
-
|
|
2804
|
+
const rows = await this.db.getAllAsync(
|
|
2563
2805
|
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC`,
|
|
2564
2806
|
[entityId]
|
|
2565
2807
|
);
|
|
2808
|
+
return rows.map(mapEvent);
|
|
2809
|
+
}
|
|
2810
|
+
/**
|
|
2811
|
+
* Return events for `entityId` strictly after the watermark `(at, id)` in
|
|
2812
|
+
* chronological (ASC, id-ASC tie-break) order. Pass `wm = null` to read
|
|
2813
|
+
* from the beginning of the entity's event log. `limit` caps the result.
|
|
2814
|
+
*
|
|
2815
|
+
* Used by the op-based librarian (PR-B) to enumerate its unprocessed tail.
|
|
2816
|
+
*/
|
|
2817
|
+
async getAfter(entityId, wm, limit) {
|
|
2818
|
+
const limitClause = limit != null ? ` LIMIT ?` : "";
|
|
2819
|
+
const params = wm ? [entityId, wm.at, wm.at, wm.id] : [entityId];
|
|
2820
|
+
const where = wm ? `entity_id = ? AND ${AFTER_SQL}` : `entity_id = ?`;
|
|
2821
|
+
if (limit != null) params.push(limit);
|
|
2822
|
+
const rows = await this.db.getAllAsync(
|
|
2823
|
+
`SELECT * FROM ${this.prefix}events WHERE ${where} ORDER BY created_at ASC, id ASC${limitClause}`,
|
|
2824
|
+
params
|
|
2825
|
+
);
|
|
2826
|
+
return rows.map(mapEvent);
|
|
2827
|
+
}
|
|
2828
|
+
/**
|
|
2829
|
+
* Count of events for `entityId` strictly after the watermark. Used as
|
|
2830
|
+
* the op-based librarian's bounded batch size and by the deferred-maintenance
|
|
2831
|
+
* pending report. Pass `wm = null` to count the entity's whole event log
|
|
2832
|
+
* (no watermark yet) — mirrors `getAfter` / `sumSummaryCharsAfter`.
|
|
2833
|
+
*/
|
|
2834
|
+
async countAfter(entityId, wm) {
|
|
2835
|
+
const params = wm ? [entityId, wm.at, wm.at, wm.id] : [entityId];
|
|
2836
|
+
const where = wm ? `entity_id = ? AND ${AFTER_SQL}` : `entity_id = ?`;
|
|
2837
|
+
const row = await this.db.getFirstAsync(
|
|
2838
|
+
`SELECT COUNT(*) as count FROM ${this.prefix}events WHERE ${where}`,
|
|
2839
|
+
params
|
|
2840
|
+
);
|
|
2841
|
+
return row?.count ?? 0;
|
|
2842
|
+
}
|
|
2843
|
+
/**
|
|
2844
|
+
* Return the event at a given chronological offset for `entityId` (0-based,
|
|
2845
|
+
* oldest first). Used by MaintenanceService.seedLibrarianWatermark to anchor
|
|
2846
|
+
* the watermark on a real row when a legacy DB first migrates to the
|
|
2847
|
+
* op-based librarian. The offset is clamped and truncated to defend against
|
|
2848
|
+
* hostile or fractional callers.
|
|
2849
|
+
*/
|
|
2850
|
+
async getAtOffset(entityId, offset, tx) {
|
|
2851
|
+
const executor = this.getExecutor(tx);
|
|
2852
|
+
const row = await executor.getFirstAsync(
|
|
2853
|
+
`SELECT * FROM ${this.prefix}events WHERE entity_id = ? ORDER BY created_at ASC, id ASC LIMIT 1 OFFSET ?`,
|
|
2854
|
+
[entityId, Math.max(0, Math.trunc(offset))]
|
|
2855
|
+
);
|
|
2856
|
+
return row ? mapEvent(row) : null;
|
|
2857
|
+
}
|
|
2858
|
+
/**
|
|
2859
|
+
* Sum of `summary` lengths (in characters) for events for `entityId`
|
|
2860
|
+
* strictly after the watermark. Pass `wm = null` to sum the entity's whole
|
|
2861
|
+
* event log (no watermark yet). Used by the op-based librarian's token
|
|
2862
|
+
* budget estimator (PR-B) and by the auto-trigger's pending-size check
|
|
2863
|
+
* (PR-C). SQLite's `LENGTH` returns character count for UTF-8 text input.
|
|
2864
|
+
*/
|
|
2865
|
+
async sumSummaryCharsAfter(entityId, wm, tx) {
|
|
2866
|
+
const executor = this.getExecutor(tx);
|
|
2867
|
+
const params = wm ? [entityId, wm.at, wm.at, wm.id] : [entityId];
|
|
2868
|
+
const where = wm ? `entity_id = ? AND ${AFTER_SQL}` : `entity_id = ?`;
|
|
2869
|
+
const row = await executor.getFirstAsync(
|
|
2870
|
+
`SELECT SUM(LENGTH(summary)) as total FROM ${this.prefix}events WHERE ${where}`,
|
|
2871
|
+
params
|
|
2872
|
+
);
|
|
2873
|
+
return row?.total ?? 0;
|
|
2566
2874
|
}
|
|
2567
2875
|
};
|
|
2568
2876
|
|
|
@@ -2691,6 +2999,7 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
2691
2999
|
) >= ?
|
|
2692
3000
|
AND n.source_type NOT IN (${excludeSourceTypesPlaceholders})
|
|
2693
3001
|
AND (? = 0 OR n.lifecycle_status != 'draft')
|
|
3002
|
+
AND ${liveAtSql(opts.live.mode, "n.")}
|
|
2694
3003
|
WHERE w.depth < ?
|
|
2695
3004
|
AND instr(w.visited, ',' || (CASE WHEN e.source_id = w.node_id THEN e.target_id ELSE e.source_id END) || ',') = 0
|
|
2696
3005
|
)
|
|
@@ -2711,6 +3020,8 @@ var EdgeRepository = class extends BaseRepository {
|
|
|
2711
3020
|
minConfidenceRank,
|
|
2712
3021
|
...opts.excludeSourceTypes,
|
|
2713
3022
|
opts.excludeDrafts === true ? 1 : 0,
|
|
3023
|
+
opts.live.t,
|
|
3024
|
+
opts.live.t,
|
|
2714
3025
|
opts.maxDepth,
|
|
2715
3026
|
opts.maxNodes
|
|
2716
3027
|
];
|
|
@@ -2960,6 +3271,37 @@ var MetadataRepository = class extends BaseRepository {
|
|
|
2960
3271
|
[entityId]
|
|
2961
3272
|
);
|
|
2962
3273
|
}
|
|
3274
|
+
/**
|
|
3275
|
+
* Read the op-based librarian's high-water mark. Returns `null` when the
|
|
3276
|
+
* entity has never had its watermark set.
|
|
3277
|
+
*/
|
|
3278
|
+
async getLibrarianWatermark(entityId, tx) {
|
|
3279
|
+
const executor = this.getExecutor(tx);
|
|
3280
|
+
const row = await executor.getFirstAsync(
|
|
3281
|
+
`SELECT librarian_watermark_at, librarian_watermark_id FROM ${this.prefix}checkpoints WHERE entity_id = ?`,
|
|
3282
|
+
[entityId]
|
|
3283
|
+
);
|
|
3284
|
+
if (!row || row.librarian_watermark_at == null || row.librarian_watermark_id == null) {
|
|
3285
|
+
return null;
|
|
3286
|
+
}
|
|
3287
|
+
return { at: row.librarian_watermark_at, id: row.librarian_watermark_id };
|
|
3288
|
+
}
|
|
3289
|
+
/**
|
|
3290
|
+
* Write the op-based librarian's high-water mark. Uses `ON CONFLICT`
|
|
3291
|
+
* `DO UPDATE` so existing `memory_checkpoint` / `heal_checkpoint` rows
|
|
3292
|
+
* are preserved verbatim.
|
|
3293
|
+
*/
|
|
3294
|
+
async setLibrarianWatermark(entityId, wm, tx) {
|
|
3295
|
+
const executor = this.getExecutor(tx);
|
|
3296
|
+
await executor.runAsync(
|
|
3297
|
+
`INSERT INTO ${this.prefix}checkpoints (entity_id, memory_checkpoint, heal_checkpoint, librarian_watermark_at, librarian_watermark_id)
|
|
3298
|
+
VALUES (?, 0, 0, ?, ?)
|
|
3299
|
+
ON CONFLICT(entity_id) DO UPDATE SET
|
|
3300
|
+
librarian_watermark_at = excluded.librarian_watermark_at,
|
|
3301
|
+
librarian_watermark_id = excluded.librarian_watermark_id`,
|
|
3302
|
+
[entityId, wm.at, wm.id]
|
|
3303
|
+
);
|
|
3304
|
+
}
|
|
2963
3305
|
// META TABLE METHODS
|
|
2964
3306
|
async getMeta(key, tx) {
|
|
2965
3307
|
const executor = this.getExecutor(tx);
|
|
@@ -4482,7 +4824,14 @@ var SEVERITY = {
|
|
|
4482
4824
|
grounding_missing: "warn",
|
|
4483
4825
|
grounding_failed: "warn",
|
|
4484
4826
|
classification_low_confidence: "info",
|
|
4485
|
-
classification_invalid: "warn"
|
|
4827
|
+
classification_invalid: "warn",
|
|
4828
|
+
llm_usage: "info",
|
|
4829
|
+
librarian_gate: "info",
|
|
4830
|
+
librarian_op_rejected: "warn",
|
|
4831
|
+
contradicts_document: "warn",
|
|
4832
|
+
resolve_failed: "warn",
|
|
4833
|
+
read_budget: "info",
|
|
4834
|
+
event_retention_held: "info"
|
|
4486
4835
|
};
|
|
4487
4836
|
var MESSAGE = {
|
|
4488
4837
|
ingest_chunk_failed: "One or more document chunks failed to parse or generate.",
|
|
@@ -4497,7 +4846,14 @@ var MESSAGE = {
|
|
|
4497
4846
|
grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
|
|
4498
4847
|
grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
|
|
4499
4848
|
classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
|
|
4500
|
-
classification_invalid: "A classifier answer was invalid and was not applied."
|
|
4849
|
+
classification_invalid: "A classifier answer was invalid and was not applied.",
|
|
4850
|
+
llm_usage: "An LLM call completed; token usage attached.",
|
|
4851
|
+
librarian_gate: "Librarian gate outcome counts for one pass.",
|
|
4852
|
+
librarian_op_rejected: "A librarian reconciliation op was invalid and the candidate was added instead.",
|
|
4853
|
+
contradicts_document: "A candidate contradicted a document fact and was stored as a draft.",
|
|
4854
|
+
resolve_failed: "Reconciliation failed for a candidate; it was stored as a draft.",
|
|
4855
|
+
read_budget: "A read was packed to a token budget.",
|
|
4856
|
+
event_retention_held: "Event pruning was held back because the librarian has not processed those events yet."
|
|
4501
4857
|
};
|
|
4502
4858
|
var warnedNonFunction = /* @__PURE__ */ new WeakSet();
|
|
4503
4859
|
function copyDetail(detail) {
|
|
@@ -4624,6 +4980,21 @@ function groundingEvidenceBlock(writer, cfg) {
|
|
|
4624
4980
|
return `EVIDENCE REQUIREMENT: every object in "${key}" must also carry an "evidence" array of 1 to ${cfg.maxEvidence} quotes. Each quote must be an exact substring copied character-for-character from ${source} (the SOURCE section), at least ${cfg.minEvidenceChars} characters long. Do not paraphrase. Do not quote these instructions, the ontology manifest, or any existing fact. A fact whose quotes cannot be found in the SOURCE section is stored as an unreviewed draft.
|
|
4625
4981
|
"evidence": ["exact substring copied from the SOURCE section"]`;
|
|
4626
4982
|
}
|
|
4983
|
+
var OPS_EXTRACT_SYSTEM_PROMPT = `You are a memory extraction agent. Read the numbered events and extract durable facts about the user or entity, and actionable tasks.
|
|
4984
|
+
Return ONLY a valid JSON object matching this schema:
|
|
4985
|
+
{
|
|
4986
|
+
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative", "source_event": "the event label, e.g. e1", "valid_from": "optional ISO-8601 date the fact became true, only when the events state it" }],
|
|
4987
|
+
"tasks": [{ "description": "string", "priority": "number (0-10)" }]
|
|
4988
|
+
}
|
|
4989
|
+
One fact per claim. Every fact must name the single event it comes from in "source_event". Do not return markdown, just raw JSON.`;
|
|
4990
|
+
var OPS_RESOLVE_SYSTEM_PROMPT = `You are a memory reconciliation agent. Each item has a NEW candidate fact and EXISTING facts labelled n1, n2, ... that look similar to it. Choose exactly one op per item:
|
|
4991
|
+
- "ADD": the candidate is new information; the existing facts are about something else.
|
|
4992
|
+
- "UPDATE": one existing fact states the same thing and the candidate adds detail. Give the merged "title" and "body" for that existing fact.
|
|
4993
|
+
- "SUPERSEDE": the candidate replaces an existing fact that is no longer true (moved, changed jobs, changed preference, reversed decision).
|
|
4994
|
+
- "NOOP": an existing fact already states the candidate.
|
|
4995
|
+
Return ONLY a valid JSON object matching this schema:
|
|
4996
|
+
{ "ops": [{ "item": 0, "op": "ADD|UPDATE|SUPERSEDE|NOOP", "target": "n1", "title": "UPDATE only", "body": "UPDATE only" }] }
|
|
4997
|
+
"target" is required for UPDATE, SUPERSEDE and NOOP and must be one of that item's own labels. Do not return markdown, just raw JSON.`;
|
|
4627
4998
|
|
|
4628
4999
|
// src/utils/healConstants.ts
|
|
4629
5000
|
var HEAL_MAX_ANCHORS = 50;
|
|
@@ -4808,6 +5179,22 @@ ${JSON.stringify(facts, null, 2)}`
|
|
|
4808
5179
|
${JSON.stringify(facts, null, 2)}`
|
|
4809
5180
|
};
|
|
4810
5181
|
}
|
|
5182
|
+
/** Ops extract (spec §5.2): events only — no existing facts. */
|
|
5183
|
+
buildOpsExtractPrompt(events, ontologyContext) {
|
|
5184
|
+
const lines = events.map((l) => `${l.label} [${new Date(l.at).toISOString()}] (${l.event.event_type}) ${l.event.summary}`);
|
|
5185
|
+
const grounding = this.groundingFor("librarian");
|
|
5186
|
+
return {
|
|
5187
|
+
systemPrompt: this.appendGrounding(this.appendOntology(OPS_EXTRACT_SYSTEM_PROMPT, ontologyContext), "librarian"),
|
|
5188
|
+
userPrompt: `Events:
|
|
5189
|
+
${lines.join("\n")}`,
|
|
5190
|
+
...grounding ? { groundingCorpus: buildGroundingCorpus(events.map((l) => l.event.summary)) } : {}
|
|
5191
|
+
};
|
|
5192
|
+
}
|
|
5193
|
+
/** Ops resolve (spec §5.4): only each item's own neighbours. */
|
|
5194
|
+
buildOpsResolvePrompt(items) {
|
|
5195
|
+
return { systemPrompt: OPS_RESOLVE_SYSTEM_PROMPT, userPrompt: `Items:
|
|
5196
|
+
${JSON.stringify({ items })}` };
|
|
5197
|
+
}
|
|
4811
5198
|
/**
|
|
4812
5199
|
* The system prompt each writer would send, without hydrating data (spec
|
|
4813
5200
|
* §8.3). `buildSystemPrompt` with no variables hydrates only ontology
|
|
@@ -4872,6 +5259,79 @@ function toGroundingAnchor(anchor) {
|
|
|
4872
5259
|
var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
4873
5260
|
var DEFAULT_CHUNK_OVERLAP = 400;
|
|
4874
5261
|
|
|
5262
|
+
// src/utils/usage.ts
|
|
5263
|
+
function estimateTokens(text) {
|
|
5264
|
+
return Math.ceil(text.length / 4);
|
|
5265
|
+
}
|
|
5266
|
+
var UsageMeter = class {
|
|
5267
|
+
constructor(budget) {
|
|
5268
|
+
this.used = 0;
|
|
5269
|
+
this.estimated = false;
|
|
5270
|
+
if (budget !== void 0 && (typeof budget !== "number" || !Number.isFinite(budget) || budget < 0)) {
|
|
5271
|
+
throw new TypeError("tokenBudget must be a finite number >= 0");
|
|
5272
|
+
}
|
|
5273
|
+
this.budget = budget;
|
|
5274
|
+
}
|
|
5275
|
+
get remaining() {
|
|
5276
|
+
return this.budget === void 0 ? Infinity : Math.max(0, this.budget - this.used);
|
|
5277
|
+
}
|
|
5278
|
+
fits(estimate) {
|
|
5279
|
+
return estimate <= this.remaining;
|
|
5280
|
+
}
|
|
5281
|
+
record(r) {
|
|
5282
|
+
this.used += r.inputTokens + r.outputTokens;
|
|
5283
|
+
if (r.estimated) this.estimated = true;
|
|
5284
|
+
}
|
|
5285
|
+
};
|
|
5286
|
+
var WikiBudgetExhausted = class _WikiBudgetExhausted extends Error {
|
|
5287
|
+
constructor(requiredEstimate, remaining) {
|
|
5288
|
+
super(`LLM call needs ~${requiredEstimate} input tokens; ${remaining} remain in the budget.`);
|
|
5289
|
+
this.name = "WikiBudgetExhausted";
|
|
5290
|
+
this.requiredEstimate = requiredEstimate;
|
|
5291
|
+
this.remaining = remaining;
|
|
5292
|
+
Object.setPrototypeOf(this, _WikiBudgetExhausted.prototype);
|
|
5293
|
+
}
|
|
5294
|
+
};
|
|
5295
|
+
|
|
5296
|
+
// src/utils/llmCall.ts
|
|
5297
|
+
function validUsage(u) {
|
|
5298
|
+
if (!u || typeof u !== "object") return false;
|
|
5299
|
+
const { inputTokens, outputTokens } = u;
|
|
5300
|
+
return [inputTokens, outputTokens].every((n) => typeof n === "number" && Number.isFinite(n) && n >= 0);
|
|
5301
|
+
}
|
|
5302
|
+
async function callLlm(options, ctx, prompts) {
|
|
5303
|
+
const promptEstimate = estimateTokens(prompts.systemPrompt) + estimateTokens(prompts.userPrompt);
|
|
5304
|
+
if (ctx.meter && !ctx.meter.fits(promptEstimate)) {
|
|
5305
|
+
throw new WikiBudgetExhausted(promptEstimate, ctx.meter.remaining);
|
|
5306
|
+
}
|
|
5307
|
+
const provider = options.llmProvider;
|
|
5308
|
+
const params = { systemPrompt: prompts.systemPrompt, userPrompt: prompts.userPrompt };
|
|
5309
|
+
let text;
|
|
5310
|
+
let reported;
|
|
5311
|
+
if (typeof provider.generateTextWithUsage === "function") {
|
|
5312
|
+
const result = await provider.generateTextWithUsage(params);
|
|
5313
|
+
if (!result || typeof result.text !== "string") {
|
|
5314
|
+
throw new TypeError("generateTextWithUsage() must resolve to { text: string, usage? }");
|
|
5315
|
+
}
|
|
5316
|
+
text = result.text;
|
|
5317
|
+
reported = validUsage(result.usage) ? result.usage : void 0;
|
|
5318
|
+
} else {
|
|
5319
|
+
text = await provider.generateText(params);
|
|
5320
|
+
}
|
|
5321
|
+
const record = reported ? { inputTokens: reported.inputTokens, outputTokens: reported.outputTokens, estimated: false } : { inputTokens: promptEstimate, outputTokens: estimateTokens(text), estimated: true };
|
|
5322
|
+
ctx.meter?.record(record);
|
|
5323
|
+
if (options.config?.reportLlmUsage === true) {
|
|
5324
|
+
emitDiagnostic(options, {
|
|
5325
|
+
code: "llm_usage",
|
|
5326
|
+
operation: ctx.operation,
|
|
5327
|
+
trigger: ctx.trigger,
|
|
5328
|
+
entityId: ctx.entityId,
|
|
5329
|
+
detail: { inputTokens: record.inputTokens, outputTokens: record.outputTokens, estimated: record.estimated }
|
|
5330
|
+
});
|
|
5331
|
+
}
|
|
5332
|
+
return text;
|
|
5333
|
+
}
|
|
5334
|
+
|
|
4875
5335
|
// src/services/IngestionService.ts
|
|
4876
5336
|
var LLM_PASS_DIAGNOSTIC_CODES = ["ingest_chunk_failed", "fact_rejected", "fact_deduplicated"];
|
|
4877
5337
|
function dedupDiagnostic(entityId, sourceRef, locator) {
|
|
@@ -4948,7 +5408,7 @@ var IngestionService = class {
|
|
|
4948
5408
|
ontologyContext
|
|
4949
5409
|
);
|
|
4950
5410
|
try {
|
|
4951
|
-
const responseText = await this.options
|
|
5411
|
+
const responseText = await callLlm(this.options, { operation: "ingest", entityId, trigger: "call" }, { systemPrompt, userPrompt });
|
|
4952
5412
|
const result2 = parseJsonResponse(responseText);
|
|
4953
5413
|
const rawFacts = Array.isArray(result2.facts) ? result2.facts : [];
|
|
4954
5414
|
const facts = [];
|
|
@@ -5604,6 +6064,8 @@ async function runBatched(args) {
|
|
|
5604
6064
|
const results = [];
|
|
5605
6065
|
const skipped = [];
|
|
5606
6066
|
let batches = 0;
|
|
6067
|
+
const settled = /* @__PURE__ */ new Set();
|
|
6068
|
+
let budgetStop;
|
|
5607
6069
|
let batchSize = initialBatchSize(maxOutputTokens);
|
|
5608
6070
|
const trim = async (candidate) => {
|
|
5609
6071
|
const whole = await buildPrompt(candidate);
|
|
@@ -5637,6 +6099,7 @@ async function runBatched(args) {
|
|
|
5637
6099
|
} else {
|
|
5638
6100
|
const reason = fromCall && !isTruncationError(err) && !isConfigError(err) ? "call_error" : "non_convergent";
|
|
5639
6101
|
skipped.push({ item: batch[0], reason });
|
|
6102
|
+
settled.add(batch[0]);
|
|
5640
6103
|
onSkip?.(batch[0], err);
|
|
5641
6104
|
}
|
|
5642
6105
|
return;
|
|
@@ -5645,7 +6108,7 @@ async function runBatched(args) {
|
|
|
5645
6108
|
const mid = Math.ceil(batch.length / 2);
|
|
5646
6109
|
if (mid < batchSize) batchSize = mid;
|
|
5647
6110
|
let i = 0;
|
|
5648
|
-
while (i < batch.length) {
|
|
6111
|
+
while (i < batch.length && !budgetStop) {
|
|
5649
6112
|
const size = Math.min(batchSize, batch.length - i);
|
|
5650
6113
|
const trimmed = await trim(batch.slice(i, i + size));
|
|
5651
6114
|
await attempt(trimmed.batch, trimmed.prompts);
|
|
@@ -5653,6 +6116,7 @@ async function runBatched(args) {
|
|
|
5653
6116
|
}
|
|
5654
6117
|
};
|
|
5655
6118
|
const attempt = async (batch, prebuilt, attemptLevel = 0) => {
|
|
6119
|
+
if (budgetStop) return;
|
|
5656
6120
|
if (batch.length === 0) return;
|
|
5657
6121
|
const prompts = prebuilt && attemptLevel === 0 ? prebuilt : await buildPrompt(batch, attemptLevel);
|
|
5658
6122
|
batches++;
|
|
@@ -5660,6 +6124,10 @@ async function runBatched(args) {
|
|
|
5660
6124
|
try {
|
|
5661
6125
|
responseText = await call(prompts);
|
|
5662
6126
|
} catch (err) {
|
|
6127
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
6128
|
+
budgetStop = { requiredEstimate: err.requiredEstimate };
|
|
6129
|
+
return;
|
|
6130
|
+
}
|
|
5663
6131
|
await onFailure(batch, err, attemptLevel, true);
|
|
5664
6132
|
return;
|
|
5665
6133
|
}
|
|
@@ -5671,95 +6139,852 @@ async function runBatched(args) {
|
|
|
5671
6139
|
return;
|
|
5672
6140
|
}
|
|
5673
6141
|
results.push(result);
|
|
6142
|
+
for (const item of batch) settled.add(item);
|
|
5674
6143
|
};
|
|
5675
6144
|
let index = 0;
|
|
5676
|
-
while (index < items.length) {
|
|
6145
|
+
while (index < items.length && !budgetStop) {
|
|
5677
6146
|
const { batch, prompts } = await trim(items.slice(index, index + batchSize));
|
|
5678
6147
|
index += batch.length;
|
|
5679
6148
|
await attempt(batch, prompts, 0);
|
|
5680
6149
|
}
|
|
5681
|
-
|
|
6150
|
+
const unattempted = budgetStop ? items.filter((item) => !settled.has(item)) : [];
|
|
6151
|
+
return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
|
|
5682
6152
|
}
|
|
5683
6153
|
|
|
5684
|
-
// src/services/
|
|
6154
|
+
// src/services/librarian/constants.ts
|
|
5685
6155
|
var FUZZY_THRESHOLD = 0.5;
|
|
5686
6156
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
5687
|
-
|
|
5688
|
-
|
|
5689
|
-
var
|
|
5690
|
-
|
|
5691
|
-
|
|
5692
|
-
|
|
5693
|
-
|
|
5694
|
-
|
|
5695
|
-
|
|
5696
|
-
|
|
5697
|
-
|
|
5698
|
-
|
|
5699
|
-
|
|
5700
|
-
|
|
5701
|
-
|
|
5702
|
-
|
|
5703
|
-
|
|
5704
|
-
const
|
|
5705
|
-
|
|
5706
|
-
|
|
5707
|
-
|
|
5708
|
-
|
|
5709
|
-
|
|
5710
|
-
|
|
5711
|
-
var SKIP_ERROR_LOG_CHARS = 4096;
|
|
5712
|
-
var formatSkipError = (err) => {
|
|
5713
|
-
let base;
|
|
5714
|
-
let isErrorLike = false;
|
|
6157
|
+
|
|
6158
|
+
// src/services/librarian/legacy.ts
|
|
6159
|
+
var runLegacyLibrarianPass = async (deps, ctx) => {
|
|
6160
|
+
const { entityId, trigger } = ctx;
|
|
6161
|
+
const promptOverride = ctx.promptOverride;
|
|
6162
|
+
const events = await deps.eventRepo.getRecent(entityId, 50);
|
|
6163
|
+
const currentFactsRows = await deps.entryRepo.findRecentByEntityId(entityId, 100);
|
|
6164
|
+
const currentFacts = currentFactsRows.map((f) => {
|
|
6165
|
+
const { embedding: _embedding, embedding_blob: _blob, ...rest } = f;
|
|
6166
|
+
return {
|
|
6167
|
+
...rest,
|
|
6168
|
+
tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags
|
|
6169
|
+
};
|
|
6170
|
+
});
|
|
6171
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
6172
|
+
const promptEvents = events.reverse();
|
|
6173
|
+
const librarianGrounding = deps.promptService.groundingFor("librarian");
|
|
6174
|
+
const { systemPrompt, userPrompt, groundingCorpus: librarianCorpus = [] } = deps.promptService.buildLibrarianPrompt(
|
|
6175
|
+
promptEvents,
|
|
6176
|
+
currentFacts,
|
|
6177
|
+
promptOverride,
|
|
6178
|
+
ontologyContext
|
|
6179
|
+
);
|
|
6180
|
+
let responseText;
|
|
5715
6181
|
try {
|
|
5716
|
-
|
|
5717
|
-
|
|
5718
|
-
|
|
5719
|
-
|
|
5720
|
-
|
|
5721
|
-
}
|
|
5722
|
-
|
|
5723
|
-
|
|
5724
|
-
|
|
5725
|
-
|
|
5726
|
-
|
|
6182
|
+
responseText = await callLlm(
|
|
6183
|
+
deps.options,
|
|
6184
|
+
{ operation: "librarian", entityId, trigger, meter: ctx.meter },
|
|
6185
|
+
{ systemPrompt, userPrompt }
|
|
6186
|
+
);
|
|
6187
|
+
} catch (err) {
|
|
6188
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
6189
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
6190
|
+
}
|
|
6191
|
+
throw err;
|
|
6192
|
+
}
|
|
6193
|
+
const result = parseJsonResponse(responseText);
|
|
6194
|
+
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
6195
|
+
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
6196
|
+
const ontologyUpdates = result.ontology_updates;
|
|
6197
|
+
const diagBuffer = new DiagnosticBuffer();
|
|
6198
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
6199
|
+
const validFacts = [];
|
|
6200
|
+
const validFactItemIndexes = [];
|
|
6201
|
+
facts.forEach((raw, itemIndex) => {
|
|
6202
|
+
const valid = validateFact(raw);
|
|
6203
|
+
if (valid) {
|
|
6204
|
+
validFacts.push(valid);
|
|
6205
|
+
validFactItemIndexes.push(itemIndex);
|
|
6206
|
+
} else {
|
|
6207
|
+
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
5727
6208
|
}
|
|
5728
|
-
}
|
|
5729
|
-
|
|
5730
|
-
|
|
5731
|
-
|
|
5732
|
-
|
|
5733
|
-
|
|
5734
|
-
};
|
|
5735
|
-
|
|
5736
|
-
|
|
5737
|
-
|
|
5738
|
-
|
|
5739
|
-
|
|
5740
|
-
|
|
5741
|
-
|
|
5742
|
-
|
|
5743
|
-
|
|
5744
|
-
|
|
5745
|
-
|
|
5746
|
-
|
|
5747
|
-
|
|
5748
|
-
|
|
5749
|
-
|
|
5750
|
-
|
|
5751
|
-
|
|
5752
|
-
|
|
5753
|
-
|
|
5754
|
-
|
|
5755
|
-
|
|
5756
|
-
|
|
5757
|
-
|
|
5758
|
-
|
|
5759
|
-
|
|
5760
|
-
|
|
5761
|
-
|
|
6209
|
+
});
|
|
6210
|
+
const validTasks = [];
|
|
6211
|
+
tasks.forEach((raw, itemIndex) => {
|
|
6212
|
+
const valid = validateTask(raw);
|
|
6213
|
+
if (valid) validTasks.push(valid);
|
|
6214
|
+
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
6215
|
+
});
|
|
6216
|
+
const now = Date.now();
|
|
6217
|
+
const insertedFacts = [];
|
|
6218
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
6219
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
6220
|
+
if (mode === "emergent" && ontologyUpdates && deps.ontologyService) {
|
|
6221
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
6222
|
+
}
|
|
6223
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
6224
|
+
for (const existing of currentFactsRows) {
|
|
6225
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
6226
|
+
id: existing.id,
|
|
6227
|
+
okf_type: existing.okf_type ?? null
|
|
6228
|
+
});
|
|
6229
|
+
}
|
|
6230
|
+
const factsForDedupe = await deps.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
6231
|
+
const pendingEdges = [];
|
|
6232
|
+
for (const [k, fact] of validFacts.entries()) {
|
|
6233
|
+
const newTokens = titleTokens(fact.title);
|
|
6234
|
+
let skip = false;
|
|
6235
|
+
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
6236
|
+
for (const existing of factsForDedupe) {
|
|
6237
|
+
if (existing.source_type !== "librarian_inferred") continue;
|
|
6238
|
+
const existingTokens = titleTokens(existing.title);
|
|
6239
|
+
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
6240
|
+
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
6241
|
+
skip = true;
|
|
6242
|
+
break;
|
|
6243
|
+
}
|
|
6244
|
+
}
|
|
6245
|
+
}
|
|
6246
|
+
}
|
|
6247
|
+
if (skip) {
|
|
6248
|
+
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
6249
|
+
continue;
|
|
6250
|
+
}
|
|
6251
|
+
const ontologyFact = fact;
|
|
6252
|
+
const id = generateId("fact_");
|
|
6253
|
+
const validationDrops = [];
|
|
6254
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
6255
|
+
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
6256
|
+
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
6257
|
+
if (grounding?.diagnostic) {
|
|
6258
|
+
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
6259
|
+
}
|
|
6260
|
+
const factObj = {
|
|
6261
|
+
id,
|
|
6262
|
+
entity_id: entityId,
|
|
6263
|
+
title: fact.title,
|
|
6264
|
+
body: fact.body,
|
|
6265
|
+
tags: fact.tags,
|
|
6266
|
+
confidence: fact.confidence,
|
|
6267
|
+
source_type: "librarian_inferred",
|
|
6268
|
+
source_hash: null,
|
|
6269
|
+
source_ref: null,
|
|
6270
|
+
created_at: now,
|
|
6271
|
+
updated_at: now,
|
|
6272
|
+
last_accessed_at: null,
|
|
6273
|
+
access_count: 0,
|
|
6274
|
+
deleted_at: null,
|
|
6275
|
+
okf_type: normalized.okf_type,
|
|
6276
|
+
...grounding?.trust
|
|
6277
|
+
};
|
|
6278
|
+
await deps.entryRepo.upsert(factObj, tx);
|
|
6279
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
6280
|
+
factsForDedupe.push(factObj);
|
|
6281
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
6282
|
+
if (normalized.edges.length > 0) {
|
|
6283
|
+
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
6284
|
+
}
|
|
6285
|
+
}
|
|
6286
|
+
for (const item of pendingEdges) {
|
|
6287
|
+
const resolveDrops = [];
|
|
6288
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
6289
|
+
entityId,
|
|
6290
|
+
item.sourceId,
|
|
6291
|
+
item.sourceType,
|
|
6292
|
+
item.edges ?? [],
|
|
6293
|
+
manifest,
|
|
6294
|
+
titleIndex,
|
|
6295
|
+
tx,
|
|
6296
|
+
now,
|
|
6297
|
+
resolveDrops
|
|
6298
|
+
);
|
|
6299
|
+
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
6300
|
+
}
|
|
6301
|
+
for (const task of validTasks) {
|
|
6302
|
+
const id = generateId("task_");
|
|
6303
|
+
const taskObj = {
|
|
6304
|
+
id,
|
|
6305
|
+
entity_id: entityId,
|
|
6306
|
+
description: task.description,
|
|
6307
|
+
status: "pending",
|
|
6308
|
+
priority: task.priority,
|
|
6309
|
+
created_at: now,
|
|
6310
|
+
updated_at: now,
|
|
6311
|
+
resolved_at: null,
|
|
6312
|
+
deleted_at: null
|
|
6313
|
+
};
|
|
6314
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
6315
|
+
}
|
|
6316
|
+
});
|
|
6317
|
+
diagBuffer.flush(deps.options);
|
|
6318
|
+
await deps.searchService.syncEntries(entityId, insertedFacts.map((f) => f.id));
|
|
6319
|
+
for (const fact of insertedFacts) {
|
|
6320
|
+
await deps.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
6321
|
+
}
|
|
6322
|
+
deps.searchService.evictCache(entityId);
|
|
6323
|
+
const processedThrough = events.length > 0 ? { at: events[events.length - 1].created_at, id: events[events.length - 1].id } : null;
|
|
6324
|
+
const out = { processedThrough, factsWritten: insertedFacts.length };
|
|
6325
|
+
return out;
|
|
6326
|
+
};
|
|
6327
|
+
|
|
6328
|
+
// src/services/librarian/ops/text.ts
|
|
6329
|
+
var OPS_NEIGHBOUR_BODY_CHARS = 400;
|
|
6330
|
+
function normalizeFactText(title, body) {
|
|
6331
|
+
return `${title} ${body}`.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, " ").trim();
|
|
6332
|
+
}
|
|
6333
|
+
function labelEvents(events) {
|
|
6334
|
+
return events.map((event, i) => ({ label: `e${i + 1}`, event, at: event.occurred_at ?? event.created_at }));
|
|
6335
|
+
}
|
|
6336
|
+
function parseValidFrom(v) {
|
|
6337
|
+
if (typeof v === "number") return Number.isFinite(v) && v >= 0 ? Math.trunc(v) : void 0;
|
|
6338
|
+
if (typeof v === "string") {
|
|
6339
|
+
const t = Date.parse(v);
|
|
6340
|
+
return Number.isFinite(t) && t >= 0 ? t : void 0;
|
|
6341
|
+
}
|
|
6342
|
+
return void 0;
|
|
6343
|
+
}
|
|
6344
|
+
|
|
6345
|
+
// src/services/librarian/ops/extract.ts
|
|
6346
|
+
var OPS_MAX_EVENTS_PER_PASS = 50;
|
|
6347
|
+
var OPS_MAX_EXTRACT_CHARS = 24e3;
|
|
6348
|
+
var RESOLVE_RESERVE_FACTOR = 2;
|
|
6349
|
+
var estimatePrompt = (p) => estimateTokens(p.systemPrompt) + estimateTokens(p.userPrompt);
|
|
6350
|
+
async function selectBatch(deps, entityId, wm, meter) {
|
|
6351
|
+
const events = await deps.eventRepo.getAfter(entityId, wm, OPS_MAX_EVENTS_PER_PASS);
|
|
6352
|
+
if (events.length === 0) return { batch: [] };
|
|
6353
|
+
let n = 0;
|
|
6354
|
+
let chars = 0;
|
|
6355
|
+
for (const e of events) {
|
|
6356
|
+
if (n > 0 && chars + e.summary.length > OPS_MAX_EXTRACT_CHARS) break;
|
|
6357
|
+
chars += e.summary.length;
|
|
6358
|
+
n++;
|
|
6359
|
+
}
|
|
6360
|
+
let batch = labelEvents(events.slice(0, n));
|
|
6361
|
+
if (meter && meter.budget !== void 0) {
|
|
6362
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
6363
|
+
for (; ; ) {
|
|
6364
|
+
const need = RESOLVE_RESERVE_FACTOR * estimatePrompt(deps.promptService.buildOpsExtractPrompt(batch, ontologyContext));
|
|
6365
|
+
if (meter.fits(need)) break;
|
|
6366
|
+
if (batch.length === 1) return { budgetStop: { requiredEstimate: need } };
|
|
6367
|
+
batch = labelEvents(events.slice(0, Math.max(1, Math.floor(batch.length / 2))));
|
|
6368
|
+
}
|
|
6369
|
+
}
|
|
6370
|
+
return { batch };
|
|
6371
|
+
}
|
|
6372
|
+
async function runExtract(deps, ctx, batch) {
|
|
6373
|
+
const { entityId, trigger } = ctx;
|
|
6374
|
+
const ontologyContext = await deps.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
6375
|
+
const prompt = deps.promptService.buildOpsExtractPrompt(batch, ontologyContext);
|
|
6376
|
+
const text = await callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompt);
|
|
6377
|
+
const parsed = parseJsonResponse(text);
|
|
6378
|
+
const base = { entityId, operation: "librarian", trigger };
|
|
6379
|
+
const diagnostics = [];
|
|
6380
|
+
const byLabel = new Map(batch.map((l) => [l.label, l]));
|
|
6381
|
+
const newestAt = batch[batch.length - 1].at;
|
|
6382
|
+
const candidates = [];
|
|
6383
|
+
(Array.isArray(parsed.facts) ? parsed.facts : []).forEach((raw, index) => {
|
|
6384
|
+
const fact = validateFact(raw);
|
|
6385
|
+
if (!fact) {
|
|
6386
|
+
diagnostics.push({ ...base, code: "fact_rejected", detail: { itemIndex: index, reason: factRejectionReason(raw) } });
|
|
6387
|
+
return;
|
|
6388
|
+
}
|
|
6389
|
+
const r = raw;
|
|
6390
|
+
const sourceLabel = typeof r.source_event === "string" && byLabel.has(r.source_event) ? r.source_event : null;
|
|
6391
|
+
const validFrom = parseValidFrom(r.valid_from) ?? (sourceLabel ? byLabel.get(sourceLabel).at : newestAt);
|
|
6392
|
+
candidates.push({ index, fact, sourceLabel, validFrom });
|
|
6393
|
+
});
|
|
6394
|
+
const tasks = [];
|
|
6395
|
+
(Array.isArray(parsed.tasks) ? parsed.tasks : []).forEach((raw, index) => {
|
|
6396
|
+
const task = validateTask(raw);
|
|
6397
|
+
if (task) tasks.push(task);
|
|
6398
|
+
else diagnostics.push({ ...base, code: "task_rejected", detail: { itemIndex: index, reason: taskRejectionReason(raw) } });
|
|
6399
|
+
});
|
|
6400
|
+
return {
|
|
6401
|
+
batch,
|
|
6402
|
+
candidates,
|
|
6403
|
+
tasks,
|
|
6404
|
+
...parsed.ontology_updates ? { ontologyUpdates: parsed.ontology_updates } : {},
|
|
6405
|
+
groundingCorpus: prompt.groundingCorpus ?? null,
|
|
6406
|
+
diagnostics
|
|
6407
|
+
};
|
|
6408
|
+
}
|
|
6409
|
+
|
|
6410
|
+
// src/services/librarian/ops/gate.ts
|
|
6411
|
+
var clamp01 = (v, d) => typeof v === "number" && Number.isFinite(v) ? Math.min(1, Math.max(0, v)) : d;
|
|
6412
|
+
function resolveGateConfig(cfg) {
|
|
6413
|
+
const kRaw = cfg?.k;
|
|
6414
|
+
const k = typeof kRaw === "number" && Number.isFinite(kRaw) ? Math.min(20, Math.max(1, Math.trunc(kRaw))) : 5;
|
|
6415
|
+
const dupThreshold = clamp01(cfg?.dupThreshold, 0.89);
|
|
6416
|
+
const novelThreshold = clamp01(cfg?.novelThreshold, 0.3);
|
|
6417
|
+
if (novelThreshold > dupThreshold) throw new TypeError("librarian.gate: novelThreshold must be <= dupThreshold");
|
|
6418
|
+
return { k, dupThreshold, novelThreshold };
|
|
6419
|
+
}
|
|
6420
|
+
function titlesMatch(a, b) {
|
|
6421
|
+
const ta = titleTokens(a);
|
|
6422
|
+
const tb = titleTokens(b);
|
|
6423
|
+
if (ta.size < MIN_TOKENS_TO_QUALIFY || tb.size < MIN_TOKENS_TO_QUALIFY) return false;
|
|
6424
|
+
return jaccardScore(ta, tb) >= FUZZY_THRESHOLD;
|
|
6425
|
+
}
|
|
6426
|
+
function classifyCandidate(candidate, neighbours, cfg, mode) {
|
|
6427
|
+
const key = normalizeFactText(candidate.title, candidate.body);
|
|
6428
|
+
const same = neighbours.find((n) => normalizeFactText(n.title, n.body) === key);
|
|
6429
|
+
if (same) return { kind: "noop", target: same.id };
|
|
6430
|
+
if (neighbours.length === 0) return { kind: "add" };
|
|
6431
|
+
if (mode === "keyword") return { kind: "ambiguous" };
|
|
6432
|
+
const best = neighbours[0];
|
|
6433
|
+
const score = best.score ?? 0;
|
|
6434
|
+
if (score >= cfg.dupThreshold && titlesMatch(candidate.title, best.title)) return { kind: "noop", target: best.id };
|
|
6435
|
+
if (score < cfg.novelThreshold) return { kind: "add" };
|
|
6436
|
+
return { kind: "ambiguous" };
|
|
6437
|
+
}
|
|
6438
|
+
async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
6439
|
+
const { entityId, trigger } = ctx;
|
|
6440
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6441
|
+
const unique = [];
|
|
6442
|
+
const inBatchDuplicates = [];
|
|
6443
|
+
for (const c of candidates) {
|
|
6444
|
+
const key = normalizeFactText(c.fact.title, c.fact.body);
|
|
6445
|
+
if (seen.has(key)) inBatchDuplicates.push(c);
|
|
6446
|
+
else {
|
|
6447
|
+
seen.add(key);
|
|
6448
|
+
unique.push(c);
|
|
6449
|
+
}
|
|
6450
|
+
}
|
|
6451
|
+
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
6452
|
+
const canEmbed = typeof deps.options.llmProvider.embed === "function";
|
|
6453
|
+
const stored = [];
|
|
6454
|
+
if (canEmbed && unique.length > 0) {
|
|
6455
|
+
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
6456
|
+
if (nonLive.has(row.id)) continue;
|
|
6457
|
+
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
6458
|
+
if (vec) stored.push({ id: row.id, vec });
|
|
6459
|
+
}
|
|
6460
|
+
}
|
|
6461
|
+
const gated = [];
|
|
6462
|
+
for (const c of unique) {
|
|
6463
|
+
let ranked = null;
|
|
6464
|
+
let vector = null;
|
|
6465
|
+
let mode = "keyword";
|
|
6466
|
+
if (canEmbed) {
|
|
6467
|
+
const r = await deps.embeddingService.embedTextForFact(
|
|
6468
|
+
{ id: "candidate", entity_id: entityId, title: c.fact.title, body: c.fact.body, tags: c.fact.tags },
|
|
6469
|
+
void 0,
|
|
6470
|
+
{ markFailures: false }
|
|
6471
|
+
);
|
|
6472
|
+
if (r.ok) {
|
|
6473
|
+
vector = r.vector;
|
|
6474
|
+
if (stored.length > 0 && stored.every((s) => s.vec.length === r.vector.length)) {
|
|
6475
|
+
ranked = stored.map((s) => ({ id: s.id, score: cosineSimilarity(r.vector, s.vec) })).sort((a, b) => b.score - a.score || a.id.localeCompare(b.id)).slice(0, cfg.k);
|
|
6476
|
+
mode = "vector";
|
|
6477
|
+
}
|
|
6478
|
+
}
|
|
6479
|
+
}
|
|
6480
|
+
if (!ranked) {
|
|
6481
|
+
ranked = deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
|
|
6482
|
+
}
|
|
6483
|
+
const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
|
|
6484
|
+
const byId = new Map(facts.map((f) => [f.id, f]));
|
|
6485
|
+
const neighbours = ranked.filter((r) => byId.has(r.id)).map((r, i) => {
|
|
6486
|
+
const f = byId.get(r.id);
|
|
6487
|
+
return { id: f.id, ref: `n${i + 1}`, title: f.title, body: f.body, source_type: f.source_type, score: r.score };
|
|
6488
|
+
});
|
|
6489
|
+
gated.push({ candidate: c, neighbours, decision: classifyCandidate(c.fact, neighbours, cfg, mode), vector });
|
|
6490
|
+
}
|
|
6491
|
+
const count = (k) => gated.filter((g) => g.decision.kind === k).length;
|
|
6492
|
+
const gateDiagnostic = {
|
|
6493
|
+
code: "librarian_gate",
|
|
6494
|
+
operation: "librarian",
|
|
6495
|
+
trigger,
|
|
6496
|
+
entityId,
|
|
6497
|
+
detail: { gateNoop: count("noop"), gateAdd: count("add"), gateAmbiguous: count("ambiguous") }
|
|
6498
|
+
};
|
|
6499
|
+
return { gated, inBatchDuplicates, gateDiagnostic };
|
|
6500
|
+
}
|
|
6501
|
+
|
|
6502
|
+
// src/services/librarian/ops/resolve.ts
|
|
6503
|
+
var OPS_RESOLVE_MAX_PROMPT_CHARS = 4e4;
|
|
6504
|
+
function validateOp(raw, aliases) {
|
|
6505
|
+
if (!raw || typeof raw !== "object") return null;
|
|
6506
|
+
const r = raw;
|
|
6507
|
+
if (r.op === "ADD") return { op: "ADD" };
|
|
6508
|
+
if (r.op !== "UPDATE" && r.op !== "SUPERSEDE" && r.op !== "NOOP") return null;
|
|
6509
|
+
const targetId = typeof r.target === "string" ? aliases.get(r.target) : void 0;
|
|
6510
|
+
if (!targetId) return null;
|
|
6511
|
+
if (r.op === "NOOP") return { op: "NOOP", targetId };
|
|
6512
|
+
if (r.op === "SUPERSEDE") {
|
|
6513
|
+
const validFrom = parseValidFrom(r.valid_from);
|
|
6514
|
+
return validFrom === void 0 ? { op: "SUPERSEDE", targetId } : { op: "SUPERSEDE", targetId, validFrom };
|
|
6515
|
+
}
|
|
6516
|
+
const update = { op: "UPDATE", targetId };
|
|
6517
|
+
if (typeof r.title === "string" && r.title.trim() !== "") update.title = clip(r.title, 80);
|
|
6518
|
+
if (typeof r.body === "string" && r.body.trim() !== "") update.body = clip(r.body, 800);
|
|
6519
|
+
return update;
|
|
6520
|
+
}
|
|
6521
|
+
var aliasMapOf = (g) => new Map(g.neighbours.map((n) => [n.ref, n.id]));
|
|
6522
|
+
async function runResolve(deps, ctx, ambiguous) {
|
|
6523
|
+
const out = { ops: /* @__PURE__ */ new Map(), rejected: [], failed: [] };
|
|
6524
|
+
if (ambiguous.length === 0) return out;
|
|
6525
|
+
const { entityId, trigger } = ctx;
|
|
6526
|
+
const outcome = await runBatched({
|
|
6527
|
+
items: ambiguous,
|
|
6528
|
+
buildPrompt: (batch) => deps.promptService.buildOpsResolvePrompt(
|
|
6529
|
+
batch.map((g, i) => ({
|
|
6530
|
+
item: i,
|
|
6531
|
+
candidate: { title: g.candidate.fact.title, body: g.candidate.fact.body },
|
|
6532
|
+
existing: g.neighbours.map((n) => ({
|
|
6533
|
+
ref: n.ref,
|
|
6534
|
+
title: n.title,
|
|
6535
|
+
body: safeSlice(n.body, 0, OPS_NEIGHBOUR_BODY_CHARS)
|
|
6536
|
+
}))
|
|
6537
|
+
}))
|
|
6538
|
+
),
|
|
6539
|
+
call: (prompts) => callLlm(deps.options, { operation: "librarian", entityId, trigger, meter: ctx.meter }, prompts),
|
|
6540
|
+
parse: (text, batch) => {
|
|
6541
|
+
const parsed = parseJsonResponse(text);
|
|
6542
|
+
if (!Array.isArray(parsed.ops)) throw new Error("resolve: ops missing");
|
|
6543
|
+
return { batch, ops: parsed.ops };
|
|
6544
|
+
},
|
|
6545
|
+
maxPromptChars: OPS_RESOLVE_MAX_PROMPT_CHARS,
|
|
6546
|
+
maxOutputTokens: deps.options.llmProvider.maxOutputTokens
|
|
6547
|
+
});
|
|
6548
|
+
if (outcome.budgetStop) return { ...out, budgetStop: outcome.budgetStop };
|
|
6549
|
+
for (const { batch, ops } of outcome.results) {
|
|
6550
|
+
const byItem = /* @__PURE__ */ new Map();
|
|
6551
|
+
for (const raw of ops) {
|
|
6552
|
+
const item = raw?.item;
|
|
6553
|
+
if (typeof item === "number" && Number.isInteger(item) && item >= 0 && item < batch.length && !byItem.has(item)) {
|
|
6554
|
+
byItem.set(item, raw);
|
|
6555
|
+
}
|
|
6556
|
+
}
|
|
6557
|
+
batch.forEach((g, i) => {
|
|
6558
|
+
const op = validateOp(byItem.get(i), aliasMapOf(g));
|
|
6559
|
+
if (op) out.ops.set(g, op);
|
|
6560
|
+
else out.rejected.push(g);
|
|
6561
|
+
});
|
|
6562
|
+
}
|
|
6563
|
+
if (outcome.skipped.some((s) => s.reason === "call_error")) {
|
|
6564
|
+
throw new Error("resolve: transient provider error; retry the pass");
|
|
6565
|
+
}
|
|
6566
|
+
for (const s of outcome.skipped) out.failed.push(s.item);
|
|
6567
|
+
return out;
|
|
6568
|
+
}
|
|
6569
|
+
|
|
6570
|
+
// src/services/TemporalService.ts
|
|
6571
|
+
var HISTORY_MAX_DEPTH = 100;
|
|
6572
|
+
var TemporalService = class {
|
|
6573
|
+
constructor(db, options, entryRepo, searchService, embeddingService) {
|
|
6574
|
+
this.db = db;
|
|
6575
|
+
this.options = options;
|
|
6576
|
+
this.entryRepo = entryRepo;
|
|
6577
|
+
this.searchService = searchService;
|
|
6578
|
+
this.embeddingService = embeddingService;
|
|
6579
|
+
}
|
|
6580
|
+
/**
|
|
6581
|
+
* Link `oldId` → `newId` inside the caller's transaction. `newId` must
|
|
6582
|
+
* already exist. Throws WikiSupersedeError on any rule violation; the
|
|
6583
|
+
* caller's transaction then rolls back.
|
|
6584
|
+
*/
|
|
6585
|
+
async supersedeInTx(tx, entityId, oldId, newId, t, now) {
|
|
6586
|
+
const old = await this.entryRepo.findTemporalRow(oldId, tx);
|
|
6587
|
+
if (!old || old.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
6588
|
+
if (old.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
6589
|
+
if (old.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
6590
|
+
if (old.source_type === "immutable_document") throw new WikiSupersedeError("immutable_target");
|
|
6591
|
+
if (newId === oldId) throw new WikiSupersedeError("cycle");
|
|
6592
|
+
const repl = await this.entryRepo.findTemporalRow(newId, tx);
|
|
6593
|
+
if (!repl || repl.deleted_at != null) throw new WikiSupersedeError("not_found");
|
|
6594
|
+
if (repl.entity_id !== entityId) throw new WikiSupersedeError("cross_entity");
|
|
6595
|
+
if (repl.superseded_by != null) throw new WikiSupersedeError("already_superseded");
|
|
6596
|
+
const { predecessors } = await this.entryRepo.findSupersessionChainIds(entityId, oldId, HISTORY_MAX_DEPTH, tx);
|
|
6597
|
+
if (predecessors.includes(newId)) throw new WikiSupersedeError("cycle");
|
|
6598
|
+
await this.entryRepo.setTemporal(oldId, entityId, { valid_to: t, superseded_by: newId, superseded_at: now }, tx);
|
|
6599
|
+
if (repl.valid_from == null) {
|
|
6600
|
+
await this.entryRepo.setTemporal(newId, entityId, { valid_from: t }, tx);
|
|
6601
|
+
}
|
|
6602
|
+
}
|
|
6603
|
+
async supersede(entityId, oldId, replacement, options) {
|
|
6604
|
+
const now = Date.now();
|
|
6605
|
+
const t = options?.validFrom === void 0 ? now : assertEpochMs("options.validFrom", options.validFrom);
|
|
6606
|
+
let created = null;
|
|
6607
|
+
if (typeof replacement !== "string") {
|
|
6608
|
+
if (typeof replacement?.title !== "string" || replacement.title.trim() === "" || typeof replacement.body !== "string") {
|
|
6609
|
+
throw new TypeError("supersede(): replacement must be a fact id or { title, body }");
|
|
6610
|
+
}
|
|
6611
|
+
created = {
|
|
6612
|
+
id: generateId("fact_"),
|
|
6613
|
+
entity_id: entityId,
|
|
6614
|
+
title: replacement.title,
|
|
6615
|
+
body: replacement.body,
|
|
6616
|
+
tags: Array.isArray(replacement.tags) ? replacement.tags : [],
|
|
6617
|
+
confidence: replacement.confidence ?? "certain",
|
|
6618
|
+
source_type: replacement.source_type ?? "user_stated",
|
|
6619
|
+
source_hash: null,
|
|
6620
|
+
source_ref: null,
|
|
6621
|
+
created_at: now,
|
|
6622
|
+
updated_at: now,
|
|
6623
|
+
last_accessed_at: null,
|
|
6624
|
+
access_count: 0,
|
|
6625
|
+
deleted_at: null
|
|
6626
|
+
};
|
|
6627
|
+
}
|
|
6628
|
+
const newId = typeof replacement === "string" ? replacement : created.id;
|
|
6629
|
+
const newValidFrom = typeof replacement !== "string" && replacement.valid_from !== void 0 ? assertEpochMs("replacement.valid_from", replacement.valid_from) : void 0;
|
|
6630
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
6631
|
+
if (created) {
|
|
6632
|
+
await this.entryRepo.upsert(created, tx);
|
|
6633
|
+
if (newValidFrom !== void 0) {
|
|
6634
|
+
await this.entryRepo.setTemporal(created.id, entityId, { valid_from: newValidFrom }, tx);
|
|
6635
|
+
}
|
|
6636
|
+
}
|
|
6637
|
+
await this.supersedeInTx(tx, entityId, oldId, newId, t, now);
|
|
6638
|
+
});
|
|
6639
|
+
await this.searchService.syncEntries(entityId, [oldId, newId]);
|
|
6640
|
+
if (created) {
|
|
6641
|
+
await this.embeddingService.embedFact(
|
|
6642
|
+
{ id: created.id, entity_id: entityId, title: created.title, body: created.body, tags: created.tags },
|
|
6643
|
+
{ operation: "supersede", trigger: "call" }
|
|
6644
|
+
);
|
|
6645
|
+
}
|
|
6646
|
+
this.searchService.evictCache(entityId);
|
|
6647
|
+
return { newId };
|
|
6648
|
+
}
|
|
6649
|
+
async history(entityId, factId) {
|
|
6650
|
+
const start = await this.entryRepo.findTemporalRow(factId);
|
|
6651
|
+
if (!start || start.entity_id !== entityId || start.deleted_at != null) return [];
|
|
6652
|
+
const { predecessors, successors } = await this.entryRepo.findSupersessionChainIds(entityId, factId, HISTORY_MAX_DEPTH);
|
|
6653
|
+
const facts = await this.entryRepo.findByIds([...predecessors, factId, ...successors], [entityId]);
|
|
6654
|
+
return facts.sort((a, b) => (a.valid_from ?? a.created_at) - (b.valid_from ?? b.created_at) || a.created_at - b.created_at || a.id.localeCompare(b.id));
|
|
6655
|
+
}
|
|
6656
|
+
};
|
|
6657
|
+
|
|
6658
|
+
// src/services/librarian/ops/apply.ts
|
|
6659
|
+
async function applyOps(deps, ctx, input) {
|
|
6660
|
+
const { entityId, trigger } = ctx;
|
|
6661
|
+
const diagBase = { entityId, operation: "librarian", trigger };
|
|
6662
|
+
const grounding = deps.promptService.groundingFor("librarian");
|
|
6663
|
+
const temporal = new TemporalService(deps.db, deps.options, deps.entryRepo, deps.searchService, deps.embeddingService);
|
|
6664
|
+
const rejected = new Set(input.rejected);
|
|
6665
|
+
const failed = new Set(input.failed);
|
|
6666
|
+
const written = [];
|
|
6667
|
+
const updatedIds = [];
|
|
6668
|
+
const now = Date.now();
|
|
6669
|
+
await deps.db.withTransactionAsync(async (tx) => {
|
|
6670
|
+
let { mode, manifest } = await deps.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
6671
|
+
if (mode === "emergent" && input.ontologyUpdates && deps.ontologyService) {
|
|
6672
|
+
manifest = await deps.ontologyService.mergeEmergentUpdates(entityId, input.ontologyUpdates, tx);
|
|
6673
|
+
}
|
|
6674
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
6675
|
+
for (const f of await deps.entryRepo.findRecentByEntityId(entityId, 100, tx)) {
|
|
6676
|
+
titleIndex.set(normalizeTitleKey(f.title), { id: f.id, okf_type: f.okf_type ?? null });
|
|
6677
|
+
}
|
|
6678
|
+
const pendingEdges = [];
|
|
6679
|
+
const insert = async (g, draft, validFrom, overrides) => {
|
|
6680
|
+
const fact = g.candidate.fact;
|
|
6681
|
+
const title = overrides?.title ?? fact.title;
|
|
6682
|
+
const body = overrides?.body ?? fact.body;
|
|
6683
|
+
const id = generateId("fact_");
|
|
6684
|
+
const drops = [];
|
|
6685
|
+
const normalized = deps.ontologyService?.validateAndNormalizeFact(fact, manifest, { strict: false, drops }) ?? { okf_type: null, edges: [] };
|
|
6686
|
+
for (const d of drops) input.diagnostics.push(edgeDropDiagnostic(d, { ...diagBase, factId: id }));
|
|
6687
|
+
const g2 = grounding ? groundingOutcome(checkGrounding(fact.evidence, input.groundingCorpus ?? [], grounding), now) : null;
|
|
6688
|
+
if (g2?.diagnostic) {
|
|
6689
|
+
input.diagnostics.push({
|
|
6690
|
+
...diagBase,
|
|
6691
|
+
code: g2.diagnostic.code,
|
|
6692
|
+
detail: { factId: id, itemIndex: g.candidate.index, reason: g2.diagnostic.reason }
|
|
6693
|
+
});
|
|
6694
|
+
}
|
|
6695
|
+
const row = {
|
|
6696
|
+
id,
|
|
6697
|
+
entity_id: entityId,
|
|
6698
|
+
title,
|
|
6699
|
+
body,
|
|
6700
|
+
tags: fact.tags,
|
|
6701
|
+
confidence: fact.confidence,
|
|
6702
|
+
source_type: "librarian_inferred",
|
|
6703
|
+
source_hash: null,
|
|
6704
|
+
source_ref: null,
|
|
6705
|
+
created_at: now,
|
|
6706
|
+
updated_at: now,
|
|
6707
|
+
last_accessed_at: null,
|
|
6708
|
+
access_count: 0,
|
|
6709
|
+
deleted_at: null,
|
|
6710
|
+
okf_type: normalized.okf_type,
|
|
6711
|
+
...g2?.trust,
|
|
6712
|
+
...draft ? { lifecycle_status: "draft" } : {}
|
|
6713
|
+
};
|
|
6714
|
+
await deps.entryRepo.upsert(row, tx);
|
|
6715
|
+
await deps.entryRepo.setTemporal(id, entityId, { valid_from: validFrom }, tx);
|
|
6716
|
+
titleIndex.set(normalizeTitleKey(title), { id, okf_type: normalized.okf_type });
|
|
6717
|
+
if (normalized.edges.length > 0) pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
6718
|
+
const textChanged = title !== fact.title || body !== fact.body;
|
|
6719
|
+
written.push({ id, vector: textChanged ? null : g.vector, fact: { id, entity_id: entityId, title, body, tags: fact.tags } });
|
|
6720
|
+
return id;
|
|
6721
|
+
};
|
|
6722
|
+
for (const g of input.gated) {
|
|
6723
|
+
const idx = g.candidate.index;
|
|
6724
|
+
const vf = g.candidate.validFrom;
|
|
6725
|
+
if (g.decision.kind === "noop") continue;
|
|
6726
|
+
if (g.decision.kind === "add") {
|
|
6727
|
+
await insert(g, false, vf);
|
|
6728
|
+
continue;
|
|
6729
|
+
}
|
|
6730
|
+
if (failed.has(g)) {
|
|
6731
|
+
const id = await insert(g, true, vf);
|
|
6732
|
+
input.diagnostics.push({ ...diagBase, code: "resolve_failed", detail: { factId: id, itemIndex: idx } });
|
|
6733
|
+
continue;
|
|
6734
|
+
}
|
|
6735
|
+
const op = input.ops.get(g);
|
|
6736
|
+
if (rejected.has(g) || !op) {
|
|
6737
|
+
await insert(g, false, vf);
|
|
6738
|
+
input.diagnostics.push({ ...diagBase, code: "librarian_op_rejected", detail: { itemIndex: idx, reason: "invalid_op" } });
|
|
6739
|
+
continue;
|
|
6740
|
+
}
|
|
6741
|
+
if (op.op === "NOOP") continue;
|
|
6742
|
+
if (op.op === "ADD") {
|
|
6743
|
+
await insert(g, false, vf);
|
|
6744
|
+
continue;
|
|
6745
|
+
}
|
|
6746
|
+
const target = g.neighbours.find((n) => n.id === op.targetId);
|
|
6747
|
+
if (op.op === "UPDATE" && target.source_type === "librarian_inferred") {
|
|
6748
|
+
const [existing] = await deps.entryRepo.findByIds([op.targetId], [entityId], tx);
|
|
6749
|
+
if (existing) {
|
|
6750
|
+
const newTitle = op.title ?? g.candidate.fact.title;
|
|
6751
|
+
await deps.entryRepo.upsert({
|
|
6752
|
+
...existing,
|
|
6753
|
+
title: newTitle,
|
|
6754
|
+
body: op.body ?? g.candidate.fact.body,
|
|
6755
|
+
updated_at: now
|
|
6756
|
+
}, tx);
|
|
6757
|
+
titleIndex.delete(normalizeTitleKey(existing.title));
|
|
6758
|
+
titleIndex.set(normalizeTitleKey(newTitle), { id: existing.id, okf_type: existing.okf_type ?? null });
|
|
6759
|
+
updatedIds.push(existing.id);
|
|
6760
|
+
continue;
|
|
6761
|
+
}
|
|
6762
|
+
}
|
|
6763
|
+
const supersedeFrom = op.op === "SUPERSEDE" && op.validFrom !== void 0 ? op.validFrom : vf;
|
|
6764
|
+
const overrides = op.op === "UPDATE" ? { title: op.title, body: op.body } : void 0;
|
|
6765
|
+
if (target.source_type === "immutable_document") {
|
|
6766
|
+
const id = await insert(g, true, supersedeFrom, overrides);
|
|
6767
|
+
input.diagnostics.push({ ...diagBase, code: "contradicts_document", detail: { factId: id, itemIndex: idx } });
|
|
6768
|
+
continue;
|
|
6769
|
+
}
|
|
6770
|
+
const newId = await insert(g, false, supersedeFrom, overrides);
|
|
6771
|
+
try {
|
|
6772
|
+
await temporal.supersedeInTx(tx, entityId, op.targetId, newId, supersedeFrom, now);
|
|
6773
|
+
} catch (err) {
|
|
6774
|
+
if (!(err instanceof WikiSupersedeError)) throw err;
|
|
6775
|
+
input.diagnostics.push({
|
|
6776
|
+
...diagBase,
|
|
6777
|
+
code: "librarian_op_rejected",
|
|
6778
|
+
detail: { factId: newId, itemIndex: idx, reason: err.reason }
|
|
6779
|
+
});
|
|
6780
|
+
}
|
|
6781
|
+
}
|
|
6782
|
+
for (const item of pendingEdges) {
|
|
6783
|
+
const resolveDrops = [];
|
|
6784
|
+
await deps.ontologyService?.resolveAndPersistEdges(
|
|
6785
|
+
entityId,
|
|
6786
|
+
item.sourceId,
|
|
6787
|
+
item.sourceType,
|
|
6788
|
+
item.edges,
|
|
6789
|
+
manifest,
|
|
6790
|
+
titleIndex,
|
|
6791
|
+
tx,
|
|
6792
|
+
now,
|
|
6793
|
+
resolveDrops
|
|
6794
|
+
);
|
|
6795
|
+
for (const d of resolveDrops) input.diagnostics.push(edgeDropDiagnostic(d, diagBase));
|
|
6796
|
+
}
|
|
6797
|
+
for (const task of input.tasks) {
|
|
6798
|
+
const taskObj = {
|
|
6799
|
+
id: generateId("task_"),
|
|
6800
|
+
entity_id: entityId,
|
|
6801
|
+
description: task.description,
|
|
6802
|
+
status: "pending",
|
|
6803
|
+
priority: task.priority,
|
|
6804
|
+
created_at: now,
|
|
6805
|
+
updated_at: now,
|
|
6806
|
+
resolved_at: null,
|
|
6807
|
+
deleted_at: null
|
|
6808
|
+
};
|
|
6809
|
+
await deps.taskRepo.upsert(taskObj, tx);
|
|
6810
|
+
}
|
|
6811
|
+
});
|
|
6812
|
+
return { written, updatedIds };
|
|
6813
|
+
}
|
|
6814
|
+
|
|
6815
|
+
// src/services/librarian/ops/index.ts
|
|
6816
|
+
var runOpsLibrarianPass = async (deps, ctx) => {
|
|
6817
|
+
const { entityId, trigger } = ctx;
|
|
6818
|
+
const cfg = resolveGateConfig(deps.options.config?.librarian?.gate);
|
|
6819
|
+
await deps.seedWatermark(entityId);
|
|
6820
|
+
const wm = await deps.getWatermark(entityId);
|
|
6821
|
+
const sel = await selectBatch(deps, entityId, wm, ctx.meter);
|
|
6822
|
+
if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
|
|
6823
|
+
if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
|
|
6824
|
+
let ex;
|
|
6825
|
+
try {
|
|
6826
|
+
ex = await runExtract(deps, ctx, sel.batch);
|
|
6827
|
+
} catch (err) {
|
|
6828
|
+
if (err instanceof WikiBudgetExhausted) {
|
|
6829
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: { requiredEstimate: err.requiredEstimate } };
|
|
6830
|
+
}
|
|
6831
|
+
throw err;
|
|
6832
|
+
}
|
|
6833
|
+
const diagnostics = new DiagnosticBuffer();
|
|
6834
|
+
for (const d of ex.diagnostics) diagnostics.push(d);
|
|
6835
|
+
const { gated, inBatchDuplicates, gateDiagnostic } = await gateCandidates(deps, ctx, ex.candidates, cfg);
|
|
6836
|
+
for (const dup of inBatchDuplicates) {
|
|
6837
|
+
diagnostics.push({
|
|
6838
|
+
code: "fact_deduplicated",
|
|
6839
|
+
operation: "librarian",
|
|
6840
|
+
trigger,
|
|
6841
|
+
entityId,
|
|
6842
|
+
detail: { itemIndex: dup.index, reason: "in_batch" }
|
|
6843
|
+
});
|
|
6844
|
+
}
|
|
6845
|
+
diagnostics.push(gateDiagnostic);
|
|
6846
|
+
const res = await runResolve(deps, ctx, gated.filter((g) => g.decision.kind === "ambiguous"));
|
|
6847
|
+
if (res.budgetStop) {
|
|
6848
|
+
diagnostics.discard();
|
|
6849
|
+
return { processedThrough: null, factsWritten: 0, budgetStop: res.budgetStop };
|
|
6850
|
+
}
|
|
6851
|
+
const { written, updatedIds } = await applyOps(deps, ctx, {
|
|
6852
|
+
gated,
|
|
6853
|
+
ops: res.ops,
|
|
6854
|
+
rejected: res.rejected,
|
|
6855
|
+
failed: res.failed,
|
|
6856
|
+
tasks: ex.tasks,
|
|
6857
|
+
ontologyUpdates: ex.ontologyUpdates,
|
|
6858
|
+
groundingCorpus: ex.groundingCorpus,
|
|
6859
|
+
diagnostics
|
|
6860
|
+
});
|
|
6861
|
+
diagnostics.flush(deps.options);
|
|
6862
|
+
await deps.searchService.syncEntries(entityId, [...written.map((w) => w.id), ...updatedIds]);
|
|
6863
|
+
const embedCtx = { operation: "librarian", trigger };
|
|
6864
|
+
for (const w of written) {
|
|
6865
|
+
if (w.vector) await deps.embeddingService.storeFactVector(w.fact, w.vector, embedCtx);
|
|
6866
|
+
else await deps.embeddingService.embedFact(w.fact, embedCtx);
|
|
6867
|
+
}
|
|
6868
|
+
if (updatedIds.length > 0) {
|
|
6869
|
+
for (const f of await deps.entryRepo.findByIds(updatedIds, [entityId])) {
|
|
6870
|
+
await deps.embeddingService.embedFact(
|
|
6871
|
+
{ id: f.id, entity_id: f.entity_id, title: f.title, body: f.body, tags: f.tags },
|
|
6872
|
+
embedCtx
|
|
6873
|
+
);
|
|
6874
|
+
}
|
|
6875
|
+
}
|
|
6876
|
+
deps.searchService.evictCache(entityId);
|
|
6877
|
+
const last = sel.batch[sel.batch.length - 1].event;
|
|
6878
|
+
return {
|
|
6879
|
+
processedThrough: { at: last.created_at, id: last.id },
|
|
6880
|
+
factsWritten: written.length + updatedIds.length
|
|
6881
|
+
};
|
|
6882
|
+
};
|
|
6883
|
+
|
|
6884
|
+
// src/services/librarian/index.ts
|
|
6885
|
+
var STRATEGIES = /* @__PURE__ */ new Map([
|
|
6886
|
+
["legacy", runLegacyLibrarianPass],
|
|
6887
|
+
["ops", runOpsLibrarianPass]
|
|
6888
|
+
]);
|
|
6889
|
+
async function runLibrarianStrategy(deps, ctx) {
|
|
6890
|
+
const strategy = deps.options.config?.librarian?.strategy ?? "legacy";
|
|
6891
|
+
const fn = STRATEGIES.get(strategy);
|
|
6892
|
+
if (!fn) {
|
|
6893
|
+
throw new Error(`Unknown librarian strategy: ${strategy}`);
|
|
6894
|
+
}
|
|
6895
|
+
return fn(deps, ctx);
|
|
6896
|
+
}
|
|
6897
|
+
|
|
6898
|
+
// src/services/MaintenanceService.ts
|
|
6899
|
+
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
6900
|
+
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
6901
|
+
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
6902
|
+
var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
|
|
6903
|
+
var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
6904
|
+
var HEAL_BATCH_SIZE = 25;
|
|
6905
|
+
var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
6906
|
+
var MAX_EMBED_ATTEMPTS = 5;
|
|
6907
|
+
var EMBED_RETRY_BASE_MS = 6e4;
|
|
6908
|
+
var EMBED_RETRY_CAP_MS = 24 * 60 * 60 * 1e3;
|
|
6909
|
+
function embedRetryDelayMs(attempts) {
|
|
6910
|
+
const n = Math.max(1, Math.trunc(attempts || 0));
|
|
6911
|
+
const delay = EMBED_RETRY_BASE_MS * Math.pow(2, n - 1);
|
|
6912
|
+
return Math.min(delay, EMBED_RETRY_CAP_MS);
|
|
6913
|
+
}
|
|
6914
|
+
function classifyReembedRow(row, now, force) {
|
|
6915
|
+
if (force) return "attempt";
|
|
6916
|
+
const failedAt = row.embedding_failed_at;
|
|
6917
|
+
if (failedAt === null || failedAt === void 0) return "attempt";
|
|
6918
|
+
if (row.embedding_failure_kind === "float32_overflow") return "permanent";
|
|
6919
|
+
const attempts = row.embedding_attempts ?? 0;
|
|
6920
|
+
if (attempts >= MAX_EMBED_ATTEMPTS) return "permanent";
|
|
6921
|
+
return now - failedAt >= embedRetryDelayMs(attempts) ? "attempt" : "defer";
|
|
6922
|
+
}
|
|
6923
|
+
var SKIP_ERROR_LOG_CHARS = 4096;
|
|
6924
|
+
var formatSkipError = (err) => {
|
|
6925
|
+
let base;
|
|
6926
|
+
let isErrorLike = false;
|
|
6927
|
+
try {
|
|
6928
|
+
isErrorLike = err instanceof Error;
|
|
6929
|
+
} catch {
|
|
6930
|
+
}
|
|
6931
|
+
if (isErrorLike || typeof err !== "object" && typeof err !== "function") {
|
|
6932
|
+
base = safeErrorToString(err);
|
|
6933
|
+
} else {
|
|
6934
|
+
try {
|
|
6935
|
+
const json = JSON.stringify(err);
|
|
6936
|
+
base = typeof json === "string" ? json : safeErrorToString(err);
|
|
6937
|
+
} catch {
|
|
6938
|
+
base = safeErrorToString(err);
|
|
6939
|
+
}
|
|
6940
|
+
}
|
|
6941
|
+
if (base.length > SKIP_ERROR_LOG_CHARS) {
|
|
6942
|
+
const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
|
|
6943
|
+
return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
|
|
6944
|
+
}
|
|
6945
|
+
return base;
|
|
6946
|
+
};
|
|
6947
|
+
var MaintenanceService = class {
|
|
6948
|
+
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
6949
|
+
this.db = db;
|
|
6950
|
+
this.prefix = prefix;
|
|
6951
|
+
this.options = options;
|
|
6952
|
+
this.entryRepo = entryRepo;
|
|
6953
|
+
this.sourceRefIndexRepo = sourceRefIndexRepo;
|
|
6954
|
+
this.taskRepo = taskRepo;
|
|
6955
|
+
this.eventRepo = eventRepo;
|
|
6956
|
+
this.metadataRepo = metadataRepo;
|
|
6957
|
+
this.searchService = searchService;
|
|
6958
|
+
this.jobManager = jobManager;
|
|
6959
|
+
this.embeddingService = embeddingService;
|
|
6960
|
+
this.ontologyService = ontologyService;
|
|
6961
|
+
this.promptService = promptService ?? new PromptService(this.options.config?.prompts, resolveGrounding(this.options.config?.grounding));
|
|
6962
|
+
}
|
|
6963
|
+
async runPrune(entityId, options) {
|
|
6964
|
+
this.jobManager.acquireLock("prune", entityId);
|
|
6965
|
+
try {
|
|
6966
|
+
const retainSoftDeletedFor = options?.retainSoftDeletedFor !== void 0 ? options.retainSoftDeletedFor : this.options.config?.pruneRetainSoftDeletedFor ?? 7;
|
|
6967
|
+
const retainEventsFor = options?.retainEventsFor !== void 0 ? options.retainEventsFor : this.options.config?.pruneEventsAfter ?? 30;
|
|
6968
|
+
const vacuum = options?.vacuum ?? false;
|
|
6969
|
+
this._validatePruneDuration(retainSoftDeletedFor, "retainSoftDeletedFor");
|
|
6970
|
+
this._validatePruneDuration(retainEventsFor, "retainEventsFor");
|
|
6971
|
+
const pruneSupersededAfter = this.options.config?.pruneSupersededAfter ?? null;
|
|
6972
|
+
this._validatePruneDuration(pruneSupersededAfter, "pruneSupersededAfter");
|
|
6973
|
+
const now = Date.now();
|
|
6974
|
+
let syncedIds = [];
|
|
6975
|
+
let deletedEntries = 0;
|
|
6976
|
+
let deletedTasks = 0;
|
|
5762
6977
|
let deletedEvents = 0;
|
|
6978
|
+
if (pruneSupersededAfter !== null) {
|
|
6979
|
+
const expiredCutoff = now - pruneSupersededAfter * 864e5;
|
|
6980
|
+
const expiredIds = await this.entryRepo.findExpiredIds(entityId, expiredCutoff);
|
|
6981
|
+
if (expiredIds.length > 0) {
|
|
6982
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
6983
|
+
await this.entryRepo.softDeleteByIds(expiredIds, entityId, tx);
|
|
6984
|
+
});
|
|
6985
|
+
syncedIds = [...expiredIds];
|
|
6986
|
+
}
|
|
6987
|
+
}
|
|
5763
6988
|
if (retainSoftDeletedFor !== null) {
|
|
5764
6989
|
const cutoff = now - retainSoftDeletedFor * 864e5;
|
|
5765
6990
|
const entriesToDelete = await this.entryRepo.getPrunableMetadata(entityId, cutoff);
|
|
@@ -5775,6 +7000,7 @@ var MaintenanceService = class {
|
|
|
5775
7000
|
}
|
|
5776
7001
|
}
|
|
5777
7002
|
const succeededIds = succeeded.map((r) => r.id);
|
|
7003
|
+
syncedIds = [...syncedIds, ...succeededIds];
|
|
5778
7004
|
await this.db.withTransactionAsync(async (tx) => {
|
|
5779
7005
|
if (succeededIds.length > 0) {
|
|
5780
7006
|
deletedEntries = await this.entryRepo.bulkDeletePruned(entityId, cutoff, succeededIds, tx);
|
|
@@ -5782,7 +7008,7 @@ var MaintenanceService = class {
|
|
|
5782
7008
|
deletedTasks = await this.taskRepo.bulkDeletePruned(entityId, cutoff, tx);
|
|
5783
7009
|
});
|
|
5784
7010
|
if (failure) {
|
|
5785
|
-
await this.searchService.
|
|
7011
|
+
await this.searchService.syncEntries(entityId, syncedIds);
|
|
5786
7012
|
const remaining = entriesToDelete.length - succeeded.length - 1;
|
|
5787
7013
|
const isTimeout = failure.cause?.[HOOK_TIMEOUT_MARKER] === true;
|
|
5788
7014
|
if (isTimeout) {
|
|
@@ -5810,13 +7036,23 @@ var MaintenanceService = class {
|
|
|
5810
7036
|
}
|
|
5811
7037
|
if (retainEventsFor !== null) {
|
|
5812
7038
|
const cutoff = now - retainEventsFor * 864e5;
|
|
5813
|
-
const
|
|
5814
|
-
|
|
7039
|
+
const guarded = this.options.config?.maintenance === "deferred" || this.options.config?.librarian?.strategy === "ops";
|
|
7040
|
+
if (guarded) {
|
|
7041
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
7042
|
+
deletedEvents = (await this.eventRepo.pruneThrough(entityId, cutoff, wm)).changes;
|
|
7043
|
+
const held = await this.eventRepo.countHeldBack(entityId, cutoff, wm);
|
|
7044
|
+
if (held > 0) {
|
|
7045
|
+
emitDiagnostic(this.options, { code: "event_retention_held", operation: "prune", trigger: "call", entityId, detail: { count: held } });
|
|
7046
|
+
}
|
|
7047
|
+
} else {
|
|
7048
|
+
const eventResult = await this.eventRepo.prune(entityId, cutoff);
|
|
7049
|
+
deletedEvents = eventResult.changes;
|
|
7050
|
+
}
|
|
5815
7051
|
}
|
|
5816
7052
|
if (vacuum) {
|
|
5817
7053
|
await this.metadataRepo.vacuum();
|
|
5818
7054
|
}
|
|
5819
|
-
await this.searchService.
|
|
7055
|
+
await this.searchService.syncEntries(entityId, syncedIds);
|
|
5820
7056
|
return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
|
|
5821
7057
|
} finally {
|
|
5822
7058
|
this.jobManager.releaseLock("prune", entityId);
|
|
@@ -5986,8 +7222,8 @@ var MaintenanceService = class {
|
|
|
5986
7222
|
deletedEntries += refResult;
|
|
5987
7223
|
}
|
|
5988
7224
|
});
|
|
5989
|
-
await this.searchService.sync(entityId);
|
|
5990
7225
|
const uniqueDeletedIds = Array.from(new Set(deletedEntryIds));
|
|
7226
|
+
await this.searchService.syncEntries(entityId, uniqueDeletedIds);
|
|
5991
7227
|
for (const factId of uniqueDeletedIds) {
|
|
5992
7228
|
try {
|
|
5993
7229
|
await this.embeddingService.notifyEmbeddingPersistedOrThrow(entityId, factId, null);
|
|
@@ -6041,155 +7277,72 @@ var MaintenanceService = class {
|
|
|
6041
7277
|
}
|
|
6042
7278
|
/** Core librarian pass (locks handled by {@link runLibrarian}). Package-internal orchestration hook. */
|
|
6043
7279
|
async doRunLibrarian(entityId, promptOverride, trigger = "call") {
|
|
6044
|
-
|
|
6045
|
-
|
|
6046
|
-
|
|
6047
|
-
|
|
6048
|
-
|
|
6049
|
-
|
|
6050
|
-
|
|
6051
|
-
|
|
6052
|
-
|
|
6053
|
-
|
|
6054
|
-
|
|
6055
|
-
|
|
6056
|
-
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
6060
|
-
|
|
6061
|
-
|
|
6062
|
-
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
6063
|
-
const result = parseJsonResponse(responseText);
|
|
6064
|
-
const facts = Array.isArray(result.facts) ? result.facts : [];
|
|
6065
|
-
const tasks = Array.isArray(result.tasks) ? result.tasks : [];
|
|
6066
|
-
const ontologyUpdates = result.ontology_updates;
|
|
6067
|
-
const diagBuffer = new DiagnosticBuffer();
|
|
6068
|
-
const diagBase = { entityId, operation: "librarian", trigger };
|
|
6069
|
-
const validFacts = [];
|
|
6070
|
-
const validFactItemIndexes = [];
|
|
6071
|
-
facts.forEach((raw, itemIndex) => {
|
|
6072
|
-
const valid = validateFact(raw);
|
|
6073
|
-
if (valid) {
|
|
6074
|
-
validFacts.push(valid);
|
|
6075
|
-
validFactItemIndexes.push(itemIndex);
|
|
6076
|
-
} else {
|
|
6077
|
-
diagBuffer.push({ ...diagBase, code: "fact_rejected", detail: { itemIndex, reason: factRejectionReason(raw) } });
|
|
6078
|
-
}
|
|
6079
|
-
});
|
|
6080
|
-
const validTasks = [];
|
|
6081
|
-
tasks.forEach((raw, itemIndex) => {
|
|
6082
|
-
const valid = validateTask(raw);
|
|
6083
|
-
if (valid) validTasks.push(valid);
|
|
6084
|
-
else diagBuffer.push({ ...diagBase, code: "task_rejected", detail: { itemIndex, reason: taskRejectionReason(raw) } });
|
|
6085
|
-
});
|
|
6086
|
-
const now = Date.now();
|
|
6087
|
-
const insertedFacts = [];
|
|
6088
|
-
await this.db.withTransactionAsync(async (tx) => {
|
|
6089
|
-
let { mode, manifest } = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
6090
|
-
if (mode === "emergent" && ontologyUpdates && this.ontologyService) {
|
|
6091
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
|
|
6092
|
-
}
|
|
6093
|
-
const titleIndex = /* @__PURE__ */ new Map();
|
|
6094
|
-
for (const existing of currentFactsRows) {
|
|
6095
|
-
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
6096
|
-
id: existing.id,
|
|
6097
|
-
okf_type: existing.okf_type ?? null
|
|
6098
|
-
});
|
|
6099
|
-
}
|
|
6100
|
-
const factsForDedupe = await this.entryRepo.findRecentByEntityId(entityId, 100, tx);
|
|
6101
|
-
const pendingEdges = [];
|
|
6102
|
-
for (const [k, fact] of validFacts.entries()) {
|
|
6103
|
-
const newTokens = titleTokens(fact.title);
|
|
6104
|
-
let skip = false;
|
|
6105
|
-
if (newTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
6106
|
-
for (const existing of factsForDedupe) {
|
|
6107
|
-
if (existing.source_type !== "librarian_inferred") continue;
|
|
6108
|
-
const existingTokens = titleTokens(existing.title);
|
|
6109
|
-
if (existingTokens.size >= MIN_TOKENS_TO_QUALIFY) {
|
|
6110
|
-
if (jaccardScore(newTokens, existingTokens) >= FUZZY_THRESHOLD) {
|
|
6111
|
-
skip = true;
|
|
6112
|
-
break;
|
|
6113
|
-
}
|
|
6114
|
-
}
|
|
6115
|
-
}
|
|
6116
|
-
}
|
|
6117
|
-
if (skip) {
|
|
6118
|
-
diagBuffer.push({ ...diagBase, code: "fact_deduplicated", detail: { itemIndex: validFactItemIndexes[k], reason: "fuzzy_title" } });
|
|
6119
|
-
continue;
|
|
6120
|
-
}
|
|
6121
|
-
const ontologyFact = fact;
|
|
6122
|
-
const id = generateId("fact_");
|
|
6123
|
-
const validationDrops = [];
|
|
6124
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false, drops: validationDrops }) ?? { okf_type: null, edges: [] };
|
|
6125
|
-
for (const drop of validationDrops) diagBuffer.push(edgeDropDiagnostic(drop, { ...diagBase, factId: id }));
|
|
6126
|
-
const grounding = librarianGrounding ? groundingOutcome(checkGrounding(fact.evidence, librarianCorpus, librarianGrounding), now) : null;
|
|
6127
|
-
if (grounding?.diagnostic) {
|
|
6128
|
-
diagBuffer.push({ ...diagBase, code: grounding.diagnostic.code, detail: { factId: id, itemIndex: validFactItemIndexes[k], reason: grounding.diagnostic.reason } });
|
|
6129
|
-
}
|
|
6130
|
-
const factObj = {
|
|
6131
|
-
id,
|
|
6132
|
-
entity_id: entityId,
|
|
6133
|
-
title: fact.title,
|
|
6134
|
-
body: fact.body,
|
|
6135
|
-
tags: fact.tags,
|
|
6136
|
-
confidence: fact.confidence,
|
|
6137
|
-
source_type: "librarian_inferred",
|
|
6138
|
-
source_hash: null,
|
|
6139
|
-
source_ref: null,
|
|
6140
|
-
created_at: now,
|
|
6141
|
-
updated_at: now,
|
|
6142
|
-
last_accessed_at: null,
|
|
6143
|
-
access_count: 0,
|
|
6144
|
-
deleted_at: null,
|
|
6145
|
-
okf_type: normalized.okf_type,
|
|
6146
|
-
...grounding?.trust
|
|
6147
|
-
};
|
|
6148
|
-
await this.entryRepo.upsert(factObj, tx);
|
|
6149
|
-
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
6150
|
-
factsForDedupe.push(factObj);
|
|
6151
|
-
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
6152
|
-
if (normalized.edges.length > 0) {
|
|
6153
|
-
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
6154
|
-
}
|
|
6155
|
-
}
|
|
6156
|
-
for (const item of pendingEdges) {
|
|
6157
|
-
const resolveDrops = [];
|
|
6158
|
-
await this.ontologyService?.resolveAndPersistEdges(
|
|
6159
|
-
entityId,
|
|
6160
|
-
item.sourceId,
|
|
6161
|
-
item.sourceType,
|
|
6162
|
-
item.edges ?? [],
|
|
6163
|
-
manifest,
|
|
6164
|
-
titleIndex,
|
|
6165
|
-
tx,
|
|
6166
|
-
now,
|
|
6167
|
-
resolveDrops
|
|
6168
|
-
);
|
|
6169
|
-
for (const drop of resolveDrops) diagBuffer.push(edgeDropDiagnostic(drop, diagBase));
|
|
6170
|
-
}
|
|
6171
|
-
for (const task of validTasks) {
|
|
6172
|
-
const id = generateId("task_");
|
|
6173
|
-
const taskObj = {
|
|
6174
|
-
id,
|
|
6175
|
-
entity_id: entityId,
|
|
6176
|
-
description: task.description,
|
|
6177
|
-
status: "pending",
|
|
6178
|
-
priority: task.priority,
|
|
6179
|
-
created_at: now,
|
|
6180
|
-
updated_at: now,
|
|
6181
|
-
resolved_at: null,
|
|
6182
|
-
deleted_at: null
|
|
6183
|
-
};
|
|
6184
|
-
await this.taskRepo.upsert(taskObj, tx);
|
|
6185
|
-
}
|
|
6186
|
-
});
|
|
6187
|
-
diagBuffer.flush(this.options);
|
|
6188
|
-
await this.searchService.sync(entityId);
|
|
6189
|
-
for (const fact of insertedFacts) {
|
|
6190
|
-
await this.embeddingService.embedFact(fact, { operation: "librarian", trigger });
|
|
7280
|
+
await this.runLibrarianPass({ entityId, trigger, ...promptOverride !== void 0 ? { promptOverride } : {} });
|
|
7281
|
+
}
|
|
7282
|
+
/**
|
|
7283
|
+
* Seam between the lock wrapper and the strategy dispatcher (spec 2026-09-29
|
|
7284
|
+
* §5.1). Builds the strategy deps from `this` and dispatches via
|
|
7285
|
+
* `runLibrarianStrategy` so PR-B can plug in `ops/` without touching this
|
|
7286
|
+
* class. Returns the strategy's `LibrarianResult` so deferred/budgeted
|
|
7287
|
+
* schedulers (PR-C) can advance watermarks; the existing void-returning
|
|
7288
|
+
* `doRunLibrarian` discards it.
|
|
7289
|
+
*
|
|
7290
|
+
* Spec §10.3: one place advances the watermark, for every strategy, only
|
|
7291
|
+
* after the pass resolved (its transaction committed) and never backwards.
|
|
7292
|
+
* A budget-stopped pass wrote nothing and must not move the watermark.
|
|
7293
|
+
*/
|
|
7294
|
+
async runLibrarianPass(ctx) {
|
|
7295
|
+
const result = await runLibrarianStrategy(this.librarianDeps(), ctx);
|
|
7296
|
+
if (result.processedThrough && !result.budgetStop) {
|
|
7297
|
+
await this.advanceLibrarianWatermark(ctx.entityId, result.processedThrough);
|
|
6191
7298
|
}
|
|
6192
|
-
|
|
7299
|
+
return result;
|
|
7300
|
+
}
|
|
7301
|
+
/**
|
|
7302
|
+
* Strategy deps shared between legacy and ops librarians (spec §5.1). The
|
|
7303
|
+
* ops strategy (PR-B Task 8) reads/anchors the watermark through these two
|
|
7304
|
+
* hooks so the dispatcher itself never reaches into the metadata table —
|
|
7305
|
+
* the watermark invariant (one place advances, never backwards) lives here.
|
|
7306
|
+
*/
|
|
7307
|
+
librarianDeps() {
|
|
7308
|
+
return {
|
|
7309
|
+
db: this.db,
|
|
7310
|
+
options: this.options,
|
|
7311
|
+
entryRepo: this.entryRepo,
|
|
7312
|
+
taskRepo: this.taskRepo,
|
|
7313
|
+
eventRepo: this.eventRepo,
|
|
7314
|
+
searchService: this.searchService,
|
|
7315
|
+
embeddingService: this.embeddingService,
|
|
7316
|
+
promptService: this.promptService,
|
|
7317
|
+
ontologyService: this.ontologyService,
|
|
7318
|
+
seedWatermark: (entityId) => this.seedLibrarianWatermark(entityId),
|
|
7319
|
+
getWatermark: (entityId) => this.metadataRepo.getLibrarianWatermark(entityId, this.db)
|
|
7320
|
+
};
|
|
7321
|
+
}
|
|
7322
|
+
/**
|
|
7323
|
+
* Persist `next` as the entity's librarian watermark when it is strictly
|
|
7324
|
+
* newer than the stored value, under the (created_at, id) ordering used by
|
|
7325
|
+
* the op-based librarian (spec §10.3). Never moves backwards.
|
|
7326
|
+
*/
|
|
7327
|
+
async advanceLibrarianWatermark(entityId, next) {
|
|
7328
|
+
const current = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
7329
|
+
const newer = !current || next.at > current.at || next.at === current.at && next.id > current.id;
|
|
7330
|
+
if (newer) await this.metadataRepo.setLibrarianWatermark(entityId, next, this.db);
|
|
7331
|
+
}
|
|
7332
|
+
/**
|
|
7333
|
+
* Spec §10.3: first ops pass on a legacy DB starts after what the count
|
|
7334
|
+
* checkpoint already covered. No-op when a watermark already exists or the
|
|
7335
|
+
* memory checkpoint is 0; otherwise anchor the watermark to the event at
|
|
7336
|
+
* offset `min(memory_checkpoint, totalEvents) - 1`.
|
|
7337
|
+
*/
|
|
7338
|
+
async seedLibrarianWatermark(entityId) {
|
|
7339
|
+
if (await this.metadataRepo.getLibrarianWatermark(entityId, this.db)) return;
|
|
7340
|
+
const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
|
|
7341
|
+
const memory = cp.memory ?? 0;
|
|
7342
|
+
if (memory <= 0) return;
|
|
7343
|
+
const total = await this.eventRepo.count(entityId);
|
|
7344
|
+
const ev = await this.eventRepo.getAtOffset(entityId, Math.min(memory, total) - 1);
|
|
7345
|
+
if (ev) await this.metadataRepo.setLibrarianWatermark(entityId, { at: ev.created_at, id: ev.id }, this.db);
|
|
6193
7346
|
}
|
|
6194
7347
|
/**
|
|
6195
7348
|
* Core heal pass (locks handled by {@link runHeal}). Package-internal orchestration hook.
|
|
@@ -6243,7 +7396,7 @@ var MaintenanceService = class {
|
|
|
6243
7396
|
}
|
|
6244
7397
|
const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId, batchSize, recheckCutoff);
|
|
6245
7398
|
if (healCandidates.length === 0) {
|
|
6246
|
-
await this.searchService.
|
|
7399
|
+
await this.searchService.syncEntries(entityId, Array.from(/* @__PURE__ */ new Set([...orphanedIds, ...staleDowngradedIds])));
|
|
6247
7400
|
this.searchService.evictCache(entityId);
|
|
6248
7401
|
const counts2 = await this.entryRepo.countHealCandidatesByEntityId(entityId, recheckCutoff);
|
|
6249
7402
|
return {
|
|
@@ -6295,7 +7448,7 @@ var MaintenanceService = class {
|
|
|
6295
7448
|
if (groundingCorpus !== void 0) corpusByPrompt.set(prompts, groundingCorpus);
|
|
6296
7449
|
return prompts;
|
|
6297
7450
|
},
|
|
6298
|
-
call: (prompts) => this.options
|
|
7451
|
+
call: (prompts) => callLlm(this.options, { operation: "heal", entityId, trigger, meter: options?.meter }, prompts),
|
|
6299
7452
|
parse: (responseText, batch, prompts) => {
|
|
6300
7453
|
const result = parseJsonResponse(responseText);
|
|
6301
7454
|
return {
|
|
@@ -6399,9 +7552,10 @@ var MaintenanceService = class {
|
|
|
6399
7552
|
const callErrorIds = new Set(
|
|
6400
7553
|
outcome.skipped.filter((s) => s.reason === "call_error").map((s) => s.item.id)
|
|
6401
7554
|
);
|
|
7555
|
+
const unattemptedIds = new Set(outcome.unattempted.map((f) => f.id));
|
|
6402
7556
|
await this.entryRepo.markHealChecked(
|
|
6403
7557
|
[
|
|
6404
|
-
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id)),
|
|
7558
|
+
...healCandidates.map((f) => f.id).filter((id) => !callErrorIds.has(id) && !unattemptedIds.has(id)),
|
|
6405
7559
|
...insertedFacts.map((f) => f.id)
|
|
6406
7560
|
],
|
|
6407
7561
|
entityId,
|
|
@@ -6413,7 +7567,14 @@ var MaintenanceService = class {
|
|
|
6413
7567
|
diagBuffer.push({ ...diagBase, code: "heal_skipped", detail: { factId: item.id, reason } });
|
|
6414
7568
|
}
|
|
6415
7569
|
diagBuffer.flush(this.options);
|
|
6416
|
-
|
|
7570
|
+
const healSyncIds = Array.from(/* @__PURE__ */ new Set([
|
|
7571
|
+
...orphanedIds,
|
|
7572
|
+
...staleDowngradedIds,
|
|
7573
|
+
...safeDowngraded,
|
|
7574
|
+
...uniqueDeletedFactIds,
|
|
7575
|
+
...insertedFacts.map((f) => f.id)
|
|
7576
|
+
]));
|
|
7577
|
+
await this.searchService.syncEntries(entityId, healSyncIds);
|
|
6417
7578
|
for (const factId of uniqueDeletedFactIds) {
|
|
6418
7579
|
try {
|
|
6419
7580
|
await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
|
|
@@ -6446,7 +7607,8 @@ var MaintenanceService = class {
|
|
|
6446
7607
|
skipped: outcome.skipped.map(({ item, reason }) => ({ id: item.id, reason })),
|
|
6447
7608
|
degraded: healedDegraded,
|
|
6448
7609
|
remaining: counts.eligible,
|
|
6449
|
-
deferred: counts.deferred
|
|
7610
|
+
deferred: counts.deferred,
|
|
7611
|
+
...outcome.budgetStop ? { budgetStop: outcome.budgetStop } : {}
|
|
6450
7612
|
};
|
|
6451
7613
|
}
|
|
6452
7614
|
/** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
|
|
@@ -6490,7 +7652,7 @@ var MaintenanceService = class {
|
|
|
6490
7652
|
const outcome = await runBatched({
|
|
6491
7653
|
items: candidates,
|
|
6492
7654
|
buildPrompt,
|
|
6493
|
-
call: (prompts) => this.options
|
|
7655
|
+
call: (prompts) => callLlm(this.options, { operation: "ontologyBackfill", entityId, trigger: "call" }, prompts),
|
|
6494
7656
|
parse: (responseText, batch) => {
|
|
6495
7657
|
const parsed = parseJsonResponse(responseText);
|
|
6496
7658
|
return {
|
|
@@ -6783,6 +7945,186 @@ var MaintenanceService = class {
|
|
|
6783
7945
|
return sanitizeRankerError(err, this.options.sanitizeRankerErrors);
|
|
6784
7946
|
}
|
|
6785
7947
|
};
|
|
7948
|
+
|
|
7949
|
+
// src/services/MaintenanceScheduler.ts
|
|
7950
|
+
var MaintenanceScheduler = class {
|
|
7951
|
+
constructor(db, options, maintenanceService, metadataRepo, eventRepo, entryRepo, jobManager) {
|
|
7952
|
+
this.db = db;
|
|
7953
|
+
this.options = options;
|
|
7954
|
+
this.maintenanceService = maintenanceService;
|
|
7955
|
+
this.metadataRepo = metadataRepo;
|
|
7956
|
+
this.eventRepo = eventRepo;
|
|
7957
|
+
this.entryRepo = entryRepo;
|
|
7958
|
+
this.jobManager = jobManager;
|
|
7959
|
+
}
|
|
7960
|
+
/**
|
|
7961
|
+
* Entities with **any** pending work only, sorted by `pendingEvents` desc
|
|
7962
|
+
* then `entityId` asc. Token estimate is ceil(pending summary chars / 4),
|
|
7963
|
+
* matching the auto-trigger's estimator (PR-B).
|
|
7964
|
+
*/
|
|
7965
|
+
async getPending(entityIds) {
|
|
7966
|
+
const ids = entityIds ?? await this.metadataRepo.getDistinctEntityIds();
|
|
7967
|
+
const autoHealThreshold = this.options.config?.autoHealThreshold || 100;
|
|
7968
|
+
const canEmbed = typeof this.options.llmProvider.embed === "function";
|
|
7969
|
+
const out = [];
|
|
7970
|
+
for (const entityId of ids) {
|
|
7971
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
7972
|
+
const pendingEvents = await this.eventRepo.countAfter(entityId, wm);
|
|
7973
|
+
const chars = pendingEvents > 0 ? await this.eventRepo.sumSummaryCharsAfter(entityId, wm) : 0;
|
|
7974
|
+
const eventCount = await this.eventRepo.count(entityId);
|
|
7975
|
+
const cp = await this.metadataRepo.getCheckpoint(entityId, this.db);
|
|
7976
|
+
let healCheckpoint = cp.heal ?? 0;
|
|
7977
|
+
if (healCheckpoint > eventCount) healCheckpoint = 0;
|
|
7978
|
+
const healDue = eventCount - healCheckpoint >= autoHealThreshold;
|
|
7979
|
+
const reembedPending = canEmbed && await this.entryRepo.countReembedPending(entityId, MAX_EMBED_ATTEMPTS) > 0;
|
|
7980
|
+
if (pendingEvents > 0 || healDue || reembedPending) {
|
|
7981
|
+
out.push({ entityId, pendingEvents, pendingTokensEstimate: Math.ceil(chars / 4), healDue, reembedPending });
|
|
7982
|
+
}
|
|
7983
|
+
}
|
|
7984
|
+
return out.sort((x, y) => y.pendingEvents - x.pendingEvents || (x.entityId < y.entityId ? -1 : x.entityId > y.entityId ? 1 : 0));
|
|
7985
|
+
}
|
|
7986
|
+
/**
|
|
7987
|
+
* Execute pending maintenance (spec §6.2): fair-share rounds of at most one
|
|
7988
|
+
* unit per entity (librarian batch, else heal batch, else reembed), entities
|
|
7989
|
+
* ordered by `getPending` (pendingEvents desc, id asc). Holds the same
|
|
7990
|
+
* per-entity `JobManager` locks a manual run would; a busy entity is skipped
|
|
7991
|
+
* and reported, never thrown. Nothing is still running when this resolves.
|
|
7992
|
+
*
|
|
7993
|
+
* Stop precedence (spec §10 items 3-6): `deadline` > `budget_exhausted` >
|
|
7994
|
+
* `budget_too_small` > `complete`. `budget_too_small` means a job's smallest
|
|
7995
|
+
* unit needs more than the WHOLE budget — that job is skipped this run and
|
|
7996
|
+
* the run continues; `budget_exhausted` means a unit needs more than what
|
|
7997
|
+
* REMAINS — the run stops. The token budget meters `generateText` calls
|
|
7998
|
+
* only; embeddings are never metered.
|
|
7999
|
+
*/
|
|
8000
|
+
async run(opts = {}) {
|
|
8001
|
+
if (opts.deadlineMs !== void 0 && (typeof opts.deadlineMs !== "number" || !Number.isFinite(opts.deadlineMs) || opts.deadlineMs < 0)) {
|
|
8002
|
+
throw new TypeError("deadlineMs must be a finite number >= 0");
|
|
8003
|
+
}
|
|
8004
|
+
const meter = new UsageMeter(opts.tokenBudget);
|
|
8005
|
+
const budget = opts.tokenBudget;
|
|
8006
|
+
const deadline = opts.deadlineMs !== void 0 ? Date.now() + Math.max(0, opts.deadlineMs) : Infinity;
|
|
8007
|
+
const jobs = new Set(opts.jobs ?? ["librarian", "heal", "reembed"]);
|
|
8008
|
+
const pending = await this.getPending(opts.entityIds);
|
|
8009
|
+
const states = pending.map((p) => ({
|
|
8010
|
+
p,
|
|
8011
|
+
report: { entityId: p.entityId, librarianPasses: 0, factsWritten: 0, healPasses: 0, reembedded: false },
|
|
8012
|
+
librarian: jobs.has("librarian") && p.pendingEvents > 0,
|
|
8013
|
+
heal: jobs.has("heal") && p.healDue,
|
|
8014
|
+
reembed: jobs.has("reembed") && p.reembedPending
|
|
8015
|
+
}));
|
|
8016
|
+
let stopped = { reason: "complete" };
|
|
8017
|
+
let tooSmall = null;
|
|
8018
|
+
const classify = (job, entityId, requiredEstimate) => {
|
|
8019
|
+
if (budget !== void 0 && requiredEstimate > budget) {
|
|
8020
|
+
tooSmall ?? (tooSmall = { reason: "budget_too_small", job, entityId, requiredEstimate });
|
|
8021
|
+
return "too_small";
|
|
8022
|
+
}
|
|
8023
|
+
return "exhausted";
|
|
8024
|
+
};
|
|
8025
|
+
const exhaustedStop = () => Date.now() >= deadline ? { reason: "deadline" } : { reason: "budget_exhausted" };
|
|
8026
|
+
const markBusy = (st, job) => {
|
|
8027
|
+
var _a;
|
|
8028
|
+
((_a = st.report).busy ?? (_a.busy = [])).push(job);
|
|
8029
|
+
};
|
|
8030
|
+
rounds: for (; ; ) {
|
|
8031
|
+
let progressed = false;
|
|
8032
|
+
for (const st of states) {
|
|
8033
|
+
if (!st.librarian && !st.heal && !st.reembed) continue;
|
|
8034
|
+
if (Date.now() >= deadline) {
|
|
8035
|
+
stopped = { reason: "deadline" };
|
|
8036
|
+
break rounds;
|
|
8037
|
+
}
|
|
8038
|
+
const entityId = st.p.entityId;
|
|
8039
|
+
if (st.librarian) {
|
|
8040
|
+
if (this.jobManager.isBlocked("librarian", entityId)) {
|
|
8041
|
+
st.librarian = false;
|
|
8042
|
+
markBusy(st, "librarian");
|
|
8043
|
+
continue;
|
|
8044
|
+
}
|
|
8045
|
+
this.jobManager.acquireLock("librarian", entityId);
|
|
8046
|
+
let r;
|
|
8047
|
+
try {
|
|
8048
|
+
r = await this.maintenanceService.runLibrarianPass({ entityId, trigger: "call", meter });
|
|
8049
|
+
} finally {
|
|
8050
|
+
this.jobManager.releaseLock("librarian", entityId);
|
|
8051
|
+
}
|
|
8052
|
+
if (r.budgetStop) {
|
|
8053
|
+
if (classify("librarian", entityId, r.budgetStop.requiredEstimate) === "exhausted") {
|
|
8054
|
+
stopped = exhaustedStop();
|
|
8055
|
+
break rounds;
|
|
8056
|
+
}
|
|
8057
|
+
st.librarian = false;
|
|
8058
|
+
continue;
|
|
8059
|
+
}
|
|
8060
|
+
if (!r.processedThrough) {
|
|
8061
|
+
st.librarian = false;
|
|
8062
|
+
continue;
|
|
8063
|
+
}
|
|
8064
|
+
st.report.librarianPasses++;
|
|
8065
|
+
st.report.factsWritten += r.factsWritten;
|
|
8066
|
+
progressed = true;
|
|
8067
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, this.db);
|
|
8068
|
+
if (await this.eventRepo.countAfter(entityId, wm) === 0) st.librarian = false;
|
|
8069
|
+
continue;
|
|
8070
|
+
}
|
|
8071
|
+
if (st.heal) {
|
|
8072
|
+
if (this.jobManager.isBlocked("heal", entityId)) {
|
|
8073
|
+
st.heal = false;
|
|
8074
|
+
markBusy(st, "heal");
|
|
8075
|
+
continue;
|
|
8076
|
+
}
|
|
8077
|
+
this.jobManager.acquireLock("heal", entityId);
|
|
8078
|
+
let h;
|
|
8079
|
+
try {
|
|
8080
|
+
h = await this.maintenanceService.doRunHeal(entityId, { trigger: "call", meter });
|
|
8081
|
+
} finally {
|
|
8082
|
+
this.jobManager.releaseLock("heal", entityId);
|
|
8083
|
+
}
|
|
8084
|
+
if (h.budgetStop && h.scanned === 0) {
|
|
8085
|
+
if (classify("heal", entityId, h.budgetStop.requiredEstimate) === "exhausted") {
|
|
8086
|
+
stopped = exhaustedStop();
|
|
8087
|
+
break rounds;
|
|
8088
|
+
}
|
|
8089
|
+
st.heal = false;
|
|
8090
|
+
continue;
|
|
8091
|
+
}
|
|
8092
|
+
st.report.healPasses++;
|
|
8093
|
+
progressed = true;
|
|
8094
|
+
if (h.remaining === 0) {
|
|
8095
|
+
st.heal = false;
|
|
8096
|
+
await this.metadataRepo.updateCheckpoint(entityId, { heal: await this.eventRepo.count(entityId) }, this.db);
|
|
8097
|
+
}
|
|
8098
|
+
if (h.budgetStop) {
|
|
8099
|
+
stopped = exhaustedStop();
|
|
8100
|
+
break rounds;
|
|
8101
|
+
}
|
|
8102
|
+
continue;
|
|
8103
|
+
}
|
|
8104
|
+
if (st.reembed) {
|
|
8105
|
+
st.reembed = false;
|
|
8106
|
+
try {
|
|
8107
|
+
await this.maintenanceService.runReembed(entityId);
|
|
8108
|
+
st.report.reembedded = true;
|
|
8109
|
+
progressed = true;
|
|
8110
|
+
} catch (err) {
|
|
8111
|
+
if (err instanceof WikiBusyError) markBusy(st, "reembed");
|
|
8112
|
+
else throw err;
|
|
8113
|
+
}
|
|
8114
|
+
}
|
|
8115
|
+
}
|
|
8116
|
+
if (!progressed) break;
|
|
8117
|
+
}
|
|
8118
|
+
if (stopped.reason === "complete" && tooSmall) stopped = tooSmall;
|
|
8119
|
+
return {
|
|
8120
|
+
perEntity: states.map((s) => s.report),
|
|
8121
|
+
tokensUsed: meter.used,
|
|
8122
|
+
estimated: meter.estimated,
|
|
8123
|
+
remaining: budget === void 0 ? null : meter.remaining,
|
|
8124
|
+
stoppedReason: stopped
|
|
8125
|
+
};
|
|
8126
|
+
}
|
|
8127
|
+
};
|
|
6786
8128
|
var MAX_EMBEDDING_BLOB_BYTES = 32 * 1024;
|
|
6787
8129
|
var IMPORT_TITLE_MAX = 500;
|
|
6788
8130
|
var IMPORT_BODY_MAX = 8e3;
|
|
@@ -7021,6 +8363,12 @@ var ImportExportService = class {
|
|
|
7021
8363
|
})()
|
|
7022
8364
|
};
|
|
7023
8365
|
await this.entryRepo.upsertForImport(factObj, tx);
|
|
8366
|
+
await this.entryRepo.setTemporal(fact.id, entityId, {
|
|
8367
|
+
valid_from: fact.valid_from ?? null,
|
|
8368
|
+
valid_to: fact.valid_to ?? null,
|
|
8369
|
+
superseded_by: fact.superseded_by ?? null,
|
|
8370
|
+
superseded_at: fact.superseded_at ?? null
|
|
8371
|
+
}, tx, { outbox: false });
|
|
7024
8372
|
if (blobData != null) {
|
|
7025
8373
|
factsWithPreservedBlob.set(fact.id, blobData);
|
|
7026
8374
|
if (!fact.deleted_at) preservedBlobDims.add(blobData.byteLength / 4);
|
|
@@ -7099,7 +8447,8 @@ var ImportExportService = class {
|
|
|
7099
8447
|
event_type: event.event_type,
|
|
7100
8448
|
summary: event.summary,
|
|
7101
8449
|
related_entry_id: event.related_entry_id ?? null,
|
|
7102
|
-
created_at: event.created_at
|
|
8450
|
+
created_at: event.created_at,
|
|
8451
|
+
occurred_at: event.occurred_at ?? null
|
|
7103
8452
|
},
|
|
7104
8453
|
tx
|
|
7105
8454
|
);
|
|
@@ -7346,6 +8695,21 @@ var EmbeddingService = class {
|
|
|
7346
8695
|
}
|
|
7347
8696
|
}
|
|
7348
8697
|
async tryEmbedFact(fact, ctx) {
|
|
8698
|
+
const r = await this.embedTextForFact(fact, ctx);
|
|
8699
|
+
return r.ok ? this.storeFactVector(fact, r.vector, ctx) : r;
|
|
8700
|
+
}
|
|
8701
|
+
/**
|
|
8702
|
+
* Compute half: build the embed text, call embed(), validate the result,
|
|
8703
|
+
* mark any embed-time failure, and return the validated Float32 vector.
|
|
8704
|
+
*
|
|
8705
|
+
* Failure kinds: `no_provider`, `invalid_vector`, `float32_overflow`,
|
|
8706
|
+
* `provider_error`. All embed-time failures are marked via `markFailure`;
|
|
8707
|
+
* `no_provider` deliberately does not mark (spec §2.4).
|
|
8708
|
+
*
|
|
8709
|
+
* Storage errors are NOT possible here — that is `storeFactVector`'s domain.
|
|
8710
|
+
*/
|
|
8711
|
+
async embedTextForFact(fact, ctx, opts) {
|
|
8712
|
+
const markFailures = opts?.markFailures !== false;
|
|
7349
8713
|
if (typeof this.options.llmProvider.embed !== "function") return { ok: false, kind: "no_provider" };
|
|
7350
8714
|
let tagsStr;
|
|
7351
8715
|
if (Array.isArray(fact.tags)) {
|
|
@@ -7366,8 +8730,8 @@ var EmbeddingService = class {
|
|
|
7366
8730
|
const vector = await this.options.llmProvider.embed(text);
|
|
7367
8731
|
if (vector.length === 0 || !vector.every((v) => typeof v === "number" && isFinite(v))) {
|
|
7368
8732
|
console.warn(`[WikiMemory] embedFact: embed() returned an invalid vector for ${fact.id}; skipping.`);
|
|
7369
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
7370
|
-
await this.markFailure(fact.id, "invalid_vector");
|
|
8733
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "invalid_vector");
|
|
8734
|
+
if (markFailures) await this.markFailure(fact.id, "invalid_vector");
|
|
7371
8735
|
return { ok: false, kind: "invalid_vector" };
|
|
7372
8736
|
}
|
|
7373
8737
|
float32Vector = new Float32Array(vector);
|
|
@@ -7380,16 +8744,24 @@ var EmbeddingService = class {
|
|
|
7380
8744
|
}
|
|
7381
8745
|
if (hasNonFinite) {
|
|
7382
8746
|
console.warn(`[WikiMemory] embedFact: embed() returned values that overflow float32 for ${fact.id}; skipping.`);
|
|
7383
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
7384
|
-
await this.markFailure(fact.id, "float32_overflow");
|
|
8747
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "float32_overflow");
|
|
8748
|
+
if (markFailures) await this.markFailure(fact.id, "float32_overflow");
|
|
7385
8749
|
return { ok: false, kind: "float32_overflow" };
|
|
7386
8750
|
}
|
|
7387
8751
|
} catch (err) {
|
|
7388
8752
|
console.warn(`[WikiMemory] embedFact failed for ${fact.id}:`, err);
|
|
7389
|
-
this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
7390
|
-
await this.markFailure(fact.id, "provider_error");
|
|
8753
|
+
if (markFailures) this.reportEmbed(ctx, fact, "embedding_failed", "embed_threw");
|
|
8754
|
+
if (markFailures) await this.markFailure(fact.id, "provider_error");
|
|
7391
8755
|
return { ok: false, kind: "provider_error" };
|
|
7392
8756
|
}
|
|
8757
|
+
return { ok: true, vector: float32Vector };
|
|
8758
|
+
}
|
|
8759
|
+
/**
|
|
8760
|
+
* Storage half: persist the computed vector to the entry row, then fire the
|
|
8761
|
+
* `onEmbeddingPersisted` hook. Failure kinds: `storage_error` only — embed
|
|
8762
|
+
* failures do not originate here, so they cannot be marked as such (spec §3.2, D3).
|
|
8763
|
+
*/
|
|
8764
|
+
async storeFactVector(fact, float32Vector, ctx) {
|
|
7393
8765
|
try {
|
|
7394
8766
|
await this.storeEmbeddingDimension(float32Vector.length);
|
|
7395
8767
|
const blob = new Uint8Array(float32Vector.buffer);
|
|
@@ -7574,6 +8946,40 @@ function selectWithFloors(sortedRows, floors, maxResults) {
|
|
|
7574
8946
|
return Array.from(selected).sort((a, b) => a - b).map((i) => sortedRows[i]);
|
|
7575
8947
|
}
|
|
7576
8948
|
|
|
8949
|
+
// src/utils/budget.ts
|
|
8950
|
+
function factTokenCost(f) {
|
|
8951
|
+
return estimateTokens(`${f.title} ${f.body} ${f.tags.join(" ")}`);
|
|
8952
|
+
}
|
|
8953
|
+
function packFactsByBudget(items, budget, scores) {
|
|
8954
|
+
const candidates = items.length;
|
|
8955
|
+
if (candidates === 0) return { items: [], tokensUsed: 0, candidates: 0, packed: 0 };
|
|
8956
|
+
let top = items[0];
|
|
8957
|
+
let topCost = factTokenCost(top);
|
|
8958
|
+
if (topCost > budget) {
|
|
8959
|
+
const overhead = estimateTokens(`${top.title} ${top.tags.join(" ")}`);
|
|
8960
|
+
top = { ...top, body: safeSlice(top.body, 0, Math.max(0, (budget - overhead) * 4)) };
|
|
8961
|
+
topCost = factTokenCost(top);
|
|
8962
|
+
}
|
|
8963
|
+
const kept = [{ rank: 0, item: top }];
|
|
8964
|
+
let used = topCost;
|
|
8965
|
+
const rest = items.slice(1).map((item, i) => {
|
|
8966
|
+
const rank = i + 1;
|
|
8967
|
+
const cost = factTokenCost(item);
|
|
8968
|
+
const s = scores?.get(item.id);
|
|
8969
|
+
const value = typeof s === "number" && Number.isFinite(s) && s > 0 ? s : 1 / (rank + 1);
|
|
8970
|
+
return { rank, item, cost, density: value / Math.max(1, cost) };
|
|
8971
|
+
});
|
|
8972
|
+
rest.sort((a, b) => b.density - a.density || a.rank - b.rank);
|
|
8973
|
+
for (const r of rest) {
|
|
8974
|
+
if (used + r.cost <= budget) {
|
|
8975
|
+
kept.push({ rank: r.rank, item: r.item });
|
|
8976
|
+
used += r.cost;
|
|
8977
|
+
}
|
|
8978
|
+
}
|
|
8979
|
+
kept.sort((a, b) => a.rank - b.rank);
|
|
8980
|
+
return { items: kept.map((k) => k.item), tokensUsed: used, candidates, packed: kept.length };
|
|
8981
|
+
}
|
|
8982
|
+
|
|
7577
8983
|
// src/services/RetrievalService.ts
|
|
7578
8984
|
var EMPTY_ID_SET = /* @__PURE__ */ new Set();
|
|
7579
8985
|
var RetrievalService = class {
|
|
@@ -7609,6 +9015,28 @@ var RetrievalService = class {
|
|
|
7609
9015
|
const rawMaxResults = options?.maxResults ?? config?.maxResults ?? config?.maxFtsResults ?? 10;
|
|
7610
9016
|
const maxResults = Number.isFinite(rawMaxResults) ? Math.max(0, Math.trunc(rawMaxResults)) : 10;
|
|
7611
9017
|
const excludeDrafts = (options?.excludeDrafts ?? config?.excludeDrafts ?? false) === true;
|
|
9018
|
+
const rawAsOf = options?.asOf;
|
|
9019
|
+
if (rawAsOf !== void 0 && !(typeof rawAsOf === "number" && Number.isFinite(rawAsOf) && rawAsOf >= 0)) {
|
|
9020
|
+
throw new WikiInvalidReadOptions("asOf", "must be a finite epoch-ms number >= 0");
|
|
9021
|
+
}
|
|
9022
|
+
const tokenBudget = options?.tokenBudget;
|
|
9023
|
+
if (tokenBudget !== void 0 && !(typeof tokenBudget === "number" && Number.isFinite(tokenBudget) && tokenBudget >= 0)) {
|
|
9024
|
+
throw new WikiInvalidReadOptions("tokenBudget", "must be a finite number >= 0");
|
|
9025
|
+
}
|
|
9026
|
+
const applyBudget = (input) => {
|
|
9027
|
+
if (tokenBudget === void 0) return input;
|
|
9028
|
+
const packed = packFactsByBudget(input, Math.trunc(tokenBudget), scoreByFactId);
|
|
9029
|
+
emitDiagnostic(this.options, {
|
|
9030
|
+
code: "read_budget",
|
|
9031
|
+
operation: "read",
|
|
9032
|
+
trigger: "call",
|
|
9033
|
+
entityId: entityIds.length === 1 ? entityIds[0] : entityIds.join(","),
|
|
9034
|
+
detail: { candidates: packed.candidates, packed: packed.packed, tokensUsed: packed.tokensUsed }
|
|
9035
|
+
});
|
|
9036
|
+
return packed.items;
|
|
9037
|
+
};
|
|
9038
|
+
const liveMode = rawAsOf === void 0 ? "current" : "asOf";
|
|
9039
|
+
const liveAt = rawAsOf === void 0 ? Date.now() : Math.trunc(rawAsOf);
|
|
7612
9040
|
const trimmedQuery = query.trim();
|
|
7613
9041
|
const sanitizedTierFloors = trimmedQuery && exposeMetadata ? validateTierFloors(
|
|
7614
9042
|
entityIds,
|
|
@@ -7628,8 +9056,10 @@ var RetrievalService = class {
|
|
|
7628
9056
|
let usedEmbed = false;
|
|
7629
9057
|
const scoredEntityIds = this._filterScoredEntities(entityIds, sanitizedTierWeights, options?.includeZeroWeightEntities);
|
|
7630
9058
|
const draftIds = excludeDrafts && scoredEntityIds.length > 0 ? await this.entryRepo.findDraftIdsByEntityIds(scoredEntityIds) : EMPTY_ID_SET;
|
|
7631
|
-
const
|
|
7632
|
-
const
|
|
9059
|
+
const nonLiveIds = scoredEntityIds.length > 0 ? await this.entryRepo.findNonLiveIdsByEntityIds(scoredEntityIds, liveMode, liveAt) : EMPTY_ID_SET;
|
|
9060
|
+
const excludedIds = draftIds.size === 0 ? nonLiveIds : nonLiveIds.size === 0 ? draftIds : /* @__PURE__ */ new Set([...draftIds, ...nonLiveIds]);
|
|
9061
|
+
const excludedPad = excludedIds.size;
|
|
9062
|
+
const padLimit = (n) => n >= Number.MAX_SAFE_INTEGER ? n : n + excludedPad;
|
|
7633
9063
|
if (scoredEntityIds.length === 0) {
|
|
7634
9064
|
usedEmbed = true;
|
|
7635
9065
|
} else if (!skipEmbed && typeof this.options.llmProvider.embed === "function") {
|
|
@@ -7663,7 +9093,7 @@ var RetrievalService = class {
|
|
|
7663
9093
|
let miniSearchScores;
|
|
7664
9094
|
if (effectivePreFilterLimit !== void 0) {
|
|
7665
9095
|
populateCache = false;
|
|
7666
|
-
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !
|
|
9096
|
+
const preResults = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, Number.MAX_SAFE_INTEGER).filter((r) => !excludedIds.has(r.id));
|
|
7667
9097
|
if (preResults.length === 0) {
|
|
7668
9098
|
candidateRows = null;
|
|
7669
9099
|
} else {
|
|
@@ -7685,9 +9115,9 @@ var RetrievalService = class {
|
|
|
7685
9115
|
}
|
|
7686
9116
|
} else {
|
|
7687
9117
|
if (useRanker) {
|
|
7688
|
-
candidateRows = this.
|
|
9118
|
+
candidateRows = this._withoutIds(await this.entryRepo.findMetadataByEntityIds(scoredEntityIds), excludedIds);
|
|
7689
9119
|
} else {
|
|
7690
|
-
candidateRows = this.
|
|
9120
|
+
candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
|
|
7691
9121
|
}
|
|
7692
9122
|
if (weight !== void 0 && weight < 1) {
|
|
7693
9123
|
miniSearchScores = this.searchService.getMiniSearchScores(trimmedQuery, scoredEntityIds);
|
|
@@ -7972,7 +9402,7 @@ var RetrievalService = class {
|
|
|
7972
9402
|
// read() re-sorts after applying tier weights
|
|
7973
9403
|
});
|
|
7974
9404
|
}
|
|
7975
|
-
if (
|
|
9405
|
+
if (excludedPad > 0) scored = scored.filter((s) => !excludedIds.has(s.id));
|
|
7976
9406
|
if (scored.length > 0) {
|
|
7977
9407
|
scored = scored.map((row) => ({
|
|
7978
9408
|
...row,
|
|
@@ -8036,7 +9466,7 @@ var RetrievalService = class {
|
|
|
8036
9466
|
if (!usedEmbed && scoredEntityIds.length > 0) {
|
|
8037
9467
|
const hasActiveFloors = sanitizedTierFloors !== void 0 && Object.values(sanitizedTierFloors).some((f) => f > 0);
|
|
8038
9468
|
const fallbackOversampledLimit = hasActiveFloors ? Number.MAX_SAFE_INTEGER : padLimit(Math.max(maxResults * 2, maxResults + 50));
|
|
8039
|
-
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !
|
|
9469
|
+
const results = this.searchService.searchKeyword(trimmedQuery, scoredEntityIds, fallbackOversampledLimit).filter((r) => !excludedIds.has(r.id));
|
|
8040
9470
|
const candidates = results.map((r) => ({
|
|
8041
9471
|
id: r.id,
|
|
8042
9472
|
entity_id: r.entity_id,
|
|
@@ -8054,13 +9484,18 @@ var RetrievalService = class {
|
|
|
8054
9484
|
}
|
|
8055
9485
|
}
|
|
8056
9486
|
}
|
|
9487
|
+
facts = applyBudget(facts);
|
|
8057
9488
|
if (facts.length > 0) {
|
|
8058
9489
|
const ids = facts.map((f) => f.id);
|
|
8059
9490
|
const now = Date.now();
|
|
8060
9491
|
await this.entryRepo.trackAccess(ids, now);
|
|
8061
9492
|
}
|
|
8062
9493
|
} else {
|
|
8063
|
-
facts =
|
|
9494
|
+
facts = await this.entryRepo.findRecentByEntityIds(entityIds, maxResults, void 0, {
|
|
9495
|
+
...excludeDrafts ? { excludeDrafts: true } : {},
|
|
9496
|
+
live: { mode: liveMode, t: liveAt }
|
|
9497
|
+
});
|
|
9498
|
+
facts = applyBudget(facts);
|
|
8064
9499
|
}
|
|
8065
9500
|
const eventsLimit = Math.min(10 * entityIds.length, 100);
|
|
8066
9501
|
const [tasks, events] = await Promise.all([
|
|
@@ -8096,8 +9531,8 @@ var RetrievalService = class {
|
|
|
8096
9531
|
_tieBreakSort(items) {
|
|
8097
9532
|
items.sort((a, b) => this._compareScoredRows(a, b));
|
|
8098
9533
|
}
|
|
8099
|
-
|
|
8100
|
-
return
|
|
9534
|
+
_withoutIds(rows, excludeIds) {
|
|
9535
|
+
return excludeIds.size === 0 ? rows : rows.filter((row) => !excludeIds.has(row.id));
|
|
8101
9536
|
}
|
|
8102
9537
|
/**
|
|
8103
9538
|
* Comparator for score + deterministic tie-break fields.
|
|
@@ -8179,6 +9614,18 @@ var WriteService = class {
|
|
|
8179
9614
|
this.metadataRepo = metadataRepo;
|
|
8180
9615
|
this.jobManager = jobManager;
|
|
8181
9616
|
this.maintenanceService = maintenanceService;
|
|
9617
|
+
this.inflight = /* @__PURE__ */ new Set();
|
|
9618
|
+
}
|
|
9619
|
+
track(p) {
|
|
9620
|
+
this.inflight.add(p);
|
|
9621
|
+
p.finally(() => this.inflight.delete(p)).catch(() => {
|
|
9622
|
+
});
|
|
9623
|
+
}
|
|
9624
|
+
/** Await every background job this instance started (spec §6.2). Loops because a finishing job can start another. */
|
|
9625
|
+
async drain() {
|
|
9626
|
+
while (this.inflight.size > 0) {
|
|
9627
|
+
await Promise.allSettled([...this.inflight]);
|
|
9628
|
+
}
|
|
8182
9629
|
}
|
|
8183
9630
|
async write(entityId, event) {
|
|
8184
9631
|
if (typeof entityId !== "string" || entityId.length === 0 || entityId.length > 200 || entityId.includes("\0")) {
|
|
@@ -8205,6 +9652,7 @@ var WriteService = class {
|
|
|
8205
9652
|
}
|
|
8206
9653
|
const id = generateId("evt_");
|
|
8207
9654
|
const now = Date.now();
|
|
9655
|
+
const occurredAt = event.occurred_at == null ? null : assertEpochMs("event.occurred_at", event.occurred_at);
|
|
8208
9656
|
let eventType = event.event_type;
|
|
8209
9657
|
if (!["observation", "decision", "action", "outcome"].includes(eventType)) {
|
|
8210
9658
|
eventType = "observation";
|
|
@@ -8215,8 +9663,14 @@ var WriteService = class {
|
|
|
8215
9663
|
event_type: eventType,
|
|
8216
9664
|
summary,
|
|
8217
9665
|
related_entry_id: relatedEntryId,
|
|
8218
|
-
created_at: now
|
|
9666
|
+
created_at: now,
|
|
9667
|
+
occurred_at: occurredAt
|
|
8219
9668
|
};
|
|
9669
|
+
const deferred = this.options.config?.maintenance === "deferred";
|
|
9670
|
+
if (deferred) {
|
|
9671
|
+
await this.eventRepo.add(newEvent);
|
|
9672
|
+
return;
|
|
9673
|
+
}
|
|
8220
9674
|
let shouldRunLibrarian = false;
|
|
8221
9675
|
let librarianCount = 0;
|
|
8222
9676
|
let prevMemoryCheckpoint = 0;
|
|
@@ -8238,27 +9692,41 @@ var WriteService = class {
|
|
|
8238
9692
|
prevMemoryCheckpoint = memoryCheckpoint;
|
|
8239
9693
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
8240
9694
|
}
|
|
9695
|
+
} else {
|
|
9696
|
+
const tokenThreshold = this.options.config?.autoLibrarianTokenThreshold;
|
|
9697
|
+
if (typeof tokenThreshold === "number" && tokenThreshold > 0 && !this.jobManager.isBlocked("librarian", entityId)) {
|
|
9698
|
+
const wm = await this.metadataRepo.getLibrarianWatermark(entityId, tx);
|
|
9699
|
+
const chars = await this.eventRepo.sumSummaryCharsAfter(entityId, wm, tx);
|
|
9700
|
+
if (Math.ceil(chars / 4) >= tokenThreshold) {
|
|
9701
|
+
shouldRunLibrarian = true;
|
|
9702
|
+
librarianCount = count;
|
|
9703
|
+
prevMemoryCheckpoint = memoryCheckpoint;
|
|
9704
|
+
await this.metadataRepo.updateCheckpoint(entityId, { memory: count }, tx);
|
|
9705
|
+
}
|
|
9706
|
+
}
|
|
8241
9707
|
}
|
|
8242
9708
|
});
|
|
8243
9709
|
if (shouldRunLibrarian) {
|
|
8244
9710
|
try {
|
|
8245
9711
|
this.jobManager.acquireLock("librarian", entityId);
|
|
8246
9712
|
const stage = { job: "librarian" };
|
|
8247
|
-
this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
9713
|
+
const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
8248
9714
|
console.error(err);
|
|
8249
9715
|
this.reportBackgroundFailure(entityId, stage.job);
|
|
8250
9716
|
}).finally(() => {
|
|
8251
9717
|
this.jobManager.releaseLock("librarian", entityId);
|
|
8252
9718
|
});
|
|
9719
|
+
this.track(job);
|
|
8253
9720
|
} catch (e) {
|
|
8254
9721
|
if (!(e instanceof WikiBusyError)) throw e;
|
|
8255
9722
|
await this.metadataRepo.updateCheckpoint(entityId, { memory: prevMemoryCheckpoint }, this.db);
|
|
8256
9723
|
}
|
|
8257
9724
|
} else if (!this.jobManager.isBlocked("librarian", entityId)) {
|
|
8258
|
-
this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
9725
|
+
const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
8259
9726
|
console.error(err);
|
|
8260
9727
|
this.reportBackgroundFailure(entityId, "heal");
|
|
8261
9728
|
});
|
|
9729
|
+
this.track(healJob);
|
|
8262
9730
|
}
|
|
8263
9731
|
}
|
|
8264
9732
|
async runLibrarianThenMaybeHeal(entityId, currentEventCount, prevCheckpoint, stage) {
|
|
@@ -8489,6 +9957,15 @@ var GraphTraversalService = class {
|
|
|
8489
9957
|
const defaultMaxNodes = Number.isFinite(rawConfigDefault) && rawConfigDefault >= 1 ? Math.floor(rawConfigDefault) : fallbackMaxNodes;
|
|
8490
9958
|
const rawMaxNodes = options.maxTraversalNodes ?? defaultMaxNodes;
|
|
8491
9959
|
const maxNodes = Number.isFinite(rawMaxNodes) && rawMaxNodes >= 1 ? Math.floor(rawMaxNodes) : defaultMaxNodes;
|
|
9960
|
+
const rawAsOf = options.asOf;
|
|
9961
|
+
if (rawAsOf !== void 0 && !(typeof rawAsOf === "number" && Number.isFinite(rawAsOf) && rawAsOf >= 0)) {
|
|
9962
|
+
throw new WikiInvalidReadOptions("asOf", "must be a finite epoch-ms number >= 0");
|
|
9963
|
+
}
|
|
9964
|
+
const rawTokenBudget = options.tokenBudget;
|
|
9965
|
+
if (rawTokenBudget !== void 0 && !(typeof rawTokenBudget === "number" && Number.isFinite(rawTokenBudget) && rawTokenBudget >= 0)) {
|
|
9966
|
+
throw new WikiInvalidReadOptions("tokenBudget", "must be a finite number >= 0");
|
|
9967
|
+
}
|
|
9968
|
+
const live = rawAsOf === void 0 ? { mode: "current", t: Date.now() } : { mode: "asOf", t: Math.trunc(rawAsOf) };
|
|
8492
9969
|
const opts = {
|
|
8493
9970
|
maxDepth: Math.max(1, Math.min(options.maxDepth ?? 1, 3)),
|
|
8494
9971
|
direction: options.direction ?? this.config.traversalDirection ?? "both",
|
|
@@ -8496,16 +9973,32 @@ var GraphTraversalService = class {
|
|
|
8496
9973
|
minConfidence: options.minTraversalConfidence ?? this.config.minTraversalConfidence ?? "tentative",
|
|
8497
9974
|
excludeSourceTypes: options.excludeSourceTypes ?? this.config.excludeSourceTypes ?? [],
|
|
8498
9975
|
excludeDrafts: options.excludeDrafts ?? this.config.excludeDrafts ?? false,
|
|
9976
|
+
live,
|
|
8499
9977
|
maxNodes
|
|
8500
9978
|
};
|
|
8501
9979
|
const { nodeIds, edges } = await this.edgeRepo.getNeighborhood(entityId, options.sourceId, opts);
|
|
8502
9980
|
if (nodeIds.length === 0) return { nodes: [], edges: [] };
|
|
9981
|
+
const [anchor] = await this.entryRepo.findByIds([options.sourceId], [entityId]);
|
|
9982
|
+
if (anchor && !isLiveAt(anchor, live.mode, live.t)) return { nodes: [anchor], edges: [] };
|
|
8503
9983
|
const nodes = await this.entryRepo.findByIds(nodeIds, [entityId]);
|
|
8504
9984
|
const hydratedIds = new Set(nodes.map((node) => node.id));
|
|
8505
9985
|
const filteredEdges = edges.filter(
|
|
8506
9986
|
(edge) => hydratedIds.has(edge.source_id) && hydratedIds.has(edge.target_id)
|
|
8507
9987
|
);
|
|
8508
|
-
return { nodes, edges: filteredEdges };
|
|
9988
|
+
if (options.tokenBudget === void 0) return { nodes, edges: filteredEdges };
|
|
9989
|
+
const budget = Math.trunc(options.tokenBudget);
|
|
9990
|
+
const [anchorPacked] = packFactsByBudget([nodes[0]], budget).items;
|
|
9991
|
+
const kept = [anchorPacked];
|
|
9992
|
+
let used = factTokenCost(anchorPacked);
|
|
9993
|
+
for (const node of nodes.slice(1)) {
|
|
9994
|
+
const cost = factTokenCost(node);
|
|
9995
|
+
if (used + cost <= budget) {
|
|
9996
|
+
kept.push(node);
|
|
9997
|
+
used += cost;
|
|
9998
|
+
}
|
|
9999
|
+
}
|
|
10000
|
+
const keptIds = new Set(kept.map((n) => n.id));
|
|
10001
|
+
return { nodes: kept, edges: filteredEdges.filter((e) => keptIds.has(e.source_id) && keptIds.has(e.target_id)) };
|
|
8509
10002
|
}
|
|
8510
10003
|
};
|
|
8511
10004
|
|
|
@@ -8806,6 +10299,7 @@ var WikiMemory = class {
|
|
|
8806
10299
|
this.lintService = new LintService(this.lintRepo, this.ontologyService);
|
|
8807
10300
|
this.embeddingService = new EmbeddingService(this.db, this.options, this.entryRepo, this.metadataRepo);
|
|
8808
10301
|
this.searchService = new SearchService(this.entryRepo);
|
|
10302
|
+
this.temporalService = new TemporalService(this.db, this.options, this.entryRepo, this.searchService, this.embeddingService);
|
|
8809
10303
|
this.jobManager = new JobManager(this.prefix);
|
|
8810
10304
|
this.promptService = new PromptService(options.config?.prompts, resolveGrounding(options.config?.grounding));
|
|
8811
10305
|
this.ingestionService = new IngestionService(
|
|
@@ -8837,6 +10331,15 @@ var WikiMemory = class {
|
|
|
8837
10331
|
this.promptService,
|
|
8838
10332
|
this.ontologyService
|
|
8839
10333
|
);
|
|
10334
|
+
this.maintenanceScheduler = new MaintenanceScheduler(
|
|
10335
|
+
this.db,
|
|
10336
|
+
this.options,
|
|
10337
|
+
this.maintenanceService,
|
|
10338
|
+
this.metadataRepo,
|
|
10339
|
+
this.eventRepo,
|
|
10340
|
+
this.entryRepo,
|
|
10341
|
+
this.jobManager
|
|
10342
|
+
);
|
|
8840
10343
|
this.importExportService = new ImportExportService(
|
|
8841
10344
|
this.db,
|
|
8842
10345
|
this.entryRepo,
|
|
@@ -8887,11 +10390,13 @@ var WikiMemory = class {
|
|
|
8887
10390
|
importExportService: this.importExportService,
|
|
8888
10391
|
ingestionService: this.ingestionService,
|
|
8889
10392
|
maintenanceService: this.maintenanceService,
|
|
10393
|
+
maintenanceScheduler: this.maintenanceScheduler,
|
|
8890
10394
|
retrievalService: this.retrievalService,
|
|
8891
10395
|
searchService: this.searchService,
|
|
8892
10396
|
writeService: this.writeService,
|
|
8893
10397
|
promptService: this.promptService,
|
|
8894
10398
|
graphTraversalService: this.graphTraversalService,
|
|
10399
|
+
temporalService: this.temporalService,
|
|
8895
10400
|
entryRepo: this.entryRepo,
|
|
8896
10401
|
sourceRefIndexRepo: this.sourceRefIndexRepo,
|
|
8897
10402
|
metadataRepo: this.metadataRepo,
|
|
@@ -9109,6 +10614,23 @@ var WikiMemory = class {
|
|
|
9109
10614
|
async write(entityId, event) {
|
|
9110
10615
|
return this.writeService.write(entityId, event);
|
|
9111
10616
|
}
|
|
10617
|
+
/** Await auto-mode background librarian/heal jobs started by write(). No-op in 'deferred' mode. */
|
|
10618
|
+
async drain() {
|
|
10619
|
+
return this.writeService.drain();
|
|
10620
|
+
}
|
|
10621
|
+
/**
|
|
10622
|
+
* Report which entities have pending maintenance work (librarian events
|
|
10623
|
+
* behind the watermark, heal backlog, unembedded facts) without running
|
|
10624
|
+
* anything. Entities with no pending work are omitted; sorted by
|
|
10625
|
+
* `pendingEvents` desc then `entityId` asc.
|
|
10626
|
+
*/
|
|
10627
|
+
async getPendingMaintenance(entityIds) {
|
|
10628
|
+
return this.maintenanceScheduler.getPending(entityIds);
|
|
10629
|
+
}
|
|
10630
|
+
/** Run pending librarian/heal/reembed work under an optional token budget and deadline (spec §6.2). Nothing runs after it resolves. */
|
|
10631
|
+
async runPendingMaintenance(options) {
|
|
10632
|
+
return this.maintenanceScheduler.run(options);
|
|
10633
|
+
}
|
|
9112
10634
|
/**
|
|
9113
10635
|
* @param options.promptOverride - Applies only to this manual call. Does NOT affect
|
|
9114
10636
|
* WriteService-triggered auto-runs. For persistent prompt customization across auto-runs,
|
|
@@ -9182,6 +10704,18 @@ var WikiMemory = class {
|
|
|
9182
10704
|
async forget(entityId, params, opts) {
|
|
9183
10705
|
return this.maintenanceService.forget(entityId, params, opts);
|
|
9184
10706
|
}
|
|
10707
|
+
/**
|
|
10708
|
+
* End `oldId`'s validity and link it to its replacement (spec §4.3). One
|
|
10709
|
+
* transaction. Throws `WikiSupersedeError` for missing, foreign, already
|
|
10710
|
+
* superseded, immutable-document, or cyclic targets.
|
|
10711
|
+
*/
|
|
10712
|
+
async supersede(entityId, oldId, replacement, options) {
|
|
10713
|
+
return this.temporalService.supersede(entityId, oldId, replacement, options);
|
|
10714
|
+
}
|
|
10715
|
+
/** The supersession chain containing `factId`, oldest first. [] when unknown. */
|
|
10716
|
+
async history(entityId, factId) {
|
|
10717
|
+
return this.temporalService.history(entityId, factId);
|
|
10718
|
+
}
|
|
9185
10719
|
/**
|
|
9186
10720
|
* Full and partial ingestion both reject foreign-owned generated fact IDs with
|
|
9187
10721
|
* `WikiGraphNodeOwnershipConflict`. Such collisions are unexpected. Ingestion
|
|
@@ -9500,10 +11034,11 @@ function formatContext(bundle, options) {
|
|
|
9500
11034
|
maxFacts: options?.maxFacts ?? 10,
|
|
9501
11035
|
maxTasks: options?.maxTasks ?? 10,
|
|
9502
11036
|
maxEvents: options?.maxEvents ?? 10,
|
|
9503
|
-
includeConfidence: options?.includeConfidence ?? true,
|
|
9504
|
-
includeTags: options?.includeTags ?? true,
|
|
9505
|
-
includeEntityIds: options?.includeEntityIds ?? false,
|
|
9506
|
-
includeFactScores: options?.includeFactScores ?? false,
|
|
11037
|
+
includeConfidence: !(options?.compact ?? false) && (options?.includeConfidence ?? true),
|
|
11038
|
+
includeTags: !(options?.compact ?? false) && (options?.includeTags ?? true),
|
|
11039
|
+
includeEntityIds: !(options?.compact ?? false) && (options?.includeEntityIds ?? false),
|
|
11040
|
+
includeFactScores: !(options?.compact ?? false) && (options?.includeFactScores ?? false),
|
|
11041
|
+
compact: options?.compact ?? false,
|
|
9507
11042
|
factWeights: {
|
|
9508
11043
|
confidence: options?.factWeights?.confidence ?? 1,
|
|
9509
11044
|
accessCount: options?.factWeights?.accessCount ?? 0.3,
|
|
@@ -9794,7 +11329,12 @@ function factFrontmatterV2(f) {
|
|
|
9794
11329
|
// `verified` key", not from "empty `verified` list").
|
|
9795
11330
|
...f.okf_verified && f.okf_verified.length > 0 ? { verified: f.okf_verified } : {},
|
|
9796
11331
|
...f.okf_sources && f.okf_sources.length > 0 ? { sources: f.okf_sources } : {},
|
|
9797
|
-
...f.okf_usage_window ? { usage_window: f.okf_usage_window } : {}
|
|
11332
|
+
...f.okf_usage_window ? { usage_window: f.okf_usage_window } : {},
|
|
11333
|
+
// Temporal (spec 2026-09-29 §10.9): flat epoch-ms keys, like created_at.
|
|
11334
|
+
...f.valid_from != null ? { valid_from: f.valid_from } : {},
|
|
11335
|
+
...f.valid_to != null ? { valid_to: f.valid_to } : {},
|
|
11336
|
+
...f.superseded_by != null ? { superseded_by: f.superseded_by } : {},
|
|
11337
|
+
...f.superseded_at != null ? { superseded_at: f.superseded_at } : {}
|
|
9798
11338
|
};
|
|
9799
11339
|
}
|
|
9800
11340
|
function taskFrontmatterV2(t) {
|
|
@@ -10122,9 +11662,16 @@ function frontmatterToFact(entityId, id, frontmatter, body, now) {
|
|
|
10122
11662
|
okf_verified: verified,
|
|
10123
11663
|
okf_usage_window: usageWindow,
|
|
10124
11664
|
last_verified_at: lastV?.at ?? null,
|
|
10125
|
-
last_verified_by: lastV?.by ?? null
|
|
11665
|
+
last_verified_by: lastV?.by ?? null,
|
|
11666
|
+
...optionalEpoch("valid_from", frontmatter.valid_from),
|
|
11667
|
+
...optionalEpoch("valid_to", frontmatter.valid_to),
|
|
11668
|
+
...typeof frontmatter.superseded_by === "string" ? { superseded_by: frontmatter.superseded_by } : {},
|
|
11669
|
+
...optionalEpoch("superseded_at", frontmatter.superseded_at)
|
|
10126
11670
|
};
|
|
10127
11671
|
}
|
|
11672
|
+
function optionalEpoch(key, v) {
|
|
11673
|
+
return typeof v === "number" && Number.isFinite(v) && v >= 0 ? { [key]: Math.trunc(v) } : {};
|
|
11674
|
+
}
|
|
10128
11675
|
function frontmatterToTask(entityId, id, frontmatter, now, usesV1StatusRule) {
|
|
10129
11676
|
const created_at = parseFrontmatterTimestamp(frontmatter.created_at, now);
|
|
10130
11677
|
const updated_at = parseFrontmatterTimestamp(
|
|
@@ -10381,6 +11928,7 @@ exports.WikiMemory = WikiMemory;
|
|
|
10381
11928
|
exports.WikiParseError = WikiParseError;
|
|
10382
11929
|
exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
|
|
10383
11930
|
exports.WikiStrictOntologyViolation = WikiStrictOntologyViolation;
|
|
11931
|
+
exports.WikiSupersedeError = WikiSupersedeError;
|
|
10384
11932
|
exports.WikiTransactionError = WikiTransactionError;
|
|
10385
11933
|
exports.chunkText = chunkText;
|
|
10386
11934
|
exports.configureRandomSource = configureRandomSource;
|