@equationalapplications/core-llm-wiki 5.1.1 → 5.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/{chunk-YKXCMOHH.mjs → chunk-6HNIOKT4.mjs} +258 -70
- package/dist/chunk-6HNIOKT4.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +608 -85
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +354 -20
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-DCRNie7k.d.mts → testing-CAk9oFvw.d.mts} +242 -3
- package/dist/{testing-DCRNie7k.d.ts → testing-CAk9oFvw.d.ts} +242 -3
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +309 -67
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-YKXCMOHH.mjs.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -6,6 +6,59 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
|
6
6
|
|
|
7
7
|
var MiniSearch__default = /*#__PURE__*/_interopDefault(MiniSearch);
|
|
8
8
|
|
|
9
|
+
// src/db/sqliteCodes.ts
|
|
10
|
+
var SQLITE_RESULT_CODE_NAMES = {
|
|
11
|
+
1: "SQLITE_ERROR",
|
|
12
|
+
2: "SQLITE_INTERNAL",
|
|
13
|
+
3: "SQLITE_PERM",
|
|
14
|
+
4: "SQLITE_ABORT",
|
|
15
|
+
5: "SQLITE_BUSY",
|
|
16
|
+
6: "SQLITE_LOCKED",
|
|
17
|
+
7: "SQLITE_NOMEM",
|
|
18
|
+
8: "SQLITE_READONLY",
|
|
19
|
+
9: "SQLITE_INTERRUPT",
|
|
20
|
+
10: "SQLITE_IOERR",
|
|
21
|
+
11: "SQLITE_CORRUPT",
|
|
22
|
+
12: "SQLITE_NOTFOUND",
|
|
23
|
+
13: "SQLITE_FULL",
|
|
24
|
+
14: "SQLITE_CANTOPEN",
|
|
25
|
+
15: "SQLITE_PROTOCOL",
|
|
26
|
+
16: "SQLITE_EMPTY",
|
|
27
|
+
17: "SQLITE_SCHEMA",
|
|
28
|
+
18: "SQLITE_TOOBIG",
|
|
29
|
+
19: "SQLITE_CONSTRAINT",
|
|
30
|
+
20: "SQLITE_MISMATCH",
|
|
31
|
+
21: "SQLITE_MISUSE",
|
|
32
|
+
22: "SQLITE_NOLFS",
|
|
33
|
+
23: "SQLITE_AUTH",
|
|
34
|
+
24: "SQLITE_FORMAT",
|
|
35
|
+
25: "SQLITE_RANGE",
|
|
36
|
+
26: "SQLITE_NOTADB",
|
|
37
|
+
27: "SQLITE_NOTICE",
|
|
38
|
+
28: "SQLITE_WARNING",
|
|
39
|
+
// Extended result codes (https://sqlite.org/rescode.html#extended_result_code_list).
|
|
40
|
+
// 2067 = SQLITE_CONSTRAINT_UNIQUE — raised when a UNIQUE index/PRIMARY KEY
|
|
41
|
+
// rejects an INSERT/UPDATE; expo-sqlite reports this as a numeric "Error code N:"
|
|
42
|
+
// prefix and we normalize it back to the symbolic name so callers can branch on
|
|
43
|
+
// the same string the rest of the ecosystem uses.
|
|
44
|
+
2067: "SQLITE_CONSTRAINT_UNIQUE",
|
|
45
|
+
100: "SQLITE_ROW",
|
|
46
|
+
101: "SQLITE_DONE"
|
|
47
|
+
};
|
|
48
|
+
function nameForSqliteResultCode(n) {
|
|
49
|
+
return SQLITE_RESULT_CODE_NAMES[n] ?? `SQLITE_${n}`;
|
|
50
|
+
}
|
|
51
|
+
function extractSqliteCode(err) {
|
|
52
|
+
if (typeof err !== "object" || err === null) return void 0;
|
|
53
|
+
const { code, message } = err;
|
|
54
|
+
if (typeof code === "string" && code.startsWith("SQLITE_")) return code;
|
|
55
|
+
if (typeof message === "string") {
|
|
56
|
+
const m = /^Error code (\d+):/.exec(message);
|
|
57
|
+
if (m) return nameForSqliteResultCode(Number(m[1]));
|
|
58
|
+
}
|
|
59
|
+
return void 0;
|
|
60
|
+
}
|
|
61
|
+
|
|
9
62
|
// src/types.ts
|
|
10
63
|
var WikiBusyError = class extends Error {
|
|
11
64
|
constructor(operation, entityId) {
|
|
@@ -15,6 +68,22 @@ var WikiBusyError = class extends Error {
|
|
|
15
68
|
this.entityId = entityId;
|
|
16
69
|
}
|
|
17
70
|
};
|
|
71
|
+
var WikiDuplicateHashError = class extends Error {
|
|
72
|
+
constructor(params) {
|
|
73
|
+
super(`Duplicate source hash for entity ${params.entityId}; another ref already holds this content`);
|
|
74
|
+
this.name = "WikiDuplicateHashError";
|
|
75
|
+
this.canonical = params.canonical;
|
|
76
|
+
this.sourceHash = params.sourceHash;
|
|
77
|
+
this.entityId = params.entityId;
|
|
78
|
+
}
|
|
79
|
+
};
|
|
80
|
+
var WikiTransactionError = class extends Error {
|
|
81
|
+
constructor(message, options) {
|
|
82
|
+
super(message, options);
|
|
83
|
+
this.name = "WikiTransactionError";
|
|
84
|
+
this.sqliteErrorCode = extractSqliteCode(options.cause);
|
|
85
|
+
}
|
|
86
|
+
};
|
|
18
87
|
var PrunePartialFailureError = class extends Error {
|
|
19
88
|
constructor(deleted, failedAt, remaining, cause, deletedTasks = 0, deletedEvents = 0) {
|
|
20
89
|
super(`Prune partially failed: deleted ${deleted}, failed at ${failedAt}, ${remaining} remaining`);
|
|
@@ -574,6 +643,15 @@ var ImportExportService = class {
|
|
|
574
643
|
const safeTitle = clip(String(fact.title ?? ""), IMPORT_TITLE_MAX);
|
|
575
644
|
const safeBody = clip(String(fact.body ?? ""), IMPORT_BODY_MAX);
|
|
576
645
|
clippedTextByFactId.set(fact.id, { title: safeTitle, body: safeBody });
|
|
646
|
+
let normalizedSourceRef = null;
|
|
647
|
+
if (fact.source_ref !== null && fact.source_ref !== void 0) {
|
|
648
|
+
normalizedSourceRef = normalizeSourceRef(fact.source_ref);
|
|
649
|
+
if (normalizedSourceRef === null) {
|
|
650
|
+
throw new Error(
|
|
651
|
+
`importDump: invalid source_ref ${JSON.stringify(fact.source_ref)} for entity "${entityId}" fact "${fact.id}" (must normalize to a non-empty string; see ingestDocument's sourceRef validation)`
|
|
652
|
+
);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
577
655
|
const factObj = {
|
|
578
656
|
id: fact.id,
|
|
579
657
|
entity_id: entityId,
|
|
@@ -583,7 +661,7 @@ var ImportExportService = class {
|
|
|
583
661
|
confidence: fact.confidence,
|
|
584
662
|
source_type: sourceType,
|
|
585
663
|
source_hash: fact.source_hash,
|
|
586
|
-
source_ref:
|
|
664
|
+
source_ref: normalizedSourceRef,
|
|
587
665
|
created_at: fact.created_at,
|
|
588
666
|
updated_at: safeUpdatedAt,
|
|
589
667
|
last_accessed_at: fact.last_accessed_at,
|
|
@@ -975,18 +1053,19 @@ var DEFAULT_CHUNK_OVERLAP = 400;
|
|
|
975
1053
|
|
|
976
1054
|
// src/services/IngestionService.ts
|
|
977
1055
|
var IngestionService = class {
|
|
978
|
-
constructor(db, prefix, options, entryRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1056
|
+
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
979
1057
|
this.db = db;
|
|
980
1058
|
this.prefix = prefix;
|
|
981
1059
|
this.options = options;
|
|
982
1060
|
this.entryRepo = entryRepo;
|
|
1061
|
+
this.sourceRefIndexRepo = sourceRefIndexRepo;
|
|
983
1062
|
this.searchService = searchService;
|
|
984
1063
|
this.jobManager = jobManager;
|
|
985
1064
|
this.embeddingService = embeddingService;
|
|
986
1065
|
this.ontologyService = ontologyService;
|
|
987
1066
|
this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
|
|
988
1067
|
}
|
|
989
|
-
async ingestDocument(entityId, params) {
|
|
1068
|
+
async ingestDocument(entityId, params, opts) {
|
|
990
1069
|
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
991
1070
|
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
992
1071
|
const sourceHash = normalizeSourceHash(params.sourceHash);
|
|
@@ -1002,8 +1081,18 @@ var IngestionService = class {
|
|
|
1002
1081
|
if (typeof params.documentChunk !== "string") {
|
|
1003
1082
|
throw new Error(`documentChunk must be a string, received ${typeof params.documentChunk}`);
|
|
1004
1083
|
}
|
|
1005
|
-
this.jobManager.
|
|
1084
|
+
const releaseIngestLocks = await this.jobManager.acquireIngestLocks(entityId, sourceRef, sourceHash);
|
|
1006
1085
|
try {
|
|
1086
|
+
const onDuplicateHash = opts?.onDuplicateHash ?? "ingest";
|
|
1087
|
+
if (onDuplicateHash !== "ingest") {
|
|
1088
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
|
|
1089
|
+
if (canonical !== null && canonical !== sourceRef) {
|
|
1090
|
+
if (onDuplicateHash === "throw") {
|
|
1091
|
+
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
1092
|
+
}
|
|
1093
|
+
return { truncated: false, chunks: 0, duplicateOf: canonical };
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1007
1096
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
1008
1097
|
if (chunks.length === 0) return { truncated: false, chunks: 0 };
|
|
1009
1098
|
const chunkResults = await withConcurrency(
|
|
@@ -1039,68 +1128,81 @@ var IngestionService = class {
|
|
|
1039
1128
|
const now = Date.now();
|
|
1040
1129
|
const insertedFacts = [];
|
|
1041
1130
|
const deletedSourceFactIds = [];
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
1057
|
-
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
1058
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
1059
|
-
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
1060
|
-
mode = ontologyState.mode;
|
|
1131
|
+
try {
|
|
1132
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
1133
|
+
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
1134
|
+
await this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, null);
|
|
1135
|
+
await this.sourceRefIndexRepo.softDeleteByEntityAndSourceRef(entityId, sourceRef, tx);
|
|
1136
|
+
await this.sourceRefIndexRepo.upsert(entityId, sourceHash, sourceRef, tx);
|
|
1137
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
1138
|
+
const pendingEdges = [];
|
|
1139
|
+
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx);
|
|
1140
|
+
for (const existing of existingFacts) {
|
|
1141
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
1142
|
+
id: existing.id,
|
|
1143
|
+
okf_type: existing.okf_type ?? null
|
|
1144
|
+
});
|
|
1061
1145
|
}
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1146
|
+
let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
1147
|
+
let { mode, manifest } = ontologyState;
|
|
1148
|
+
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
1149
|
+
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
1150
|
+
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
1151
|
+
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
1152
|
+
mode = ontologyState.mode;
|
|
1153
|
+
}
|
|
1154
|
+
for (const fact of facts) {
|
|
1155
|
+
const ontologyFact = fact;
|
|
1156
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest) ?? { okf_type: null, edges: [] };
|
|
1157
|
+
const id = generateId("fact_");
|
|
1158
|
+
const wikiFact = {
|
|
1159
|
+
id,
|
|
1160
|
+
entity_id: entityId,
|
|
1161
|
+
title: fact.title,
|
|
1162
|
+
body: fact.body,
|
|
1163
|
+
tags: fact.tags,
|
|
1164
|
+
confidence: fact.confidence,
|
|
1165
|
+
source_type: "immutable_document",
|
|
1166
|
+
source_hash: sourceHash,
|
|
1167
|
+
source_ref: sourceRef,
|
|
1168
|
+
created_at: now,
|
|
1169
|
+
updated_at: now,
|
|
1170
|
+
last_accessed_at: null,
|
|
1171
|
+
access_count: 0,
|
|
1172
|
+
deleted_at: null,
|
|
1173
|
+
okf_type: normalized.okf_type
|
|
1174
|
+
};
|
|
1175
|
+
await this.entryRepo.upsert(wikiFact, tx);
|
|
1176
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
1177
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
1178
|
+
if (normalized.edges.length > 0) {
|
|
1179
|
+
pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
1180
|
+
}
|
|
1088
1181
|
}
|
|
1089
1182
|
}
|
|
1183
|
+
for (const item of pendingEdges) {
|
|
1184
|
+
await this.ontologyService?.resolveAndPersistEdges(
|
|
1185
|
+
entityId,
|
|
1186
|
+
item.sourceId,
|
|
1187
|
+
item.sourceType,
|
|
1188
|
+
item.edges ?? [],
|
|
1189
|
+
manifest,
|
|
1190
|
+
titleIndex,
|
|
1191
|
+
tx,
|
|
1192
|
+
now
|
|
1193
|
+
);
|
|
1194
|
+
}
|
|
1195
|
+
});
|
|
1196
|
+
} catch (err) {
|
|
1197
|
+
const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
|
|
1198
|
+
if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
|
|
1199
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
|
|
1200
|
+
if (canonical === null) throw err;
|
|
1201
|
+
if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
|
|
1202
|
+
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
1090
1203
|
}
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
entityId,
|
|
1094
|
-
item.sourceId,
|
|
1095
|
-
item.sourceType,
|
|
1096
|
-
item.edges ?? [],
|
|
1097
|
-
manifest,
|
|
1098
|
-
titleIndex,
|
|
1099
|
-
tx,
|
|
1100
|
-
now
|
|
1101
|
-
);
|
|
1102
|
-
}
|
|
1103
|
-
});
|
|
1204
|
+
return { truncated: false, chunks: 0, duplicateOf: canonical };
|
|
1205
|
+
}
|
|
1104
1206
|
await this.searchService.sync(entityId);
|
|
1105
1207
|
const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
|
|
1106
1208
|
for (const factId of uniqueDeletedSourceFactIds) {
|
|
@@ -1116,7 +1218,7 @@ var IngestionService = class {
|
|
|
1116
1218
|
this.searchService.evictCache(entityId);
|
|
1117
1219
|
return { truncated, chunks: chunks.length };
|
|
1118
1220
|
} finally {
|
|
1119
|
-
|
|
1221
|
+
releaseIngestLocks();
|
|
1120
1222
|
}
|
|
1121
1223
|
}
|
|
1122
1224
|
};
|
|
@@ -1268,11 +1370,12 @@ var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
|
1268
1370
|
var HEAL_BATCH_SIZE = 25;
|
|
1269
1371
|
var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
1270
1372
|
var MaintenanceService = class {
|
|
1271
|
-
constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1373
|
+
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1272
1374
|
this.db = db;
|
|
1273
1375
|
this.prefix = prefix;
|
|
1274
1376
|
this.options = options;
|
|
1275
1377
|
this.entryRepo = entryRepo;
|
|
1378
|
+
this.sourceRefIndexRepo = sourceRefIndexRepo;
|
|
1276
1379
|
this.taskRepo = taskRepo;
|
|
1277
1380
|
this.eventRepo = eventRepo;
|
|
1278
1381
|
this.metadataRepo = metadataRepo;
|
|
@@ -1431,10 +1534,13 @@ var MaintenanceService = class {
|
|
|
1431
1534
|
this.jobManager.releaseLock(op, entityId ?? "*");
|
|
1432
1535
|
}
|
|
1433
1536
|
}
|
|
1434
|
-
async forget(entityId, params) {
|
|
1537
|
+
async forget(entityId, params, opts) {
|
|
1435
1538
|
if (params.clearAll && (params.entryId !== void 0 || params.taskId !== void 0 || params.sourceRef !== void 0 || params.sourceHash !== void 0)) {
|
|
1436
1539
|
throw new Error("forget() clearAll is mutually exclusive with entryId, taskId, sourceRef, and sourceHash");
|
|
1437
1540
|
}
|
|
1541
|
+
if (opts?.dryRun === true) {
|
|
1542
|
+
return this.forgetDryRun(entityId, params);
|
|
1543
|
+
}
|
|
1438
1544
|
this.jobManager.acquireLock("forget", entityId);
|
|
1439
1545
|
try {
|
|
1440
1546
|
const now = Date.now();
|
|
@@ -1468,10 +1574,12 @@ var MaintenanceService = class {
|
|
|
1468
1574
|
const entryPromise = params.entryId ? this.entryRepo.softDelete(params.entryId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
1469
1575
|
const taskDeletedPromise = params.taskId ? this.taskRepo.softDeleteById(params.taskId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
1470
1576
|
const refPromise = sourceRef || sourceHash ? this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, sourceHash) : null;
|
|
1577
|
+
const sourceRefIndexPromise = sourceRef ? this.sourceRefIndexRepo.softDeleteByEntityAndSourceRef(entityId, sourceRef, tx).then(() => void 0) : null;
|
|
1471
1578
|
const [entryResult, taskResult, refResult] = await Promise.all([
|
|
1472
1579
|
entryPromise ?? Promise.resolve(false),
|
|
1473
1580
|
taskDeletedPromise ?? Promise.resolve(false),
|
|
1474
|
-
refPromise ?? Promise.resolve(0)
|
|
1581
|
+
refPromise ?? Promise.resolve(0),
|
|
1582
|
+
sourceRefIndexPromise ?? Promise.resolve(void 0)
|
|
1475
1583
|
]);
|
|
1476
1584
|
if (entryResult) deletedEntries++;
|
|
1477
1585
|
if (taskResult) deletedTasks++;
|
|
@@ -1495,11 +1603,42 @@ var MaintenanceService = class {
|
|
|
1495
1603
|
throw new Error(`forget(${entityId}/${factId}) failed: ANN cleanup hook rejected`, { cause: this._sanitizeRankerError(hookErr) });
|
|
1496
1604
|
}
|
|
1497
1605
|
}
|
|
1498
|
-
return { deleted: { entries: deletedEntries, tasks: deletedTasks } };
|
|
1606
|
+
return params.clearAll ? { deleted: { entries: deletedEntries, tasks: deletedTasks }, metadataReset: true } : { deleted: { entries: deletedEntries, tasks: deletedTasks } };
|
|
1499
1607
|
} finally {
|
|
1500
1608
|
this.jobManager.releaseLock("forget", entityId);
|
|
1501
1609
|
}
|
|
1502
1610
|
}
|
|
1611
|
+
/**
|
|
1612
|
+
* Read-only dry-run preview of {@link forget}. Returns the same shape as a
|
|
1613
|
+
* real call would, without acquiring the lock, opening a transaction, staging
|
|
1614
|
+
* outbox events, or firing embedding hooks. Counts may be off-by-N during a
|
|
1615
|
+
* concurrent real forget — this is accepted, not considered a bug.
|
|
1616
|
+
*
|
|
1617
|
+
* `metadataReset` mirrors the real call: only `clearAll: true` returns true,
|
|
1618
|
+
* because only `clearAll` actually resets the metadata checkpoint.
|
|
1619
|
+
* `standard` (sourceRef/sourceHash) returns no metadataReset field, just like
|
|
1620
|
+
* the real call. `entryId`/`taskId` selectors are rejected with a clear error
|
|
1621
|
+
* (the spec keeps dry-run narrow).
|
|
1622
|
+
*/
|
|
1623
|
+
async forgetDryRun(entityId, params) {
|
|
1624
|
+
if (params.entryId !== void 0 || params.taskId !== void 0) {
|
|
1625
|
+
throw new Error("forget({ dryRun: true }) does not support entryId/taskId selectors; use sourceRef/sourceHash or clearAll");
|
|
1626
|
+
}
|
|
1627
|
+
if (params.clearAll) {
|
|
1628
|
+
const entries2 = await this.entryRepo.countLiveByEntityId(entityId);
|
|
1629
|
+
const tasks = await this.taskRepo.countLiveByEntityId(entityId);
|
|
1630
|
+
return { deleted: { entries: entries2, tasks }, metadataReset: true };
|
|
1631
|
+
}
|
|
1632
|
+
const sourceRef = params.sourceRef !== void 0 ? normalizeSourceRef(params.sourceRef) : null;
|
|
1633
|
+
if (params.sourceRef !== void 0 && !sourceRef) throw new Error("Invalid sourceRef");
|
|
1634
|
+
const sourceHash = params.sourceHash !== void 0 ? normalizeSourceHash(params.sourceHash) : null;
|
|
1635
|
+
if (params.sourceHash !== void 0 && !sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
1636
|
+
if (sourceRef === null && sourceHash === null) {
|
|
1637
|
+
return { deleted: { entries: 0, tasks: 0 } };
|
|
1638
|
+
}
|
|
1639
|
+
const entries = await this.entryRepo.countLiveBySource(entityId, sourceRef, sourceHash);
|
|
1640
|
+
return { deleted: { entries, tasks: 0 } };
|
|
1641
|
+
}
|
|
1503
1642
|
/** Core librarian pass (locks handled by {@link runLibrarian}). Package-internal orchestration hook. */
|
|
1504
1643
|
async doRunLibrarian(entityId, promptOverride) {
|
|
1505
1644
|
const events = await this.eventRepo.getRecent(entityId, 50);
|
|
@@ -2816,6 +2955,19 @@ var JobManager = class {
|
|
|
2816
2955
|
this.prefix = prefix;
|
|
2817
2956
|
this.activeMaintenanceJobs = /* @__PURE__ */ new Set();
|
|
2818
2957
|
this.activeIngestJobs = /* @__PURE__ */ new Map();
|
|
2958
|
+
/**
|
|
2959
|
+
* Per-(entityId, sourceHash) promise-chain lock. Serializes ingest races
|
|
2960
|
+
* at the application level (the v9 partial UNIQUE index enforces it at
|
|
2961
|
+
* the DB level as defense-in-depth). Two callers holding the same hash
|
|
2962
|
+
* see FIFO order; different hashes never block one another.
|
|
2963
|
+
*
|
|
2964
|
+
* Keyed `${entityId}\0${sourceHash}` so distinct (entityId, sourceHash)
|
|
2965
|
+
* pairs can never share a lock. A raw concatenation would let
|
|
2966
|
+
* ("ab", "c…") collide with ("a", "bc…"); the NUL separator makes that
|
|
2967
|
+
* impossible because neither component can contain NUL (entityIds are
|
|
2968
|
+
* user-supplied identifiers; sourceHashes are hex digests).
|
|
2969
|
+
*/
|
|
2970
|
+
this.hashLocks = /* @__PURE__ */ new Map();
|
|
2819
2971
|
this.statusSubscribers = /* @__PURE__ */ new Map();
|
|
2820
2972
|
/**
|
|
2821
2973
|
* Lookup table for acquireLock/releaseLock's dynamic-dispatch branch.
|
|
@@ -2833,6 +2985,51 @@ var JobManager = class {
|
|
|
2833
2985
|
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
2834
2986
|
};
|
|
2835
2987
|
}
|
|
2988
|
+
_hashLockKey(entityId, sourceHash) {
|
|
2989
|
+
return `${entityId}\0${sourceHash}`;
|
|
2990
|
+
}
|
|
2991
|
+
/**
|
|
2992
|
+
* Acquire a promise-chain lock scoped to one `(entityId, sourceHash)` pair.
|
|
2993
|
+
* Returns a zero-argument release closure that removes the current tail
|
|
2994
|
+
* from the chain. Releasing is idempotent so a holder that double-fires
|
|
2995
|
+
* (e.g. on a finally that races an explicit release) cannot unblock
|
|
2996
|
+
* the next holder prematurely.
|
|
2997
|
+
*
|
|
2998
|
+
* Documented ordering: ingest acquires the hash lock FIRST, then the
|
|
2999
|
+
* synchronous sourceRef ingest lock, and the release closure unwinds in
|
|
3000
|
+
* the opposite order (sourceRef first, hash second). Hash-then-sourceRef
|
|
3001
|
+
* is required because the v9 partial UNIQUE index conflicts on the
|
|
3002
|
+
* (entity_id, source_hash) pair regardless of source_ref, so two callers
|
|
3003
|
+
* with DIFFERENT source_refs racing the same hash are the exact case the
|
|
3004
|
+
* TOCTOU race fix needs to serialize.
|
|
3005
|
+
*
|
|
3006
|
+
* Mechanics: each holder owns a private `released` promise that resolves
|
|
3007
|
+
* only when THIS holder calls release(). The map stores the holder's
|
|
3008
|
+
* `released` promise as the "tail" — the NEXT caller awaits that tail
|
|
3009
|
+
* via `previous = this.hashLocks.get(key)`. The current caller gets
|
|
3010
|
+
* their unique release function once `previous` (the prior holder's
|
|
3011
|
+
* release signal) settles, so each holder is guaranteed a UNIQUE release
|
|
3012
|
+
* closure bound to their own `released` promise.
|
|
3013
|
+
*/
|
|
3014
|
+
acquireHashLock(entityId, sourceHash) {
|
|
3015
|
+
const key = this._hashLockKey(entityId, sourceHash);
|
|
3016
|
+
const previous = this.hashLocks.get(key) ?? Promise.resolve();
|
|
3017
|
+
let resolveReleased;
|
|
3018
|
+
const released = new Promise((resolve) => {
|
|
3019
|
+
resolveReleased = resolve;
|
|
3020
|
+
});
|
|
3021
|
+
let fired = false;
|
|
3022
|
+
const release = () => {
|
|
3023
|
+
if (fired) return;
|
|
3024
|
+
fired = true;
|
|
3025
|
+
if (this.hashLocks.get(key) === released) {
|
|
3026
|
+
this.hashLocks.delete(key);
|
|
3027
|
+
}
|
|
3028
|
+
resolveReleased();
|
|
3029
|
+
};
|
|
3030
|
+
this.hashLocks.set(key, released);
|
|
3031
|
+
return previous.then(() => release);
|
|
3032
|
+
}
|
|
2836
3033
|
_pruneKey(entityId) {
|
|
2837
3034
|
return `${this.prefix}:${entityId}:prune`;
|
|
2838
3035
|
}
|
|
@@ -3018,6 +3215,19 @@ var JobManager = class {
|
|
|
3018
3215
|
return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
3019
3216
|
case "prune":
|
|
3020
3217
|
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
3218
|
+
case "reembed":
|
|
3219
|
+
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey()) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
3220
|
+
case "global_reembed":
|
|
3221
|
+
return this.activeMaintenanceJobs.has(this._globalReembedKey()) || this._isAnyMaintenanceActiveWithSuffix(":reembed") || this._isAnyMaintenanceActiveWithSuffix(":prune") || this._isAnyMaintenanceActiveWithSuffix(":librarian") || this._isAnyMaintenanceActiveWithSuffix(":heal") || this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill") || this.activeIngestJobs.size > 0 || this._isAnyMaintenanceActiveWithSuffix(":import") || this._isAnyMaintenanceActiveWithSuffix(":forget");
|
|
3222
|
+
case "import":
|
|
3223
|
+
case "forget": {
|
|
3224
|
+
const selfKey = operation === "import" ? this._importKey(entityId) : this._forgetKey(entityId);
|
|
3225
|
+
return this.activeMaintenanceJobs.has(selfKey) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
3226
|
+
}
|
|
3227
|
+
case "global_import":
|
|
3228
|
+
return this.activeMaintenanceJobs.has(this._globalImportKey());
|
|
3229
|
+
case "ingest":
|
|
3230
|
+
return this._hasIngestJob(entityId) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
3021
3231
|
default:
|
|
3022
3232
|
return false;
|
|
3023
3233
|
}
|
|
@@ -3033,6 +3243,38 @@ var JobManager = class {
|
|
|
3033
3243
|
this._notifyStatusSubscribers(entityId);
|
|
3034
3244
|
return true;
|
|
3035
3245
|
}
|
|
3246
|
+
/**
|
|
3247
|
+
* Centralized ingest lock acquisition. Acquires the hash lock FIRST
|
|
3248
|
+
* (FIFO across callers racing the same hash), then the synchronous
|
|
3249
|
+
* sourceRef ingest lock (rejects cross-entity conflicts and busy
|
|
3250
|
+
* operations). Returns a single zero-argument release closure that
|
|
3251
|
+
* unwinds in the opposite order — sourceRef first, hash second — so
|
|
3252
|
+
* the sourceRef lock is freed the instant its work is done and only
|
|
3253
|
+
* the hash lock keeps serializing the duplicate-content race window.
|
|
3254
|
+
*
|
|
3255
|
+
* On a sourceRef conflict (WikiBusyError thrown synchronously by the
|
|
3256
|
+
* underlying acquireLock) the hash lock is released BEFORE the error
|
|
3257
|
+
* propagates, so the next caller in the hash chain is not held up by
|
|
3258
|
+
* a request that never reached the DB.
|
|
3259
|
+
*/
|
|
3260
|
+
async acquireIngestLocks(entityId, sourceRef, sourceHash) {
|
|
3261
|
+
const releaseHash = await this.acquireHashLock(entityId, sourceHash);
|
|
3262
|
+
let sourceLockAcquired = false;
|
|
3263
|
+
try {
|
|
3264
|
+
this.acquireLock("ingest", entityId, sourceRef);
|
|
3265
|
+
sourceLockAcquired = true;
|
|
3266
|
+
} catch (err) {
|
|
3267
|
+
releaseHash();
|
|
3268
|
+
throw err;
|
|
3269
|
+
}
|
|
3270
|
+
return () => {
|
|
3271
|
+
try {
|
|
3272
|
+
if (sourceLockAcquired) this.releaseLock("ingest", entityId, sourceRef);
|
|
3273
|
+
} finally {
|
|
3274
|
+
releaseHash();
|
|
3275
|
+
}
|
|
3276
|
+
};
|
|
3277
|
+
}
|
|
3036
3278
|
/**
|
|
3037
3279
|
* Validates then acquires global + per-entity import locks atomically.
|
|
3038
3280
|
* Validates all entities before acquiring any lock (same as current importDump semantics).
|