@equationalapplications/core-llm-wiki 5.2.0 → 5.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/dist/{chunk-J3N3WRK7.mjs → chunk-KIJINNYU.mjs} +367 -84
- package/dist/chunk-KIJINNYU.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +664 -103
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +300 -24
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-DyXISRwS.d.mts → testing-zf3Jp_RL.d.mts} +336 -29
- package/dist/{testing-DyXISRwS.d.ts → testing-zf3Jp_RL.d.ts} +336 -29
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +378 -74
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-J3N3WRK7.mjs.map +0 -1
package/dist/testing.js
CHANGED
|
@@ -6,6 +6,59 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
|
6
6
|
|
|
7
7
|
var MiniSearch__default = /*#__PURE__*/_interopDefault(MiniSearch);
|
|
8
8
|
|
|
9
|
+
// src/db/sqliteCodes.ts
|
|
10
|
+
var SQLITE_RESULT_CODE_NAMES = {
|
|
11
|
+
1: "SQLITE_ERROR",
|
|
12
|
+
2: "SQLITE_INTERNAL",
|
|
13
|
+
3: "SQLITE_PERM",
|
|
14
|
+
4: "SQLITE_ABORT",
|
|
15
|
+
5: "SQLITE_BUSY",
|
|
16
|
+
6: "SQLITE_LOCKED",
|
|
17
|
+
7: "SQLITE_NOMEM",
|
|
18
|
+
8: "SQLITE_READONLY",
|
|
19
|
+
9: "SQLITE_INTERRUPT",
|
|
20
|
+
10: "SQLITE_IOERR",
|
|
21
|
+
11: "SQLITE_CORRUPT",
|
|
22
|
+
12: "SQLITE_NOTFOUND",
|
|
23
|
+
13: "SQLITE_FULL",
|
|
24
|
+
14: "SQLITE_CANTOPEN",
|
|
25
|
+
15: "SQLITE_PROTOCOL",
|
|
26
|
+
16: "SQLITE_EMPTY",
|
|
27
|
+
17: "SQLITE_SCHEMA",
|
|
28
|
+
18: "SQLITE_TOOBIG",
|
|
29
|
+
19: "SQLITE_CONSTRAINT",
|
|
30
|
+
20: "SQLITE_MISMATCH",
|
|
31
|
+
21: "SQLITE_MISUSE",
|
|
32
|
+
22: "SQLITE_NOLFS",
|
|
33
|
+
23: "SQLITE_AUTH",
|
|
34
|
+
24: "SQLITE_FORMAT",
|
|
35
|
+
25: "SQLITE_RANGE",
|
|
36
|
+
26: "SQLITE_NOTADB",
|
|
37
|
+
27: "SQLITE_NOTICE",
|
|
38
|
+
28: "SQLITE_WARNING",
|
|
39
|
+
// Extended result codes (https://sqlite.org/rescode.html#extended_result_code_list).
|
|
40
|
+
// 2067 = SQLITE_CONSTRAINT_UNIQUE — raised when a UNIQUE index/PRIMARY KEY
|
|
41
|
+
// rejects an INSERT/UPDATE; expo-sqlite reports this as a numeric "Error code N:"
|
|
42
|
+
// prefix and we normalize it back to the symbolic name so callers can branch on
|
|
43
|
+
// the same string the rest of the ecosystem uses.
|
|
44
|
+
2067: "SQLITE_CONSTRAINT_UNIQUE",
|
|
45
|
+
100: "SQLITE_ROW",
|
|
46
|
+
101: "SQLITE_DONE"
|
|
47
|
+
};
|
|
48
|
+
function nameForSqliteResultCode(n) {
|
|
49
|
+
return SQLITE_RESULT_CODE_NAMES[n] ?? `SQLITE_${n}`;
|
|
50
|
+
}
|
|
51
|
+
function extractSqliteCode(err) {
|
|
52
|
+
if (typeof err !== "object" || err === null) return void 0;
|
|
53
|
+
const { code, message } = err;
|
|
54
|
+
if (typeof code === "string" && code.startsWith("SQLITE_")) return code;
|
|
55
|
+
if (typeof message === "string") {
|
|
56
|
+
const m = /^Error code (\d+):/.exec(message);
|
|
57
|
+
if (m) return nameForSqliteResultCode(Number(m[1]));
|
|
58
|
+
}
|
|
59
|
+
return void 0;
|
|
60
|
+
}
|
|
61
|
+
|
|
9
62
|
// src/types.ts
|
|
10
63
|
var WikiBusyError = class extends Error {
|
|
11
64
|
constructor(operation, entityId) {
|
|
@@ -24,6 +77,25 @@ var WikiDuplicateHashError = class extends Error {
|
|
|
24
77
|
this.entityId = params.entityId;
|
|
25
78
|
}
|
|
26
79
|
};
|
|
80
|
+
var WikiStrictOntologyViolation = class extends Error {
|
|
81
|
+
constructor(entityId, kind, type) {
|
|
82
|
+
super(
|
|
83
|
+
`Out-of-manifest ${kind} type "${type}" for entity "${entityId}" under strict mode.`
|
|
84
|
+
);
|
|
85
|
+
this.code = "WIKI_STRICT_ONTOLOGY_VIOLATION";
|
|
86
|
+
this.entityId = entityId;
|
|
87
|
+
this.kind = kind;
|
|
88
|
+
this.type = type;
|
|
89
|
+
this.name = "WikiStrictOntologyViolation";
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
var WikiTransactionError = class extends Error {
|
|
93
|
+
constructor(message, options) {
|
|
94
|
+
super(message, options);
|
|
95
|
+
this.name = "WikiTransactionError";
|
|
96
|
+
this.sqliteErrorCode = extractSqliteCode(options.cause);
|
|
97
|
+
}
|
|
98
|
+
};
|
|
27
99
|
var PrunePartialFailureError = class extends Error {
|
|
28
100
|
constructor(deleted, failedAt, remaining, cause, deletedTasks = 0, deletedEvents = 0) {
|
|
29
101
|
super(`Prune partially failed: deleted ${deleted}, failed at ${failedAt}, ${remaining} remaining`);
|
|
@@ -993,11 +1065,14 @@ var DEFAULT_CHUNK_OVERLAP = 400;
|
|
|
993
1065
|
|
|
994
1066
|
// src/services/IngestionService.ts
|
|
995
1067
|
var IngestionService = class {
|
|
996
|
-
constructor(db, prefix, options, entryRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1068
|
+
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
997
1069
|
this.db = db;
|
|
998
1070
|
this.prefix = prefix;
|
|
999
1071
|
this.options = options;
|
|
1000
1072
|
this.entryRepo = entryRepo;
|
|
1073
|
+
this.sourceRefIndexRepo = sourceRefIndexRepo;
|
|
1074
|
+
this.metadataRepo = metadataRepo;
|
|
1075
|
+
this.edgeRepo = edgeRepo;
|
|
1001
1076
|
this.searchService = searchService;
|
|
1002
1077
|
this.jobManager = jobManager;
|
|
1003
1078
|
this.embeddingService = embeddingService;
|
|
@@ -1009,18 +1084,6 @@ var IngestionService = class {
|
|
|
1009
1084
|
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
1010
1085
|
const sourceHash = normalizeSourceHash(params.sourceHash);
|
|
1011
1086
|
if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
|
|
1012
|
-
const onDuplicateHash = opts?.onDuplicateHash ?? "ingest";
|
|
1013
|
-
if (onDuplicateHash !== "ingest") {
|
|
1014
|
-
const refs = await this.entryRepo.findSourceRefsByHash(entityId, sourceHash);
|
|
1015
|
-
const others = refs.filter((r) => r !== sourceRef);
|
|
1016
|
-
if (others.length > 0) {
|
|
1017
|
-
const canonical = others[0];
|
|
1018
|
-
if (onDuplicateHash === "throw") {
|
|
1019
|
-
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
1020
|
-
}
|
|
1021
|
-
return { truncated: false, chunks: 0, duplicateOf: canonical };
|
|
1022
|
-
}
|
|
1023
|
-
}
|
|
1024
1087
|
const maxChunkLength = params.maxChunkLength ?? this.options.config?.maxChunkLength ?? DEFAULT_MAX_CHUNK_LENGTH;
|
|
1025
1088
|
const rawOverlap = params.chunkOverlap ?? this.options.config?.chunkOverlap ?? DEFAULT_CHUNK_OVERLAP;
|
|
1026
1089
|
const chunkOverlap = Math.min(
|
|
@@ -1032,8 +1095,18 @@ var IngestionService = class {
|
|
|
1032
1095
|
if (typeof params.documentChunk !== "string") {
|
|
1033
1096
|
throw new Error(`documentChunk must be a string, received ${typeof params.documentChunk}`);
|
|
1034
1097
|
}
|
|
1035
|
-
this.jobManager.
|
|
1098
|
+
const releaseIngestLocks = await this.jobManager.acquireIngestLocks(entityId, sourceRef, sourceHash);
|
|
1036
1099
|
try {
|
|
1100
|
+
const onDuplicateHash = opts?.onDuplicateHash ?? "ingest";
|
|
1101
|
+
if (onDuplicateHash !== "ingest") {
|
|
1102
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
|
|
1103
|
+
if (canonical !== null && canonical !== sourceRef) {
|
|
1104
|
+
if (onDuplicateHash === "throw") {
|
|
1105
|
+
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
1106
|
+
}
|
|
1107
|
+
return { truncated: false, chunks: 0, duplicateOf: canonical };
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1037
1110
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
1038
1111
|
if (chunks.length === 0) return { truncated: false, chunks: 0 };
|
|
1039
1112
|
const chunkResults = await withConcurrency(
|
|
@@ -1069,68 +1142,85 @@ var IngestionService = class {
|
|
|
1069
1142
|
const now = Date.now();
|
|
1070
1143
|
const insertedFacts = [];
|
|
1071
1144
|
const deletedSourceFactIds = [];
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
});
|
|
1083
|
-
}
|
|
1084
|
-
let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
1085
|
-
let { mode, manifest } = ontologyState;
|
|
1086
|
-
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
1087
|
-
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
1088
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
1089
|
-
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
1090
|
-
mode = ontologyState.mode;
|
|
1145
|
+
try {
|
|
1146
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
1147
|
+
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
1148
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
1149
|
+
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
1150
|
+
for (const existing of existingFacts) {
|
|
1151
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
1152
|
+
id: existing.id,
|
|
1153
|
+
okf_type: existing.okf_type ?? null
|
|
1154
|
+
});
|
|
1091
1155
|
}
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1156
|
+
let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
1157
|
+
let { mode, manifest } = ontologyState;
|
|
1158
|
+
const hostNodes = [];
|
|
1159
|
+
const rawEdgeRequests = [];
|
|
1160
|
+
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
1161
|
+
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
1162
|
+
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
1163
|
+
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
1164
|
+
mode = ontologyState.mode;
|
|
1165
|
+
}
|
|
1166
|
+
for (const fact of facts) {
|
|
1167
|
+
const ontologyFact = fact;
|
|
1168
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
1169
|
+
const id = generateId("fact_");
|
|
1170
|
+
hostNodes.push({
|
|
1171
|
+
id,
|
|
1172
|
+
type: ontologyFact.okf_type ?? "",
|
|
1173
|
+
title: fact.title,
|
|
1174
|
+
body: fact.body,
|
|
1175
|
+
// Forward the LLM-extracted tags/confidence through the
|
|
1176
|
+
// extracted-shape upsertGraphCore path so the entries row
|
|
1177
|
+
// stores them (search filterability, heal-candidate
|
|
1178
|
+
// selection, runReembed embedding-text signal). The public
|
|
1179
|
+
// WikiMemory.upsertGraph API doesn't accept tags/confidence
|
|
1180
|
+
// — host-supplied deterministic nodes default to [] and
|
|
1181
|
+
// 'certain' as documented.
|
|
1182
|
+
tags: fact.tags,
|
|
1183
|
+
confidence: fact.confidence
|
|
1184
|
+
});
|
|
1185
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
1186
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
1187
|
+
if (normalized.edges.length > 0) {
|
|
1188
|
+
rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
1189
|
+
}
|
|
1118
1190
|
}
|
|
1119
1191
|
}
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1192
|
+
const hostEdges = [];
|
|
1193
|
+
for (const req of rawEdgeRequests) {
|
|
1194
|
+
const resolved = this.ontologyService?.resolveEdges(
|
|
1195
|
+
entityId,
|
|
1196
|
+
req.sourceId,
|
|
1197
|
+
req.sourceType,
|
|
1198
|
+
req.edges,
|
|
1199
|
+
manifest,
|
|
1200
|
+
titleIndex,
|
|
1201
|
+
now
|
|
1202
|
+
) ?? [];
|
|
1203
|
+
for (const e of resolved) {
|
|
1204
|
+
hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
await this.upsertGraphCore(
|
|
1123
1208
|
entityId,
|
|
1124
|
-
|
|
1125
|
-
item.sourceType,
|
|
1126
|
-
item.edges ?? [],
|
|
1127
|
-
manifest,
|
|
1128
|
-
titleIndex,
|
|
1209
|
+
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
1129
1210
|
tx,
|
|
1130
|
-
|
|
1211
|
+
{ strict: false }
|
|
1131
1212
|
);
|
|
1213
|
+
});
|
|
1214
|
+
} catch (err) {
|
|
1215
|
+
const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
|
|
1216
|
+
if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
|
|
1217
|
+
const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
|
|
1218
|
+
if (canonical === null) throw err;
|
|
1219
|
+
if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
|
|
1220
|
+
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
1132
1221
|
}
|
|
1133
|
-
|
|
1222
|
+
return { truncated: false, chunks: 0, duplicateOf: canonical };
|
|
1223
|
+
}
|
|
1134
1224
|
await this.searchService.sync(entityId);
|
|
1135
1225
|
const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
|
|
1136
1226
|
for (const factId of uniqueDeletedSourceFactIds) {
|
|
@@ -1146,9 +1236,129 @@ var IngestionService = class {
|
|
|
1146
1236
|
this.searchService.evictCache(entityId);
|
|
1147
1237
|
return { truncated, chunks: chunks.length };
|
|
1148
1238
|
} finally {
|
|
1149
|
-
|
|
1239
|
+
releaseIngestLocks();
|
|
1150
1240
|
}
|
|
1151
1241
|
}
|
|
1242
|
+
/**
|
|
1243
|
+
* Implements spec §2 data flow steps a–j with the host-facing
|
|
1244
|
+
* `{sourceRef, sourceHash, nodes, edges}` parameter shape. Shared between
|
|
1245
|
+
* `ingestDocument` (with `{ strict: false }` to preserve current behavior)
|
|
1246
|
+
* and `WikiMemory.upsertGraph` (which lets the persisted ontology mode
|
|
1247
|
+
* decide strictness).
|
|
1248
|
+
*
|
|
1249
|
+
* Runs ENTIRELY inside the supplied `tx`. Does not acquire locks, does not
|
|
1250
|
+
* open a nested transaction, does not perform post-commit work (search
|
|
1251
|
+
* sync, embedding, cache eviction). Those concerns remain in `ingestDocument`.
|
|
1252
|
+
*
|
|
1253
|
+
* The pre-flight validation (steps a–c) lives INSIDE this method per the
|
|
1254
|
+
* spec, so the public `WikiMemory.upsertGraph` can be a thin wrapper that
|
|
1255
|
+
* performs only the C2 probe before delegating.
|
|
1256
|
+
*
|
|
1257
|
+
* @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
|
|
1258
|
+
* (manifest-valid edges persisted), superseded (prior facts soft-deleted
|
|
1259
|
+
* plus prior source-ref edges hard-deleted).
|
|
1260
|
+
*/
|
|
1261
|
+
async upsertGraphCore(entityId, params, tx, opts) {
|
|
1262
|
+
const now = Date.now();
|
|
1263
|
+
const ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
1264
|
+
let { mode, manifest } = ontologyState;
|
|
1265
|
+
const strictEffective = opts?.strict === false ? false : opts?.strict === true || mode === "strict";
|
|
1266
|
+
const wikiFacts = [];
|
|
1267
|
+
for (const node of params.nodes) {
|
|
1268
|
+
const factLike = { okf_type: node.type, edges: [] };
|
|
1269
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(factLike, manifest, {
|
|
1270
|
+
strict: strictEffective,
|
|
1271
|
+
entityId
|
|
1272
|
+
}) ?? { okf_type: null};
|
|
1273
|
+
const wikiFact = {
|
|
1274
|
+
id: node.id,
|
|
1275
|
+
entity_id: entityId,
|
|
1276
|
+
title: node.title,
|
|
1277
|
+
body: node.body ?? "",
|
|
1278
|
+
// Public upsertGraph callers don't supply tags/confidence, so they
|
|
1279
|
+
// default to [] / 'certain' for deterministic AST-grade facts.
|
|
1280
|
+
// ingestDocument routes through this method for the LLM path and
|
|
1281
|
+
// forwards the LLM-supplied tags + confidence, preserving the prior
|
|
1282
|
+
// behavior (search filterability, heal-candidate selection, runReembed
|
|
1283
|
+
// embedding-text signal).
|
|
1284
|
+
tags: node.tags ? Array.from(node.tags) : [],
|
|
1285
|
+
confidence: node.confidence ?? "certain",
|
|
1286
|
+
source_type: "immutable_document",
|
|
1287
|
+
source_hash: params.sourceHash,
|
|
1288
|
+
source_ref: params.sourceRef,
|
|
1289
|
+
created_at: now,
|
|
1290
|
+
updated_at: now,
|
|
1291
|
+
last_accessed_at: null,
|
|
1292
|
+
access_count: 0,
|
|
1293
|
+
deleted_at: null,
|
|
1294
|
+
okf_type: normalized.okf_type
|
|
1295
|
+
};
|
|
1296
|
+
wikiFacts.push(wikiFact);
|
|
1297
|
+
}
|
|
1298
|
+
const sourceIdToType = /* @__PURE__ */ new Map();
|
|
1299
|
+
for (const fact of wikiFacts) sourceIdToType.set(fact.id, fact.okf_type ?? null);
|
|
1300
|
+
const hasConstraints = (manifest.node_types?.length ?? 0) > 0 || (manifest.edge_types?.length ?? 0) > 0;
|
|
1301
|
+
const validEdges = [];
|
|
1302
|
+
for (const edge of params.edges) {
|
|
1303
|
+
if (!hasConstraints) {
|
|
1304
|
+
validEdges.push({
|
|
1305
|
+
id: edge.id ?? generateId(),
|
|
1306
|
+
entity_id: entityId,
|
|
1307
|
+
source_id: edge.sourceId,
|
|
1308
|
+
target_id: edge.targetId,
|
|
1309
|
+
edge_type: edge.type,
|
|
1310
|
+
created_at: now
|
|
1311
|
+
});
|
|
1312
|
+
continue;
|
|
1313
|
+
}
|
|
1314
|
+
if (!sourceIdToType.has(edge.sourceId)) {
|
|
1315
|
+
validEdges.push({
|
|
1316
|
+
id: edge.id ?? generateId(),
|
|
1317
|
+
entity_id: entityId,
|
|
1318
|
+
source_id: edge.sourceId,
|
|
1319
|
+
target_id: edge.targetId,
|
|
1320
|
+
edge_type: edge.type,
|
|
1321
|
+
created_at: now
|
|
1322
|
+
});
|
|
1323
|
+
continue;
|
|
1324
|
+
}
|
|
1325
|
+
const sourceType = sourceIdToType.get(edge.sourceId);
|
|
1326
|
+
const candidates = (manifest.edge_types ?? []).filter(
|
|
1327
|
+
(d) => d.type.toLowerCase() === edge.type.toLowerCase() && d.source_type.toLowerCase() === (sourceType ?? "").toLowerCase()
|
|
1328
|
+
);
|
|
1329
|
+
const match = candidates[0];
|
|
1330
|
+
if (!match) {
|
|
1331
|
+
if (strictEffective) throw new WikiStrictOntologyViolation(entityId, "edge", edge.type);
|
|
1332
|
+
continue;
|
|
1333
|
+
}
|
|
1334
|
+
validEdges.push({
|
|
1335
|
+
id: edge.id ?? generateId(),
|
|
1336
|
+
entity_id: entityId,
|
|
1337
|
+
source_id: edge.sourceId,
|
|
1338
|
+
target_id: edge.targetId,
|
|
1339
|
+
edge_type: match.type,
|
|
1340
|
+
created_at: now
|
|
1341
|
+
});
|
|
1342
|
+
}
|
|
1343
|
+
const deletedFactIds = await this.entryRepo.findIdsBySource(entityId, params.sourceRef, null, tx, false);
|
|
1344
|
+
await this.entryRepo.softDeleteBySource(entityId, tx, params.sourceRef, null);
|
|
1345
|
+
const deletedEdgeCount = await this.edgeRepo.softDeleteBySourceFactIds(entityId, deletedFactIds, tx);
|
|
1346
|
+
await this.sourceRefIndexRepo.softDeleteByEntityAndSourceRef(entityId, params.sourceRef, tx);
|
|
1347
|
+
await this.sourceRefIndexRepo.upsert(entityId, params.sourceHash, params.sourceRef, tx);
|
|
1348
|
+
for (const wikiFact of wikiFacts) {
|
|
1349
|
+
await this.entryRepo.upsert(wikiFact, tx);
|
|
1350
|
+
}
|
|
1351
|
+
let edgesInsertedCount = 0;
|
|
1352
|
+
for (const edge of validEdges) {
|
|
1353
|
+
const inserted = await this.edgeRepo.addIgnoreDuplicate(edge, tx);
|
|
1354
|
+
if (inserted) edgesInsertedCount++;
|
|
1355
|
+
}
|
|
1356
|
+
return {
|
|
1357
|
+
nodesWritten: wikiFacts.length,
|
|
1358
|
+
edgesWritten: edgesInsertedCount,
|
|
1359
|
+
superseded: deletedFactIds.length + deletedEdgeCount
|
|
1360
|
+
};
|
|
1361
|
+
}
|
|
1152
1362
|
};
|
|
1153
1363
|
|
|
1154
1364
|
// src/utils/embedding.ts
|
|
@@ -1298,11 +1508,12 @@ var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
|
1298
1508
|
var HEAL_BATCH_SIZE = 25;
|
|
1299
1509
|
var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
1300
1510
|
var MaintenanceService = class {
|
|
1301
|
-
constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1511
|
+
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1302
1512
|
this.db = db;
|
|
1303
1513
|
this.prefix = prefix;
|
|
1304
1514
|
this.options = options;
|
|
1305
1515
|
this.entryRepo = entryRepo;
|
|
1516
|
+
this.sourceRefIndexRepo = sourceRefIndexRepo;
|
|
1306
1517
|
this.taskRepo = taskRepo;
|
|
1307
1518
|
this.eventRepo = eventRepo;
|
|
1308
1519
|
this.metadataRepo = metadataRepo;
|
|
@@ -1501,10 +1712,12 @@ var MaintenanceService = class {
|
|
|
1501
1712
|
const entryPromise = params.entryId ? this.entryRepo.softDelete(params.entryId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
1502
1713
|
const taskDeletedPromise = params.taskId ? this.taskRepo.softDeleteById(params.taskId, entityId, tx).then((r) => r.changes > 0) : null;
|
|
1503
1714
|
const refPromise = sourceRef || sourceHash ? this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, sourceHash) : null;
|
|
1715
|
+
const sourceRefIndexPromise = sourceRef ? this.sourceRefIndexRepo.softDeleteByEntityAndSourceRef(entityId, sourceRef, tx).then(() => void 0) : null;
|
|
1504
1716
|
const [entryResult, taskResult, refResult] = await Promise.all([
|
|
1505
1717
|
entryPromise ?? Promise.resolve(false),
|
|
1506
1718
|
taskDeletedPromise ?? Promise.resolve(false),
|
|
1507
|
-
refPromise ?? Promise.resolve(0)
|
|
1719
|
+
refPromise ?? Promise.resolve(0),
|
|
1720
|
+
sourceRefIndexPromise ?? Promise.resolve(void 0)
|
|
1508
1721
|
]);
|
|
1509
1722
|
if (entryResult) deletedEntries++;
|
|
1510
1723
|
if (taskResult) deletedTasks++;
|
|
@@ -1622,7 +1835,7 @@ var MaintenanceService = class {
|
|
|
1622
1835
|
}
|
|
1623
1836
|
if (skip) continue;
|
|
1624
1837
|
const ontologyFact = fact;
|
|
1625
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest) ?? { okf_type: null, edges: [] };
|
|
1838
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
1626
1839
|
const id = generateId("fact_");
|
|
1627
1840
|
const factObj = {
|
|
1628
1841
|
id,
|
|
@@ -2007,7 +2220,8 @@ var MaintenanceService = class {
|
|
|
2007
2220
|
okf_type: classification.okf_type,
|
|
2008
2221
|
edges: classification.edges
|
|
2009
2222
|
},
|
|
2010
|
-
manifest
|
|
2223
|
+
manifest,
|
|
2224
|
+
{ strict: false }
|
|
2011
2225
|
);
|
|
2012
2226
|
if (!normalized.okf_type) {
|
|
2013
2227
|
failedValidation++;
|
|
@@ -2880,6 +3094,19 @@ var JobManager = class {
|
|
|
2880
3094
|
this.prefix = prefix;
|
|
2881
3095
|
this.activeMaintenanceJobs = /* @__PURE__ */ new Set();
|
|
2882
3096
|
this.activeIngestJobs = /* @__PURE__ */ new Map();
|
|
3097
|
+
/**
|
|
3098
|
+
* Per-(entityId, sourceHash) promise-chain lock. Serializes ingest races
|
|
3099
|
+
* at the application level (the v9 partial UNIQUE index enforces it at
|
|
3100
|
+
* the DB level as defense-in-depth). Two callers holding the same hash
|
|
3101
|
+
* see FIFO order; different hashes never block one another.
|
|
3102
|
+
*
|
|
3103
|
+
* Keyed `${entityId}\0${sourceHash}` so distinct (entityId, sourceHash)
|
|
3104
|
+
* pairs can never share a lock. A raw concatenation would let
|
|
3105
|
+
* ("ab", "c…") collide with ("a", "bc…"); the NUL separator makes that
|
|
3106
|
+
* impossible because neither component can contain NUL (entityIds are
|
|
3107
|
+
* user-supplied identifiers; sourceHashes are hex digests).
|
|
3108
|
+
*/
|
|
3109
|
+
this.hashLocks = /* @__PURE__ */ new Map();
|
|
2883
3110
|
this.statusSubscribers = /* @__PURE__ */ new Map();
|
|
2884
3111
|
/**
|
|
2885
3112
|
* Lookup table for acquireLock/releaseLock's dynamic-dispatch branch.
|
|
@@ -2897,6 +3124,51 @@ var JobManager = class {
|
|
|
2897
3124
|
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
2898
3125
|
};
|
|
2899
3126
|
}
|
|
3127
|
+
_hashLockKey(entityId, sourceHash) {
|
|
3128
|
+
return `${entityId}\0${sourceHash}`;
|
|
3129
|
+
}
|
|
3130
|
+
/**
|
|
3131
|
+
* Acquire a promise-chain lock scoped to one `(entityId, sourceHash)` pair.
|
|
3132
|
+
* Returns a zero-argument release closure that removes the current tail
|
|
3133
|
+
* from the chain. Releasing is idempotent so a holder that double-fires
|
|
3134
|
+
* (e.g. on a finally that races an explicit release) cannot unblock
|
|
3135
|
+
* the next holder prematurely.
|
|
3136
|
+
*
|
|
3137
|
+
* Documented ordering: ingest acquires the hash lock FIRST, then the
|
|
3138
|
+
* synchronous sourceRef ingest lock, and the release closure unwinds in
|
|
3139
|
+
* the opposite order (sourceRef first, hash second). Hash-then-sourceRef
|
|
3140
|
+
* is required because the v9 partial UNIQUE index conflicts on the
|
|
3141
|
+
* (entity_id, source_hash) pair regardless of source_ref, so two callers
|
|
3142
|
+
* with DIFFERENT source_refs racing the same hash are the exact case the
|
|
3143
|
+
* TOCTOU race fix needs to serialize.
|
|
3144
|
+
*
|
|
3145
|
+
* Mechanics: each holder owns a private `released` promise that resolves
|
|
3146
|
+
* only when THIS holder calls release(). The map stores the holder's
|
|
3147
|
+
* `released` promise as the "tail" — the NEXT caller awaits that tail
|
|
3148
|
+
* via `previous = this.hashLocks.get(key)`. The current caller gets
|
|
3149
|
+
* their unique release function once `previous` (the prior holder's
|
|
3150
|
+
* release signal) settles, so each holder is guaranteed a UNIQUE release
|
|
3151
|
+
* closure bound to their own `released` promise.
|
|
3152
|
+
*/
|
|
3153
|
+
acquireHashLock(entityId, sourceHash) {
|
|
3154
|
+
const key = this._hashLockKey(entityId, sourceHash);
|
|
3155
|
+
const previous = this.hashLocks.get(key) ?? Promise.resolve();
|
|
3156
|
+
let resolveReleased;
|
|
3157
|
+
const released = new Promise((resolve) => {
|
|
3158
|
+
resolveReleased = resolve;
|
|
3159
|
+
});
|
|
3160
|
+
let fired = false;
|
|
3161
|
+
const release = () => {
|
|
3162
|
+
if (fired) return;
|
|
3163
|
+
fired = true;
|
|
3164
|
+
if (this.hashLocks.get(key) === released) {
|
|
3165
|
+
this.hashLocks.delete(key);
|
|
3166
|
+
}
|
|
3167
|
+
resolveReleased();
|
|
3168
|
+
};
|
|
3169
|
+
this.hashLocks.set(key, released);
|
|
3170
|
+
return previous.then(() => release);
|
|
3171
|
+
}
|
|
2900
3172
|
_pruneKey(entityId) {
|
|
2901
3173
|
return `${this.prefix}:${entityId}:prune`;
|
|
2902
3174
|
}
|
|
@@ -3110,6 +3382,38 @@ var JobManager = class {
|
|
|
3110
3382
|
this._notifyStatusSubscribers(entityId);
|
|
3111
3383
|
return true;
|
|
3112
3384
|
}
|
|
3385
|
+
/**
|
|
3386
|
+
* Centralized ingest lock acquisition. Acquires the hash lock FIRST
|
|
3387
|
+
* (FIFO across callers racing the same hash), then the synchronous
|
|
3388
|
+
* sourceRef ingest lock (rejects cross-entity conflicts and busy
|
|
3389
|
+
* operations). Returns a single zero-argument release closure that
|
|
3390
|
+
* unwinds in the opposite order — sourceRef first, hash second — so
|
|
3391
|
+
* the sourceRef lock is freed the instant its work is done and only
|
|
3392
|
+
* the hash lock keeps serializing the duplicate-content race window.
|
|
3393
|
+
*
|
|
3394
|
+
* On a sourceRef conflict (WikiBusyError thrown synchronously by the
|
|
3395
|
+
* underlying acquireLock) the hash lock is released BEFORE the error
|
|
3396
|
+
* propagates, so the next caller in the hash chain is not held up by
|
|
3397
|
+
* a request that never reached the DB.
|
|
3398
|
+
*/
|
|
3399
|
+
async acquireIngestLocks(entityId, sourceRef, sourceHash) {
|
|
3400
|
+
const releaseHash = await this.acquireHashLock(entityId, sourceHash);
|
|
3401
|
+
let sourceLockAcquired = false;
|
|
3402
|
+
try {
|
|
3403
|
+
this.acquireLock("ingest", entityId, sourceRef);
|
|
3404
|
+
sourceLockAcquired = true;
|
|
3405
|
+
} catch (err) {
|
|
3406
|
+
releaseHash();
|
|
3407
|
+
throw err;
|
|
3408
|
+
}
|
|
3409
|
+
return () => {
|
|
3410
|
+
try {
|
|
3411
|
+
if (sourceLockAcquired) this.releaseLock("ingest", entityId, sourceRef);
|
|
3412
|
+
} finally {
|
|
3413
|
+
releaseHash();
|
|
3414
|
+
}
|
|
3415
|
+
};
|
|
3416
|
+
}
|
|
3113
3417
|
/**
|
|
3114
3418
|
* Validates then acquires global + per-entity import locks atomically.
|
|
3115
3419
|
* Validates all entities before acquiring any lock (same as current importDump semantics).
|