@equationalapplications/core-llm-wiki 5.4.0 → 5.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -125,6 +125,25 @@ var PrunePartialFailureError = class extends Error {
125
125
  }
126
126
  };
127
127
  var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
128
+ var WikiParseError = class extends Error {
129
+ constructor(message, opts) {
130
+ super(message);
131
+ this.name = "WikiParseError";
132
+ this.tier = opts.tier;
133
+ this.position = opts.position ?? null;
134
+ this.slice = opts.slice ?? "";
135
+ }
136
+ };
137
+ var WikiIngestEmptyError = class extends Error {
138
+ constructor(params) {
139
+ const summary = `All ${params.chunks} chunks failed for sourceRef "${params.sourceRef}"; see parseFailures for per-chunk detail`;
140
+ super(summary);
141
+ this.name = "WikiIngestEmptyError";
142
+ this.parseFailures = params.parseFailures;
143
+ this.sourceRef = params.sourceRef;
144
+ this.chunks = params.chunks;
145
+ }
146
+ };
128
147
 
129
148
  // src/utils/ids.ts
130
149
  var _injectedGetRandomValues = null;
@@ -945,24 +964,67 @@ var JobManager = class {
945
964
  };
946
965
 
947
966
  // src/utils/pure.ts
967
+ var MAX_REPAIR_CANDIDATES = 5;
968
+ function throwNoJsonFound(text, start) {
969
+ throw new WikiParseError(
970
+ "No JSON object/array found in LLM response",
971
+ { tier: "strict", position: start, slice: text }
972
+ );
973
+ }
948
974
  function parseJsonResponse(text) {
949
975
  const firstBrace = text.indexOf("{");
950
976
  const firstBracket = text.indexOf("[");
951
977
  let start;
952
978
  let openChar;
953
- let closeChar;
954
979
  const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
955
980
  if (useBrace) {
956
981
  start = firstBrace;
957
982
  openChar = "{";
958
- closeChar = "}";
959
983
  } else if (firstBracket !== -1) {
960
984
  start = firstBracket;
961
985
  openChar = "[";
962
- closeChar = "]";
963
986
  } else {
964
- throw new SyntaxError("No JSON object/array found in LLM response");
987
+ throwNoJsonFound(text, null);
988
+ }
989
+ const slice = scanJsonSlice(text, start, openChar);
990
+ if (slice !== null) {
991
+ try {
992
+ return JSON.parse(slice);
993
+ } catch {
994
+ }
995
+ }
996
+ const repairOdd = containerAwareRepair(text, start, false);
997
+ if (repairOdd.success !== null && !repairOdd.ambiguous) {
998
+ return repairOdd.success;
965
999
  }
1000
+ const repairEven = containerAwareRepair(text, start, true);
1001
+ if (repairOdd.success !== null && repairEven.success !== null) {
1002
+ const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
1003
+ return pick.success;
1004
+ }
1005
+ if (repairOdd.success !== null) {
1006
+ return repairOdd.success;
1007
+ }
1008
+ if (repairEven.success !== null) {
1009
+ return repairEven.success;
1010
+ }
1011
+ const repair = repairEven.failed !== null ? repairEven : repairOdd;
1012
+ if (repair.failed !== null) {
1013
+ throw new WikiParseError(
1014
+ `Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
1015
+ { tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
1016
+ );
1017
+ }
1018
+ if (slice === null) {
1019
+ throwNoJsonFound(text, start);
1020
+ }
1021
+ throw new WikiParseError(
1022
+ "No parsable JSON candidate found",
1023
+ { tier: "all", position: start, slice: text }
1024
+ );
1025
+ }
1026
+ function scanJsonSlice(text, start, openChar) {
1027
+ const closeChar = openChar === "{" ? "}" : "]";
966
1028
  let depth = 0;
967
1029
  let inString = false;
968
1030
  let escape = false;
@@ -994,20 +1056,248 @@ function parseJsonResponse(text) {
994
1056
  }
995
1057
  }
996
1058
  }
997
- if (end === -1) throw new SyntaxError("No JSON object/array found in LLM response");
998
- return JSON.parse(text.slice(start, end + 1));
1059
+ if (end === -1) return null;
1060
+ return safeSlice(text, start, end + 1);
1061
+ }
1062
+ function containerAwareRepair(text, start, closeOnEvenParity) {
1063
+ const stack = [];
1064
+ let inString = false;
1065
+ let stringRole = null;
1066
+ let escape = false;
1067
+ let i = start;
1068
+ let out = "";
1069
+ let bareQuoteCount = 0;
1070
+ let bestFailed = null;
1071
+ let escapes = 0;
1072
+ let ambiguous = false;
1073
+ let bestSuccess = null;
1074
+ let attempts = 0;
1075
+ function tryEmitCandidate(candidate) {
1076
+ attempts++;
1077
+ try {
1078
+ bestSuccess = JSON.parse(candidate);
1079
+ return true;
1080
+ } catch (err) {
1081
+ if (bestFailed === null) {
1082
+ bestFailed = {
1083
+ candidate,
1084
+ position: extractParsePosition(err),
1085
+ message: safeErrorToString(err)
1086
+ };
1087
+ }
1088
+ return false;
1089
+ }
1090
+ }
1091
+ while (i < text.length) {
1092
+ const ch = text[i];
1093
+ if (stack.length === 0 && !inString) {
1094
+ if (ch === "{" || ch === "[") {
1095
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
1096
+ out += ch;
1097
+ }
1098
+ i++;
1099
+ continue;
1100
+ }
1101
+ if (escape) {
1102
+ out += ch;
1103
+ escape = false;
1104
+ i++;
1105
+ continue;
1106
+ }
1107
+ if (inString && ch === "\\") {
1108
+ out += ch;
1109
+ escape = true;
1110
+ i++;
1111
+ continue;
1112
+ }
1113
+ if (inString) {
1114
+ const code = ch.charCodeAt(0);
1115
+ if (code < 32) {
1116
+ out += controlCharEscape(code);
1117
+ i++;
1118
+ continue;
1119
+ }
1120
+ }
1121
+ if (ch === '"') {
1122
+ let j = i + 1;
1123
+ while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
1124
+ j++;
1125
+ }
1126
+ const next = j < text.length ? text[j] : "";
1127
+ if (!inString) {
1128
+ if (next === "}" || next === "]") {
1129
+ if (j > i + 1) out += text.slice(i + 1, j);
1130
+ out += next;
1131
+ i = j + 1;
1132
+ stack.pop();
1133
+ if (stack.length === 0) {
1134
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
1135
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
1136
+ out = "";
1137
+ }
1138
+ continue;
1139
+ }
1140
+ const top = stack[stack.length - 1];
1141
+ stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
1142
+ out += ch;
1143
+ inString = true;
1144
+ i++;
1145
+ continue;
1146
+ }
1147
+ const commaOnly = next === ",";
1148
+ const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
1149
+ const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
1150
+ if (commaOnly && bareQuoteCount > 0) ambiguous = true;
1151
+ const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
1152
+ const isClosing = isKeyClose || isValueClose || shouldCommaClose;
1153
+ if (isClosing) {
1154
+ out += ch;
1155
+ inString = false;
1156
+ const topFrame = stack[stack.length - 1];
1157
+ if (topFrame && topFrame.container === "object") {
1158
+ topFrame.expectKey = stringRole === "value";
1159
+ }
1160
+ stringRole = null;
1161
+ bareQuoteCount = 0;
1162
+ i++;
1163
+ continue;
1164
+ }
1165
+ out += "\\" + ch;
1166
+ bareQuoteCount++;
1167
+ escapes++;
1168
+ i++;
1169
+ continue;
1170
+ }
1171
+ if (!inString && (ch === "{" || ch === "[")) {
1172
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
1173
+ out += ch;
1174
+ i++;
1175
+ continue;
1176
+ }
1177
+ if (!inString && ch === ",") {
1178
+ const topFrame = stack[stack.length - 1];
1179
+ if (topFrame && topFrame.container === "object") {
1180
+ topFrame.expectKey = true;
1181
+ }
1182
+ out += ch;
1183
+ i++;
1184
+ continue;
1185
+ }
1186
+ if (!inString && (ch === "}" || ch === "]")) {
1187
+ out += ch;
1188
+ stack.pop();
1189
+ if (stack.length === 0) {
1190
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
1191
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
1192
+ out = "";
1193
+ }
1194
+ i++;
1195
+ continue;
1196
+ }
1197
+ out += ch;
1198
+ i++;
1199
+ }
1200
+ return { success: null, failed: bestFailed, escapes, ambiguous };
1201
+ }
1202
+ function controlCharEscape(code) {
1203
+ switch (code) {
1204
+ case 10:
1205
+ return "\\n";
1206
+ case 9:
1207
+ return "\\t";
1208
+ case 13:
1209
+ return "\\r";
1210
+ case 8:
1211
+ return "\\b";
1212
+ case 12:
1213
+ return "\\f";
1214
+ default:
1215
+ return "\\u" + code.toString(16).padStart(4, "0");
1216
+ }
1217
+ }
1218
+ function extractParsePosition(err) {
1219
+ const match = /position\s+(\d+)/.exec(safeErrorToString(err));
1220
+ return match ? Number(match[1]) : null;
1221
+ }
1222
+ function safeErrorToString(e) {
1223
+ let isErrorLike = false;
1224
+ try {
1225
+ isErrorLike = e instanceof Error;
1226
+ } catch {
1227
+ }
1228
+ if (isErrorLike) {
1229
+ const msg = readErrorField(e, "message");
1230
+ if (typeof msg === "string" && msg.length > 0) return msg;
1231
+ const name = readErrorField(e, "name");
1232
+ if (typeof name === "string" && name.length > 0) return name;
1233
+ return "[Error]";
1234
+ }
1235
+ try {
1236
+ return String(e);
1237
+ } catch {
1238
+ try {
1239
+ return Object.prototype.toString.call(e);
1240
+ } catch {
1241
+ return "[unstringifiable error]";
1242
+ }
1243
+ }
1244
+ }
1245
+ function readErrorField(e, key) {
1246
+ try {
1247
+ return e[key];
1248
+ } catch {
1249
+ return void 0;
1250
+ }
999
1251
  }
1000
1252
  function sanitizeRankerError(err, sanitizeRankerErrors) {
1253
+ let isErrorLike = false;
1254
+ try {
1255
+ isErrorLike = err instanceof Error;
1256
+ } catch {
1257
+ }
1001
1258
  if (sanitizeRankerErrors === false) {
1002
- return err instanceof Error ? err : new Error(String(err));
1259
+ return isErrorLike ? err : new Error(safeErrorToString(err));
1260
+ }
1261
+ let errLike = null;
1262
+ if (isErrorLike) errLike = err;
1263
+ let typeName;
1264
+ try {
1265
+ if (errLike) {
1266
+ const rawName = errLike.constructor?.name;
1267
+ typeName = typeof rawName === "string" ? rawName : "Error";
1268
+ } else {
1269
+ typeName = typeof err;
1270
+ }
1271
+ } catch {
1272
+ typeName = isErrorLike ? "Error" : typeof err;
1273
+ }
1274
+ let innerCause;
1275
+ if (errLike) {
1276
+ let cause;
1277
+ try {
1278
+ cause = errLike.cause;
1279
+ } catch {
1280
+ cause = void 0;
1281
+ }
1282
+ if (cause !== void 0) {
1283
+ let causeName;
1284
+ try {
1285
+ const rawCauseName = cause?.constructor?.name;
1286
+ causeName = typeof rawCauseName === "string" ? rawCauseName : typeof cause;
1287
+ } catch {
1288
+ causeName = typeof cause;
1289
+ }
1290
+ innerCause = new Error(`Caused by: ${causeName}`);
1291
+ }
1003
1292
  }
1004
- const typeName = err instanceof Error ? err.constructor?.name ?? "Error" : typeof err;
1005
- const innerCause = err instanceof Error && err.cause !== void 0 ? new Error(`Caused by: ${err.cause?.constructor?.name ?? typeof err.cause}`) : void 0;
1006
1293
  const sanitized = new Error(
1007
1294
  `VectorRanker ${typeName} (message scrubbed for security)`,
1008
1295
  innerCause ? { cause: innerCause } : void 0
1009
1296
  );
1010
- sanitized.name = typeName;
1297
+ try {
1298
+ sanitized.name = typeName;
1299
+ } catch {
1300
+ }
1011
1301
  return sanitized;
1012
1302
  }
1013
1303
  function safeSlice(value, start, end) {
@@ -1180,7 +1470,7 @@ Return ONLY a valid JSON object matching this schema:
1180
1470
  {
1181
1471
  "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
1182
1472
  }
1183
- Extract verbatim factual content. Do not return markdown, just raw JSON.`;
1473
+ Extract verbatim factual content. JSON escaping rules: every literal " character in the source must be escaped as \\" inside any JSON string body, and every literal newline as \\n. Source prose containing quotes (e.g. a worked example with "...") must still be reproduced exactly \u2014 re-escape, do not omit. Do not return markdown, just raw JSON.`;
1184
1474
  var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
1185
1475
  Return ONLY a valid JSON object matching this schema:
1186
1476
  {
@@ -1189,7 +1479,7 @@ Return ONLY a valid JSON object matching this schema:
1189
1479
  ]
1190
1480
  }
1191
1481
  If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
1192
- Do not return markdown, just raw JSON.`;
1482
+ When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
1193
1483
 
1194
1484
  // src/services/PromptService.ts
1195
1485
  var PromptService = class {
@@ -1302,6 +1592,16 @@ var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
1302
1592
  var DEFAULT_CHUNK_OVERLAP = 400;
1303
1593
 
1304
1594
  // src/services/IngestionService.ts
1595
+ function zeroChunkResult(duplicateOf) {
1596
+ const result = {
1597
+ truncated: false,
1598
+ chunks: 0,
1599
+ ingestedChunks: 0,
1600
+ failedChunks: 0
1601
+ };
1602
+ if (duplicateOf !== void 0) result.duplicateOf = duplicateOf;
1603
+ return result;
1604
+ }
1305
1605
  var IngestionService = class {
1306
1606
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
1307
1607
  this.db = db;
@@ -1342,113 +1642,110 @@ var IngestionService = class {
1342
1642
  if (onDuplicateHash === "throw") {
1343
1643
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
1344
1644
  }
1345
- return { truncated: false, chunks: 0, duplicateOf: canonical };
1645
+ return zeroChunkResult(canonical);
1346
1646
  }
1347
1647
  }
1348
1648
  const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
1349
- if (chunks.length === 0) return { truncated: false, chunks: 0 };
1649
+ if (chunks.length === 0) return zeroChunkResult();
1650
+ const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
1350
1651
  const chunkResults = await withConcurrency(
1351
- chunks.map((chunk) => async () => {
1352
- const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
1652
+ chunks.map((chunk, chunkIndex) => async () => {
1353
1653
  const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
1354
1654
  chunk,
1355
1655
  params.promptOverride,
1356
1656
  ontologyContext
1357
1657
  );
1358
- const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
1359
- const result = parseJsonResponse(responseText);
1360
- return {
1361
- facts: (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null),
1362
- ontology_updates: result.ontology_updates
1363
- };
1658
+ try {
1659
+ const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
1660
+ const result2 = parseJsonResponse(responseText);
1661
+ return {
1662
+ status: "ok",
1663
+ facts: (Array.isArray(result2.facts) ? result2.facts : []).map(validateFact).filter((f) => f !== null),
1664
+ ontology_updates: result2.ontology_updates
1665
+ };
1666
+ } catch (e) {
1667
+ const failure = e instanceof WikiParseError ? {
1668
+ chunkIndex,
1669
+ sourceRef,
1670
+ source: "parse",
1671
+ tier: e.tier,
1672
+ position: e.position,
1673
+ message: e.message
1674
+ } : {
1675
+ chunkIndex,
1676
+ sourceRef,
1677
+ source: "llm",
1678
+ // `safeErrorToString` is a non-throwing coercion: an LLM
1679
+ // provider may reject with a non-Error value, and even
1680
+ // `String(e)` itself can throw when `e.toString()` throws.
1681
+ // Letting such an exception escape this catch would cause
1682
+ // `withConcurrency` to reject, discarding every sibling
1683
+ // chunk's results — the exact failure mode this per-chunk
1684
+ // try/catch exists to prevent. Keeping the value in the
1685
+ // `ChunkFailure` preserves the typed diagnostic for
1686
+ // callers (see parseFailures) without unwinding the loop.
1687
+ position: null,
1688
+ message: safeErrorToString(e)
1689
+ };
1690
+ const total = chunks.length;
1691
+ const tags = [];
1692
+ if (failure.tier) tags.push(`tier=${failure.tier}`);
1693
+ if (failure.position !== null) tags.push(`position=${failure.position}`);
1694
+ const tagsSuffix = tags.length > 0 ? `; ${tags.join(" ")}` : "";
1695
+ console.warn(
1696
+ `[WikiMemory] ingest chunk ${chunkIndex + 1}/${total} ${failure.source} failed (sourceRef=${sourceRef}${tagsSuffix})`
1697
+ );
1698
+ return { status: "failed", error: failure };
1699
+ }
1364
1700
  }),
1365
1701
  chunkConcurrency
1366
1702
  );
1703
+ let ingestedChunks = 0;
1704
+ let failedChunks = 0;
1705
+ const failures = [];
1367
1706
  const seen = /* @__PURE__ */ new Set();
1368
1707
  const orderedChunkFacts = [];
1369
- for (const chunkResult of chunkResults) {
1708
+ for (const slot of chunkResults) {
1709
+ if (slot.status === "failed") {
1710
+ failedChunks++;
1711
+ failures.push(slot.error);
1712
+ continue;
1713
+ }
1714
+ ingestedChunks++;
1370
1715
  const dedupedFacts = [];
1371
- for (const fact of chunkResult.facts) {
1716
+ for (const fact of slot.facts) {
1372
1717
  const normalizedTitle = normalizeTitleKey(fact.title);
1373
1718
  if (!seen.has(normalizedTitle)) {
1374
1719
  seen.add(normalizedTitle);
1375
1720
  dedupedFacts.push(fact);
1376
1721
  }
1377
1722
  }
1378
- orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: chunkResult.ontology_updates });
1723
+ orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: slot.ontology_updates });
1724
+ }
1725
+ if (failedChunks === chunks.length) {
1726
+ throw new WikiIngestEmptyError({
1727
+ parseFailures: failures,
1728
+ sourceRef,
1729
+ chunks: chunks.length
1730
+ });
1379
1731
  }
1380
- const now = Date.now();
1381
1732
  const insertedFacts = [];
1382
1733
  const deletedSourceFactIds = [];
1383
1734
  try {
1384
- await this.db.withTransactionAsync(async (tx) => {
1385
- deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
1386
- const titleIndex = /* @__PURE__ */ new Map();
1387
- const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
1388
- for (const existing of existingFacts) {
1389
- titleIndex.set(normalizeTitleKey(existing.title), {
1390
- id: existing.id,
1391
- okf_type: existing.okf_type ?? null
1392
- });
1393
- }
1394
- let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
1395
- let { mode, manifest } = ontologyState;
1396
- const hostNodes = [];
1397
- const rawEdgeRequests = [];
1398
- for (const { facts, ontology_updates } of orderedChunkFacts) {
1399
- if (mode === "emergent" && ontology_updates && this.ontologyService) {
1400
- manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
1401
- ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
1402
- mode = ontologyState.mode;
1403
- }
1404
- for (const fact of facts) {
1405
- const ontologyFact = fact;
1406
- const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
1407
- const id = generateId("fact_");
1408
- hostNodes.push({
1409
- id,
1410
- type: ontologyFact.okf_type ?? "",
1411
- title: fact.title,
1412
- body: fact.body,
1413
- // Forward the LLM-extracted tags/confidence through the
1414
- // extracted-shape upsertGraphCore path so the entries row
1415
- // stores them (search filterability, heal-candidate
1416
- // selection, runReembed embedding-text signal). The public
1417
- // WikiMemory.upsertGraph API doesn't accept tags/confidence
1418
- // — host-supplied deterministic nodes default to [] and
1419
- // 'certain' as documented.
1420
- tags: fact.tags,
1421
- confidence: fact.confidence
1422
- });
1423
- insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
1424
- titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
1425
- if (normalized.edges.length > 0) {
1426
- rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
1427
- }
1428
- }
1429
- }
1430
- const hostEdges = [];
1431
- for (const req of rawEdgeRequests) {
1432
- const resolved = this.ontologyService?.resolveEdges(
1433
- entityId,
1434
- req.sourceId,
1435
- req.sourceType,
1436
- req.edges,
1437
- manifest,
1438
- titleIndex,
1439
- now
1440
- ) ?? [];
1441
- for (const e of resolved) {
1442
- hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
1443
- }
1444
- }
1445
- await this.upsertGraphCore(
1446
- entityId,
1447
- { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
1448
- tx,
1449
- { strict: false }
1450
- );
1451
- });
1735
+ if (failedChunks === 0) {
1736
+ const fullResult = await this.db.withTransactionAsync(async (tx) => {
1737
+ return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx);
1738
+ });
1739
+ deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
1740
+ insertedFacts.push(...fullResult.insertedFacts);
1741
+ } else {
1742
+ const partialResult = await this.db.withTransactionAsync(async (tx) => {
1743
+ const flat = [];
1744
+ for (const slot of orderedChunkFacts) flat.push(...slot.facts);
1745
+ return await this.appendPartialFacts(entityId, sourceRef, flat, tx);
1746
+ });
1747
+ insertedFacts.push(...partialResult.insertedDescriptors);
1748
+ }
1452
1749
  } catch (err) {
1453
1750
  const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
1454
1751
  if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
@@ -1457,22 +1754,31 @@ var IngestionService = class {
1457
1754
  if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
1458
1755
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
1459
1756
  }
1460
- return { truncated: false, chunks: 0, duplicateOf: canonical };
1757
+ return zeroChunkResult(canonical);
1461
1758
  }
1462
1759
  await this.searchService.sync(entityId);
1463
- const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
1464
- for (const factId of uniqueDeletedSourceFactIds) {
1465
- try {
1466
- await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
1467
- } catch (hookErr) {
1468
- console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
1760
+ if (failedChunks === 0) {
1761
+ const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
1762
+ for (const factId of uniqueDeletedSourceFactIds) {
1763
+ try {
1764
+ await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
1765
+ } catch (hookErr) {
1766
+ console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
1767
+ }
1469
1768
  }
1470
1769
  }
1471
1770
  for (const fact of insertedFacts) {
1472
1771
  await this.embeddingService.embedFact(fact);
1473
1772
  }
1474
1773
  this.searchService.evictCache(entityId);
1475
- return { truncated, chunks: chunks.length };
1774
+ const result = {
1775
+ truncated,
1776
+ chunks: chunks.length,
1777
+ ingestedChunks,
1778
+ failedChunks
1779
+ };
1780
+ if (failedChunks > 0) result.parseFailures = failures;
1781
+ return result;
1476
1782
  } finally {
1477
1783
  releaseIngestLocks();
1478
1784
  }
@@ -1597,6 +1903,156 @@ var IngestionService = class {
1597
1903
  superseded: deletedFactIds.length + deletedEdgeCount
1598
1904
  };
1599
1905
  }
1906
+ /**
1907
+ * Full supersession + ownership path: identical to the pre-issue-#92 behavior
1908
+ * for the happy path. Called from the `failedChunks === 0` branch of
1909
+ * `ingestDocument`. Runs INSIDE the caller's tx.
1910
+ *
1911
+ * Returns `{ deletedSourceFactIds, insertedFacts }` so the caller can fire
1912
+ * post-commit hooks (embedding lifecycle) AFTER the transaction commits.
1913
+ * On the partial path the caller never invokes this method; the empty
1914
+ * arrays stay empty.
1915
+ */
1916
+ async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx) {
1917
+ const deletedSourceFactIds = [];
1918
+ const insertedFacts = [];
1919
+ deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
1920
+ const titleIndex = /* @__PURE__ */ new Map();
1921
+ const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
1922
+ for (const existing of existingFacts) {
1923
+ titleIndex.set(normalizeTitleKey(existing.title), {
1924
+ id: existing.id,
1925
+ okf_type: existing.okf_type ?? null
1926
+ });
1927
+ }
1928
+ let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
1929
+ let { mode, manifest } = ontologyState;
1930
+ const hostNodes = [];
1931
+ const rawEdgeRequests = [];
1932
+ const now = Date.now();
1933
+ for (const { facts, ontology_updates } of orderedChunkFacts) {
1934
+ if (mode === "emergent" && ontology_updates && this.ontologyService) {
1935
+ manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
1936
+ ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
1937
+ mode = ontologyState.mode;
1938
+ }
1939
+ for (const fact of facts) {
1940
+ const ontologyFact = fact;
1941
+ const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
1942
+ const id = generateId("fact_");
1943
+ hostNodes.push({
1944
+ id,
1945
+ type: ontologyFact.okf_type ?? "",
1946
+ title: fact.title,
1947
+ body: fact.body,
1948
+ // Forward the LLM-extracted tags/confidence through the
1949
+ // extracted-shape upsertGraphCore path so the entries row
1950
+ // stores them (search filterability, heal-candidate
1951
+ // selection, runReembed embedding-text signal). The public
1952
+ // WikiMemory.upsertGraph API doesn't accept tags/confidence
1953
+ // — host-supplied deterministic nodes default to [] and
1954
+ // 'certain' as documented.
1955
+ tags: fact.tags,
1956
+ confidence: fact.confidence
1957
+ });
1958
+ insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
1959
+ titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
1960
+ if (normalized.edges.length > 0) {
1961
+ rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
1962
+ }
1963
+ }
1964
+ }
1965
+ const hostEdges = [];
1966
+ for (const req of rawEdgeRequests) {
1967
+ const resolved = this.ontologyService?.resolveEdges(
1968
+ entityId,
1969
+ req.sourceId,
1970
+ req.sourceType,
1971
+ req.edges,
1972
+ manifest,
1973
+ titleIndex,
1974
+ now
1975
+ ) ?? [];
1976
+ for (const e of resolved) {
1977
+ hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
1978
+ }
1979
+ }
1980
+ await this.upsertGraphCore(
1981
+ entityId,
1982
+ { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
1983
+ tx,
1984
+ { strict: false }
1985
+ );
1986
+ return { deletedSourceFactIds, insertedFacts };
1987
+ }
1988
+ /**
1989
+ * Partial-commit path for `ingestDocument`. Inserts ONLY facts whose
1990
+ * normalized title is not already in the live set for this (entityId,
1991
+ * sourceRef). Does NOT call `entryRepo.softDeleteBySource` (no
1992
+ * supersession) and does NOT call `sourceRefIndexRepo.upsert` (no
1993
+ * ownership). Edges are NOT resolved on this path (no ontology-context
1994
+ * build, no `resolveEdges`, no `mergeEmergentUpdates`). Stale edges from
1995
+ * prior attempts are recovered on the next full run's supersession.
1996
+ *
1997
+ * Source-hash is stored as NULL on partial rows. `findLatestSourceHash`
1998
+ * reads from the most recently updated live row for the sourceRef, so
1999
+ * storing the incoming hash here would cause `hasChanged` to return
2000
+ * `false` on a same-hash retry — the failed chunks would never get a
2001
+ * second chance. With NULL, a retry sees `storedHash === null`, `hasChanged`
2002
+ * returns `true`, and the partial commit's sibling rows remain live
2003
+ * (deduped, not superseded) for the next attempt to extend.
2004
+ *
2005
+ * Runs INSIDE the caller's `tx`. Does not open a nested transaction.
2006
+ * Returns `{ inserted, skippedDuplicate }` for observability.
2007
+ */
2008
+ async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx) {
2009
+ const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
2010
+ const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
2011
+ const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
2012
+ let inserted = 0;
2013
+ let skippedDuplicate = 0;
2014
+ const now = Date.now();
2015
+ const insertedDescriptors = [];
2016
+ for (const fact of dedupedFacts) {
2017
+ const normalizedTitle = normalizeTitleKey(fact.title);
2018
+ if (liveTitles.has(normalizedTitle)) {
2019
+ skippedDuplicate++;
2020
+ continue;
2021
+ }
2022
+ liveTitles.add(normalizedTitle);
2023
+ const id = generateId("fact_");
2024
+ const wikiFact = {
2025
+ id,
2026
+ entity_id: entityId,
2027
+ title: fact.title,
2028
+ body: fact.body,
2029
+ tags: fact.tags,
2030
+ confidence: fact.confidence,
2031
+ source_type: "immutable_document",
2032
+ // source_hash: null on partial rows — see method docstring. The
2033
+ // full path stamps the actual hash; partial rows stay hash-less so
2034
+ // `hasChanged` keeps returning true for retries.
2035
+ source_hash: null,
2036
+ source_ref: sourceRef,
2037
+ created_at: now,
2038
+ updated_at: now,
2039
+ last_accessed_at: null,
2040
+ access_count: 0,
2041
+ deleted_at: null,
2042
+ okf_type: null
2043
+ };
2044
+ await this.entryRepo.upsert(wikiFact, tx);
2045
+ insertedDescriptors.push({
2046
+ id,
2047
+ entity_id: entityId,
2048
+ title: fact.title,
2049
+ body: fact.body,
2050
+ tags: JSON.stringify(fact.tags)
2051
+ });
2052
+ inserted++;
2053
+ }
2054
+ return { inserted, skippedDuplicate, insertedDescriptors };
2055
+ }
1600
2056
  };
1601
2057
 
1602
2058
  // src/repositories/BaseRepository.ts
@@ -1785,7 +2241,12 @@ var TRUNCATION_PATTERNS = [
1785
2241
  ];
1786
2242
  var EXCEEDS_LIMIT_PATTERN = /exceed[a-z]*[^.]{0,40}\b(model|context)?[ _-]?limit/i;
1787
2243
  function isTruncationError(err) {
1788
- const message = err instanceof Error ? err.message : String(err ?? "");
2244
+ let message;
2245
+ try {
2246
+ message = err instanceof Error ? err.message : String(err ?? "");
2247
+ } catch {
2248
+ return false;
2249
+ }
1789
2250
  if (EXCEEDS_LIMIT_PATTERN.test(message)) return false;
1790
2251
  return TRUNCATION_PATTERNS.some((pattern) => pattern.test(message));
1791
2252
  }
@@ -1889,6 +2350,30 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
1889
2350
  var HEAL_MAX_PROMPT_CHARS = 4e4;
1890
2351
  var HEAL_BATCH_SIZE = 25;
1891
2352
  var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
2353
+ var SKIP_ERROR_LOG_CHARS = 4096;
2354
+ var formatSkipError = (err) => {
2355
+ let base;
2356
+ let isErrorLike = false;
2357
+ try {
2358
+ isErrorLike = err instanceof Error;
2359
+ } catch {
2360
+ }
2361
+ if (isErrorLike || typeof err !== "object" && typeof err !== "function") {
2362
+ base = safeErrorToString(err);
2363
+ } else {
2364
+ try {
2365
+ const json = JSON.stringify(err);
2366
+ base = typeof json === "string" ? json : safeErrorToString(err);
2367
+ } catch {
2368
+ base = safeErrorToString(err);
2369
+ }
2370
+ }
2371
+ if (base.length > SKIP_ERROR_LOG_CHARS) {
2372
+ const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
2373
+ return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
2374
+ }
2375
+ return base;
2376
+ };
1892
2377
  var MaintenanceService = class {
1893
2378
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
1894
2379
  this.db = db;
@@ -2369,8 +2854,7 @@ var MaintenanceService = class {
2369
2854
  maxPromptChars: HEAL_MAX_PROMPT_CHARS,
2370
2855
  onSkip: (fact, err) => {
2371
2856
  console.warn(
2372
- `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded`,
2373
- err
2857
+ `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
2374
2858
  );
2375
2859
  }
2376
2860
  });
@@ -2508,8 +2992,7 @@ var MaintenanceService = class {
2508
2992
  maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
2509
2993
  onSkip: (fact, err) => {
2510
2994
  console.warn(
2511
- `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded`,
2512
- err
2995
+ `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
2513
2996
  );
2514
2997
  }
2515
2998
  });
@@ -3567,7 +4050,12 @@ var RetrievalService = class {
3567
4050
  }
3568
4051
  }
3569
4052
  } catch (rankerErr) {
3570
- const rankerError = rankerErr instanceof Error ? rankerErr : new Error(String(rankerErr));
4053
+ let isErrorLike = false;
4054
+ try {
4055
+ isErrorLike = rankerErr instanceof Error;
4056
+ } catch {
4057
+ }
4058
+ const rankerError = isErrorLike ? rankerErr : new Error(safeErrorToString(rankerErr));
3571
4059
  const policy = this.options.vectorRankerFallback ?? "js-cosine";
3572
4060
  this.options.onVectorRankerFallback?.({
3573
4061
  error: this._sanitizeRankerError(rankerError),
@@ -3679,12 +4167,21 @@ var RetrievalService = class {
3679
4167
  }
3680
4168
  }
3681
4169
  } catch (err) {
3682
- const error = err instanceof Error ? err : new Error(String(err));
4170
+ let isErrorLike = false;
4171
+ try {
4172
+ isErrorLike = err instanceof Error;
4173
+ } catch {
4174
+ }
4175
+ const error = isErrorLike ? err : new Error(safeErrorToString(err));
3683
4176
  if (rankerShouldRethrow) {
3684
4177
  throw error;
3685
4178
  }
3686
4179
  if (pendingRankerFallbackError) {
3687
- error.cause = pendingRankerFallbackError;
4180
+ try {
4181
+ error.cause = pendingRankerFallbackError;
4182
+ } catch {
4183
+ this.options.onRetrievalFallback?.(pendingRankerFallbackError);
4184
+ }
3688
4185
  pendingRankerFallbackError = void 0;
3689
4186
  }
3690
4187
  this.options.onRetrievalFallback?.(error);
@@ -3940,6 +4437,6 @@ var WriteService = class {
3940
4437
  }
3941
4438
  };
3942
4439
 
3943
- export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, validateInlineEdges, validateManifest };
3944
- //# sourceMappingURL=chunk-N2NXN6PO.mjs.map
3945
- //# sourceMappingURL=chunk-N2NXN6PO.mjs.map
4440
+ export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiIngestEmptyError, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, validateInlineEdges, validateManifest };
4441
+ //# sourceMappingURL=chunk-JQRA6BYQ.mjs.map
4442
+ //# sourceMappingURL=chunk-JQRA6BYQ.mjs.map