@equationalapplications/core-llm-wiki 5.4.0 → 5.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -125,6 +125,25 @@ var PrunePartialFailureError = class extends Error {
125
125
  }
126
126
  };
127
127
  var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
128
+ var WikiParseError = class extends Error {
129
+ constructor(message, opts) {
130
+ super(message);
131
+ this.name = "WikiParseError";
132
+ this.tier = opts.tier;
133
+ this.position = opts.position ?? null;
134
+ this.slice = opts.slice ?? "";
135
+ }
136
+ };
137
+ var WikiIngestEmptyError = class extends Error {
138
+ constructor(params) {
139
+ const summary = `All ${params.chunks} chunks failed for sourceRef "${params.sourceRef}"; see parseFailures for per-chunk detail`;
140
+ super(summary);
141
+ this.name = "WikiIngestEmptyError";
142
+ this.parseFailures = params.parseFailures;
143
+ this.sourceRef = params.sourceRef;
144
+ this.chunks = params.chunks;
145
+ }
146
+ };
128
147
 
129
148
  // src/utils/ids.ts
130
149
  var _injectedGetRandomValues = null;
@@ -945,24 +964,67 @@ var JobManager = class {
945
964
  };
946
965
 
947
966
  // src/utils/pure.ts
967
+ var MAX_REPAIR_CANDIDATES = 5;
968
+ function throwNoJsonFound(text, start) {
969
+ throw new WikiParseError(
970
+ "No JSON object/array found in LLM response",
971
+ { tier: "strict", position: start, slice: text }
972
+ );
973
+ }
948
974
  function parseJsonResponse(text) {
949
975
  const firstBrace = text.indexOf("{");
950
976
  const firstBracket = text.indexOf("[");
951
977
  let start;
952
978
  let openChar;
953
- let closeChar;
954
979
  const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
955
980
  if (useBrace) {
956
981
  start = firstBrace;
957
982
  openChar = "{";
958
- closeChar = "}";
959
983
  } else if (firstBracket !== -1) {
960
984
  start = firstBracket;
961
985
  openChar = "[";
962
- closeChar = "]";
963
986
  } else {
964
- throw new SyntaxError("No JSON object/array found in LLM response");
987
+ throwNoJsonFound(text, null);
988
+ }
989
+ const slice = scanJsonSlice(text, start, openChar);
990
+ if (slice !== null) {
991
+ try {
992
+ return JSON.parse(slice);
993
+ } catch {
994
+ }
965
995
  }
996
+ const repairOdd = containerAwareRepair(text, start, false);
997
+ if (repairOdd.success !== null && !repairOdd.ambiguous) {
998
+ return repairOdd.success;
999
+ }
1000
+ const repairEven = containerAwareRepair(text, start, true);
1001
+ if (repairOdd.success !== null && repairEven.success !== null) {
1002
+ const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
1003
+ return pick.success;
1004
+ }
1005
+ if (repairOdd.success !== null) {
1006
+ return repairOdd.success;
1007
+ }
1008
+ if (repairEven.success !== null) {
1009
+ return repairEven.success;
1010
+ }
1011
+ const repair = repairEven.failed !== null ? repairEven : repairOdd;
1012
+ if (repair.failed !== null) {
1013
+ throw new WikiParseError(
1014
+ `Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
1015
+ { tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
1016
+ );
1017
+ }
1018
+ if (slice === null) {
1019
+ throwNoJsonFound(text, start);
1020
+ }
1021
+ throw new WikiParseError(
1022
+ "No parsable JSON candidate found",
1023
+ { tier: "all", position: start, slice: text }
1024
+ );
1025
+ }
1026
+ function scanJsonSlice(text, start, openChar) {
1027
+ const closeChar = openChar === "{" ? "}" : "]";
966
1028
  let depth = 0;
967
1029
  let inString = false;
968
1030
  let escape = false;
@@ -994,8 +1056,193 @@ function parseJsonResponse(text) {
994
1056
  }
995
1057
  }
996
1058
  }
997
- if (end === -1) throw new SyntaxError("No JSON object/array found in LLM response");
998
- return JSON.parse(text.slice(start, end + 1));
1059
+ if (end === -1) return null;
1060
+ return safeSlice(text, start, end + 1);
1061
+ }
1062
+ function containerAwareRepair(text, start, closeOnEvenParity) {
1063
+ const stack = [];
1064
+ let inString = false;
1065
+ let stringRole = null;
1066
+ let escape = false;
1067
+ let i = start;
1068
+ let out = "";
1069
+ let bareQuoteCount = 0;
1070
+ let bestFailed = null;
1071
+ let escapes = 0;
1072
+ let ambiguous = false;
1073
+ let bestSuccess = null;
1074
+ let attempts = 0;
1075
+ function tryEmitCandidate(candidate) {
1076
+ attempts++;
1077
+ try {
1078
+ bestSuccess = JSON.parse(candidate);
1079
+ return true;
1080
+ } catch (err) {
1081
+ if (bestFailed === null) {
1082
+ bestFailed = {
1083
+ candidate,
1084
+ position: extractParsePosition(err),
1085
+ message: safeErrorToString(err)
1086
+ };
1087
+ }
1088
+ return false;
1089
+ }
1090
+ }
1091
+ while (i < text.length) {
1092
+ const ch = text[i];
1093
+ if (stack.length === 0 && !inString) {
1094
+ if (ch === "{" || ch === "[") {
1095
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
1096
+ out += ch;
1097
+ }
1098
+ i++;
1099
+ continue;
1100
+ }
1101
+ if (escape) {
1102
+ out += ch;
1103
+ escape = false;
1104
+ i++;
1105
+ continue;
1106
+ }
1107
+ if (inString && ch === "\\") {
1108
+ out += ch;
1109
+ escape = true;
1110
+ i++;
1111
+ continue;
1112
+ }
1113
+ if (inString) {
1114
+ const code = ch.charCodeAt(0);
1115
+ if (code < 32) {
1116
+ out += controlCharEscape(code);
1117
+ i++;
1118
+ continue;
1119
+ }
1120
+ }
1121
+ if (ch === '"') {
1122
+ let j = i + 1;
1123
+ while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
1124
+ j++;
1125
+ }
1126
+ const next = j < text.length ? text[j] : "";
1127
+ if (!inString) {
1128
+ if (next === "}" || next === "]") {
1129
+ if (j > i + 1) out += text.slice(i + 1, j);
1130
+ out += next;
1131
+ i = j + 1;
1132
+ stack.pop();
1133
+ if (stack.length === 0) {
1134
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
1135
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
1136
+ out = "";
1137
+ }
1138
+ continue;
1139
+ }
1140
+ const top = stack[stack.length - 1];
1141
+ stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
1142
+ out += ch;
1143
+ inString = true;
1144
+ i++;
1145
+ continue;
1146
+ }
1147
+ const commaOnly = next === ",";
1148
+ const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
1149
+ const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
1150
+ if (commaOnly && bareQuoteCount > 0) ambiguous = true;
1151
+ const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
1152
+ const isClosing = isKeyClose || isValueClose || shouldCommaClose;
1153
+ if (isClosing) {
1154
+ out += ch;
1155
+ inString = false;
1156
+ const topFrame = stack[stack.length - 1];
1157
+ if (topFrame && topFrame.container === "object") {
1158
+ topFrame.expectKey = stringRole === "value";
1159
+ }
1160
+ stringRole = null;
1161
+ bareQuoteCount = 0;
1162
+ i++;
1163
+ continue;
1164
+ }
1165
+ out += "\\" + ch;
1166
+ bareQuoteCount++;
1167
+ escapes++;
1168
+ i++;
1169
+ continue;
1170
+ }
1171
+ if (!inString && (ch === "{" || ch === "[")) {
1172
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
1173
+ out += ch;
1174
+ i++;
1175
+ continue;
1176
+ }
1177
+ if (!inString && ch === ",") {
1178
+ const topFrame = stack[stack.length - 1];
1179
+ if (topFrame && topFrame.container === "object") {
1180
+ topFrame.expectKey = true;
1181
+ }
1182
+ out += ch;
1183
+ i++;
1184
+ continue;
1185
+ }
1186
+ if (!inString && (ch === "}" || ch === "]")) {
1187
+ out += ch;
1188
+ stack.pop();
1189
+ if (stack.length === 0) {
1190
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
1191
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
1192
+ out = "";
1193
+ }
1194
+ i++;
1195
+ continue;
1196
+ }
1197
+ out += ch;
1198
+ i++;
1199
+ }
1200
+ return { success: null, failed: bestFailed, escapes, ambiguous };
1201
+ }
1202
+ function controlCharEscape(code) {
1203
+ switch (code) {
1204
+ case 10:
1205
+ return "\\n";
1206
+ case 9:
1207
+ return "\\t";
1208
+ case 13:
1209
+ return "\\r";
1210
+ case 8:
1211
+ return "\\b";
1212
+ case 12:
1213
+ return "\\f";
1214
+ default:
1215
+ return "\\u" + code.toString(16).padStart(4, "0");
1216
+ }
1217
+ }
1218
+ function extractParsePosition(err) {
1219
+ const match = /position\s+(\d+)/.exec(safeErrorToString(err));
1220
+ return match ? Number(match[1]) : null;
1221
+ }
1222
+ function safeErrorToString(e) {
1223
+ if (e instanceof Error) {
1224
+ const msg = readErrorField(e, "message");
1225
+ if (typeof msg === "string" && msg.length > 0) return msg;
1226
+ const name = readErrorField(e, "name");
1227
+ if (typeof name === "string" && name.length > 0) return name;
1228
+ return "[Error]";
1229
+ }
1230
+ try {
1231
+ return String(e);
1232
+ } catch {
1233
+ try {
1234
+ return Object.prototype.toString.call(e);
1235
+ } catch {
1236
+ return "[unstringifiable error]";
1237
+ }
1238
+ }
1239
+ }
1240
+ function readErrorField(e, key) {
1241
+ try {
1242
+ return e[key];
1243
+ } catch {
1244
+ return void 0;
1245
+ }
999
1246
  }
1000
1247
  function sanitizeRankerError(err, sanitizeRankerErrors) {
1001
1248
  if (sanitizeRankerErrors === false) {
@@ -1180,7 +1427,7 @@ Return ONLY a valid JSON object matching this schema:
1180
1427
  {
1181
1428
  "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
1182
1429
  }
1183
- Extract verbatim factual content. Do not return markdown, just raw JSON.`;
1430
+ Extract verbatim factual content. JSON escaping rules: every literal " character in the source must be escaped as \\" inside any JSON string body, and every literal newline as \\n. Source prose containing quotes (e.g. a worked example with "...") must still be reproduced exactly \u2014 re-escape, do not omit. Do not return markdown, just raw JSON.`;
1184
1431
  var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
1185
1432
  Return ONLY a valid JSON object matching this schema:
1186
1433
  {
@@ -1189,7 +1436,7 @@ Return ONLY a valid JSON object matching this schema:
1189
1436
  ]
1190
1437
  }
1191
1438
  If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
1192
- Do not return markdown, just raw JSON.`;
1439
+ When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
1193
1440
 
1194
1441
  // src/services/PromptService.ts
1195
1442
  var PromptService = class {
@@ -1302,6 +1549,16 @@ var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
1302
1549
  var DEFAULT_CHUNK_OVERLAP = 400;
1303
1550
 
1304
1551
  // src/services/IngestionService.ts
1552
+ function zeroChunkResult(duplicateOf) {
1553
+ const result = {
1554
+ truncated: false,
1555
+ chunks: 0,
1556
+ ingestedChunks: 0,
1557
+ failedChunks: 0
1558
+ };
1559
+ if (duplicateOf !== void 0) result.duplicateOf = duplicateOf;
1560
+ return result;
1561
+ }
1305
1562
  var IngestionService = class {
1306
1563
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
1307
1564
  this.db = db;
@@ -1342,113 +1599,110 @@ var IngestionService = class {
1342
1599
  if (onDuplicateHash === "throw") {
1343
1600
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
1344
1601
  }
1345
- return { truncated: false, chunks: 0, duplicateOf: canonical };
1602
+ return zeroChunkResult(canonical);
1346
1603
  }
1347
1604
  }
1348
1605
  const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
1349
- if (chunks.length === 0) return { truncated: false, chunks: 0 };
1606
+ if (chunks.length === 0) return zeroChunkResult();
1607
+ const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
1350
1608
  const chunkResults = await withConcurrency(
1351
- chunks.map((chunk) => async () => {
1352
- const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
1609
+ chunks.map((chunk, chunkIndex) => async () => {
1353
1610
  const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
1354
1611
  chunk,
1355
1612
  params.promptOverride,
1356
1613
  ontologyContext
1357
1614
  );
1358
- const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
1359
- const result = parseJsonResponse(responseText);
1360
- return {
1361
- facts: (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null),
1362
- ontology_updates: result.ontology_updates
1363
- };
1615
+ try {
1616
+ const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
1617
+ const result2 = parseJsonResponse(responseText);
1618
+ return {
1619
+ status: "ok",
1620
+ facts: (Array.isArray(result2.facts) ? result2.facts : []).map(validateFact).filter((f) => f !== null),
1621
+ ontology_updates: result2.ontology_updates
1622
+ };
1623
+ } catch (e) {
1624
+ const failure = e instanceof WikiParseError ? {
1625
+ chunkIndex,
1626
+ sourceRef,
1627
+ source: "parse",
1628
+ tier: e.tier,
1629
+ position: e.position,
1630
+ message: e.message
1631
+ } : {
1632
+ chunkIndex,
1633
+ sourceRef,
1634
+ source: "llm",
1635
+ // `safeErrorToString` is a non-throwing coercion: an LLM
1636
+ // provider may reject with a non-Error value, and even
1637
+ // `String(e)` itself can throw when `e.toString()` throws.
1638
+ // Letting such an exception escape this catch would cause
1639
+ // `withConcurrency` to reject, discarding every sibling
1640
+ // chunk's results — the exact failure mode this per-chunk
1641
+ // try/catch exists to prevent. Keeping the value in the
1642
+ // `ChunkFailure` preserves the typed diagnostic for
1643
+ // callers (see parseFailures) without unwinding the loop.
1644
+ position: null,
1645
+ message: safeErrorToString(e)
1646
+ };
1647
+ const total = chunks.length;
1648
+ const tags = [];
1649
+ if (failure.tier) tags.push(`tier=${failure.tier}`);
1650
+ if (failure.position !== null) tags.push(`position=${failure.position}`);
1651
+ const tagsSuffix = tags.length > 0 ? `; ${tags.join(" ")}` : "";
1652
+ console.warn(
1653
+ `[WikiMemory] ingest chunk ${chunkIndex + 1}/${total} ${failure.source} failed (sourceRef=${sourceRef}${tagsSuffix})`
1654
+ );
1655
+ return { status: "failed", error: failure };
1656
+ }
1364
1657
  }),
1365
1658
  chunkConcurrency
1366
1659
  );
1660
+ let ingestedChunks = 0;
1661
+ let failedChunks = 0;
1662
+ const failures = [];
1367
1663
  const seen = /* @__PURE__ */ new Set();
1368
1664
  const orderedChunkFacts = [];
1369
- for (const chunkResult of chunkResults) {
1665
+ for (const slot of chunkResults) {
1666
+ if (slot.status === "failed") {
1667
+ failedChunks++;
1668
+ failures.push(slot.error);
1669
+ continue;
1670
+ }
1671
+ ingestedChunks++;
1370
1672
  const dedupedFacts = [];
1371
- for (const fact of chunkResult.facts) {
1673
+ for (const fact of slot.facts) {
1372
1674
  const normalizedTitle = normalizeTitleKey(fact.title);
1373
1675
  if (!seen.has(normalizedTitle)) {
1374
1676
  seen.add(normalizedTitle);
1375
1677
  dedupedFacts.push(fact);
1376
1678
  }
1377
1679
  }
1378
- orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: chunkResult.ontology_updates });
1680
+ orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: slot.ontology_updates });
1681
+ }
1682
+ if (failedChunks === chunks.length) {
1683
+ throw new WikiIngestEmptyError({
1684
+ parseFailures: failures,
1685
+ sourceRef,
1686
+ chunks: chunks.length
1687
+ });
1379
1688
  }
1380
- const now = Date.now();
1381
1689
  const insertedFacts = [];
1382
1690
  const deletedSourceFactIds = [];
1383
1691
  try {
1384
- await this.db.withTransactionAsync(async (tx) => {
1385
- deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
1386
- const titleIndex = /* @__PURE__ */ new Map();
1387
- const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
1388
- for (const existing of existingFacts) {
1389
- titleIndex.set(normalizeTitleKey(existing.title), {
1390
- id: existing.id,
1391
- okf_type: existing.okf_type ?? null
1392
- });
1393
- }
1394
- let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
1395
- let { mode, manifest } = ontologyState;
1396
- const hostNodes = [];
1397
- const rawEdgeRequests = [];
1398
- for (const { facts, ontology_updates } of orderedChunkFacts) {
1399
- if (mode === "emergent" && ontology_updates && this.ontologyService) {
1400
- manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
1401
- ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
1402
- mode = ontologyState.mode;
1403
- }
1404
- for (const fact of facts) {
1405
- const ontologyFact = fact;
1406
- const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
1407
- const id = generateId("fact_");
1408
- hostNodes.push({
1409
- id,
1410
- type: ontologyFact.okf_type ?? "",
1411
- title: fact.title,
1412
- body: fact.body,
1413
- // Forward the LLM-extracted tags/confidence through the
1414
- // extracted-shape upsertGraphCore path so the entries row
1415
- // stores them (search filterability, heal-candidate
1416
- // selection, runReembed embedding-text signal). The public
1417
- // WikiMemory.upsertGraph API doesn't accept tags/confidence
1418
- // — host-supplied deterministic nodes default to [] and
1419
- // 'certain' as documented.
1420
- tags: fact.tags,
1421
- confidence: fact.confidence
1422
- });
1423
- insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
1424
- titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
1425
- if (normalized.edges.length > 0) {
1426
- rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
1427
- }
1428
- }
1429
- }
1430
- const hostEdges = [];
1431
- for (const req of rawEdgeRequests) {
1432
- const resolved = this.ontologyService?.resolveEdges(
1433
- entityId,
1434
- req.sourceId,
1435
- req.sourceType,
1436
- req.edges,
1437
- manifest,
1438
- titleIndex,
1439
- now
1440
- ) ?? [];
1441
- for (const e of resolved) {
1442
- hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
1443
- }
1444
- }
1445
- await this.upsertGraphCore(
1446
- entityId,
1447
- { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
1448
- tx,
1449
- { strict: false }
1450
- );
1451
- });
1692
+ if (failedChunks === 0) {
1693
+ const fullResult = await this.db.withTransactionAsync(async (tx) => {
1694
+ return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx);
1695
+ });
1696
+ deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
1697
+ insertedFacts.push(...fullResult.insertedFacts);
1698
+ } else {
1699
+ const partialResult = await this.db.withTransactionAsync(async (tx) => {
1700
+ const flat = [];
1701
+ for (const slot of orderedChunkFacts) flat.push(...slot.facts);
1702
+ return await this.appendPartialFacts(entityId, sourceRef, flat, tx);
1703
+ });
1704
+ insertedFacts.push(...partialResult.insertedDescriptors);
1705
+ }
1452
1706
  } catch (err) {
1453
1707
  const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
1454
1708
  if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
@@ -1457,22 +1711,31 @@ var IngestionService = class {
1457
1711
  if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
1458
1712
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
1459
1713
  }
1460
- return { truncated: false, chunks: 0, duplicateOf: canonical };
1714
+ return zeroChunkResult(canonical);
1461
1715
  }
1462
1716
  await this.searchService.sync(entityId);
1463
- const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
1464
- for (const factId of uniqueDeletedSourceFactIds) {
1465
- try {
1466
- await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
1467
- } catch (hookErr) {
1468
- console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
1717
+ if (failedChunks === 0) {
1718
+ const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
1719
+ for (const factId of uniqueDeletedSourceFactIds) {
1720
+ try {
1721
+ await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
1722
+ } catch (hookErr) {
1723
+ console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
1724
+ }
1469
1725
  }
1470
1726
  }
1471
1727
  for (const fact of insertedFacts) {
1472
1728
  await this.embeddingService.embedFact(fact);
1473
1729
  }
1474
1730
  this.searchService.evictCache(entityId);
1475
- return { truncated, chunks: chunks.length };
1731
+ const result = {
1732
+ truncated,
1733
+ chunks: chunks.length,
1734
+ ingestedChunks,
1735
+ failedChunks
1736
+ };
1737
+ if (failedChunks > 0) result.parseFailures = failures;
1738
+ return result;
1476
1739
  } finally {
1477
1740
  releaseIngestLocks();
1478
1741
  }
@@ -1597,6 +1860,156 @@ var IngestionService = class {
1597
1860
  superseded: deletedFactIds.length + deletedEdgeCount
1598
1861
  };
1599
1862
  }
1863
+ /**
1864
+ * Full supersession + ownership path: identical to the pre-issue-#92 behavior
1865
+ * for the happy path. Called from the `failedChunks === 0` branch of
1866
+ * `ingestDocument`. Runs INSIDE the caller's tx.
1867
+ *
1868
+ * Returns `{ deletedSourceFactIds, insertedFacts }` so the caller can fire
1869
+ * post-commit hooks (embedding lifecycle) AFTER the transaction commits.
1870
+ * On the partial path the caller never invokes this method; the empty
1871
+ * arrays stay empty.
1872
+ */
1873
+ async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx) {
1874
+ const deletedSourceFactIds = [];
1875
+ const insertedFacts = [];
1876
+ deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
1877
+ const titleIndex = /* @__PURE__ */ new Map();
1878
+ const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
1879
+ for (const existing of existingFacts) {
1880
+ titleIndex.set(normalizeTitleKey(existing.title), {
1881
+ id: existing.id,
1882
+ okf_type: existing.okf_type ?? null
1883
+ });
1884
+ }
1885
+ let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
1886
+ let { mode, manifest } = ontologyState;
1887
+ const hostNodes = [];
1888
+ const rawEdgeRequests = [];
1889
+ const now = Date.now();
1890
+ for (const { facts, ontology_updates } of orderedChunkFacts) {
1891
+ if (mode === "emergent" && ontology_updates && this.ontologyService) {
1892
+ manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
1893
+ ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
1894
+ mode = ontologyState.mode;
1895
+ }
1896
+ for (const fact of facts) {
1897
+ const ontologyFact = fact;
1898
+ const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
1899
+ const id = generateId("fact_");
1900
+ hostNodes.push({
1901
+ id,
1902
+ type: ontologyFact.okf_type ?? "",
1903
+ title: fact.title,
1904
+ body: fact.body,
1905
+ // Forward the LLM-extracted tags/confidence through the
1906
+ // extracted-shape upsertGraphCore path so the entries row
1907
+ // stores them (search filterability, heal-candidate
1908
+ // selection, runReembed embedding-text signal). The public
1909
+ // WikiMemory.upsertGraph API doesn't accept tags/confidence
1910
+ // — host-supplied deterministic nodes default to [] and
1911
+ // 'certain' as documented.
1912
+ tags: fact.tags,
1913
+ confidence: fact.confidence
1914
+ });
1915
+ insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
1916
+ titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
1917
+ if (normalized.edges.length > 0) {
1918
+ rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
1919
+ }
1920
+ }
1921
+ }
1922
+ const hostEdges = [];
1923
+ for (const req of rawEdgeRequests) {
1924
+ const resolved = this.ontologyService?.resolveEdges(
1925
+ entityId,
1926
+ req.sourceId,
1927
+ req.sourceType,
1928
+ req.edges,
1929
+ manifest,
1930
+ titleIndex,
1931
+ now
1932
+ ) ?? [];
1933
+ for (const e of resolved) {
1934
+ hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
1935
+ }
1936
+ }
1937
+ await this.upsertGraphCore(
1938
+ entityId,
1939
+ { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
1940
+ tx,
1941
+ { strict: false }
1942
+ );
1943
+ return { deletedSourceFactIds, insertedFacts };
1944
+ }
1945
+ /**
1946
+ * Partial-commit path for `ingestDocument`. Inserts ONLY facts whose
1947
+ * normalized title is not already in the live set for this (entityId,
1948
+ * sourceRef). Does NOT call `entryRepo.softDeleteBySource` (no
1949
+ * supersession) and does NOT call `sourceRefIndexRepo.upsert` (no
1950
+ * ownership). Edges are NOT resolved on this path (no ontology-context
1951
+ * build, no `resolveEdges`, no `mergeEmergentUpdates`). Stale edges from
1952
+ * prior attempts are recovered on the next full run's supersession.
1953
+ *
1954
+ * Source-hash is stored as NULL on partial rows. `findLatestSourceHash`
1955
+ * reads from the most recently updated live row for the sourceRef, so
1956
+ * storing the incoming hash here would cause `hasChanged` to return
1957
+ * `false` on a same-hash retry — the failed chunks would never get a
1958
+ * second chance. With NULL, a retry sees `storedHash === null`, `hasChanged`
1959
+ * returns `true`, and the partial commit's sibling rows remain live
1960
+ * (deduped, not superseded) for the next attempt to extend.
1961
+ *
1962
+ * Runs INSIDE the caller's `tx`. Does not open a nested transaction.
1963
+ * Returns `{ inserted, skippedDuplicate }` for observability.
1964
+ */
1965
+ async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx) {
1966
+ const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
1967
+ const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
1968
+ const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
1969
+ let inserted = 0;
1970
+ let skippedDuplicate = 0;
1971
+ const now = Date.now();
1972
+ const insertedDescriptors = [];
1973
+ for (const fact of dedupedFacts) {
1974
+ const normalizedTitle = normalizeTitleKey(fact.title);
1975
+ if (liveTitles.has(normalizedTitle)) {
1976
+ skippedDuplicate++;
1977
+ continue;
1978
+ }
1979
+ liveTitles.add(normalizedTitle);
1980
+ const id = generateId("fact_");
1981
+ const wikiFact = {
1982
+ id,
1983
+ entity_id: entityId,
1984
+ title: fact.title,
1985
+ body: fact.body,
1986
+ tags: fact.tags,
1987
+ confidence: fact.confidence,
1988
+ source_type: "immutable_document",
1989
+ // source_hash: null on partial rows — see method docstring. The
1990
+ // full path stamps the actual hash; partial rows stay hash-less so
1991
+ // `hasChanged` keeps returning true for retries.
1992
+ source_hash: null,
1993
+ source_ref: sourceRef,
1994
+ created_at: now,
1995
+ updated_at: now,
1996
+ last_accessed_at: null,
1997
+ access_count: 0,
1998
+ deleted_at: null,
1999
+ okf_type: null
2000
+ };
2001
+ await this.entryRepo.upsert(wikiFact, tx);
2002
+ insertedDescriptors.push({
2003
+ id,
2004
+ entity_id: entityId,
2005
+ title: fact.title,
2006
+ body: fact.body,
2007
+ tags: JSON.stringify(fact.tags)
2008
+ });
2009
+ inserted++;
2010
+ }
2011
+ return { inserted, skippedDuplicate, insertedDescriptors };
2012
+ }
1600
2013
  };
1601
2014
 
1602
2015
  // src/repositories/BaseRepository.ts
@@ -1889,6 +2302,25 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
1889
2302
  var HEAL_MAX_PROMPT_CHARS = 4e4;
1890
2303
  var HEAL_BATCH_SIZE = 25;
1891
2304
  var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
2305
+ var SKIP_ERROR_LOG_CHARS = 4096;
2306
+ var formatSkipError = (err) => {
2307
+ let base;
2308
+ if (err instanceof Error || typeof err !== "object" && typeof err !== "function") {
2309
+ base = safeErrorToString(err);
2310
+ } else {
2311
+ try {
2312
+ const json = JSON.stringify(err);
2313
+ base = typeof json === "string" ? json : safeErrorToString(err);
2314
+ } catch {
2315
+ base = safeErrorToString(err);
2316
+ }
2317
+ }
2318
+ if (base.length > SKIP_ERROR_LOG_CHARS) {
2319
+ const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
2320
+ return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
2321
+ }
2322
+ return base;
2323
+ };
1892
2324
  var MaintenanceService = class {
1893
2325
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
1894
2326
  this.db = db;
@@ -2369,8 +2801,7 @@ var MaintenanceService = class {
2369
2801
  maxPromptChars: HEAL_MAX_PROMPT_CHARS,
2370
2802
  onSkip: (fact, err) => {
2371
2803
  console.warn(
2372
- `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded`,
2373
- err
2804
+ `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
2374
2805
  );
2375
2806
  }
2376
2807
  });
@@ -2508,8 +2939,7 @@ var MaintenanceService = class {
2508
2939
  maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
2509
2940
  onSkip: (fact, err) => {
2510
2941
  console.warn(
2511
- `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded`,
2512
- err
2942
+ `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
2513
2943
  );
2514
2944
  }
2515
2945
  });
@@ -3940,6 +4370,6 @@ var WriteService = class {
3940
4370
  }
3941
4371
  };
3942
4372
 
3943
- export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, validateInlineEdges, validateManifest };
3944
- //# sourceMappingURL=chunk-N2NXN6PO.mjs.map
3945
- //# sourceMappingURL=chunk-N2NXN6PO.mjs.map
4373
+ export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiIngestEmptyError, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, validateInlineEdges, validateManifest };
4374
+ //# sourceMappingURL=chunk-QVY5DFJV.mjs.map
4375
+ //# sourceMappingURL=chunk-QVY5DFJV.mjs.map