@equationalapplications/core-llm-wiki 5.3.1 → 5.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,5 @@
1
1
  import MiniSearch from 'minisearch';
2
+ import { latestVerified } from '@equationalapplications/core-okf';
2
3
 
3
4
  var __typeError = (msg) => {
4
5
  throw TypeError(msg);
@@ -124,6 +125,25 @@ var PrunePartialFailureError = class extends Error {
124
125
  }
125
126
  };
126
127
  var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
128
+ var WikiParseError = class extends Error {
129
+ constructor(message, opts) {
130
+ super(message);
131
+ this.name = "WikiParseError";
132
+ this.tier = opts.tier;
133
+ this.position = opts.position ?? null;
134
+ this.slice = opts.slice ?? "";
135
+ }
136
+ };
137
+ var WikiIngestEmptyError = class extends Error {
138
+ constructor(params) {
139
+ const summary = `All ${params.chunks} chunks failed for sourceRef "${params.sourceRef}"; see parseFailures for per-chunk detail`;
140
+ super(summary);
141
+ this.name = "WikiIngestEmptyError";
142
+ this.parseFailures = params.parseFailures;
143
+ this.sourceRef = params.sourceRef;
144
+ this.chunks = params.chunks;
145
+ }
146
+ };
127
147
 
128
148
  // src/utils/ids.ts
129
149
  var _injectedGetRandomValues = null;
@@ -944,24 +964,67 @@ var JobManager = class {
944
964
  };
945
965
 
946
966
  // src/utils/pure.ts
967
+ var MAX_REPAIR_CANDIDATES = 5;
968
+ function throwNoJsonFound(text, start) {
969
+ throw new WikiParseError(
970
+ "No JSON object/array found in LLM response",
971
+ { tier: "strict", position: start, slice: text }
972
+ );
973
+ }
947
974
  function parseJsonResponse(text) {
948
975
  const firstBrace = text.indexOf("{");
949
976
  const firstBracket = text.indexOf("[");
950
977
  let start;
951
978
  let openChar;
952
- let closeChar;
953
979
  const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
954
980
  if (useBrace) {
955
981
  start = firstBrace;
956
982
  openChar = "{";
957
- closeChar = "}";
958
983
  } else if (firstBracket !== -1) {
959
984
  start = firstBracket;
960
985
  openChar = "[";
961
- closeChar = "]";
962
986
  } else {
963
- throw new SyntaxError("No JSON object/array found in LLM response");
987
+ throwNoJsonFound(text, null);
988
+ }
989
+ const slice = scanJsonSlice(text, start, openChar);
990
+ if (slice !== null) {
991
+ try {
992
+ return JSON.parse(slice);
993
+ } catch {
994
+ }
964
995
  }
996
+ const repairOdd = containerAwareRepair(text, start, false);
997
+ if (repairOdd.success !== null && !repairOdd.ambiguous) {
998
+ return repairOdd.success;
999
+ }
1000
+ const repairEven = containerAwareRepair(text, start, true);
1001
+ if (repairOdd.success !== null && repairEven.success !== null) {
1002
+ const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
1003
+ return pick.success;
1004
+ }
1005
+ if (repairOdd.success !== null) {
1006
+ return repairOdd.success;
1007
+ }
1008
+ if (repairEven.success !== null) {
1009
+ return repairEven.success;
1010
+ }
1011
+ const repair = repairEven.failed !== null ? repairEven : repairOdd;
1012
+ if (repair.failed !== null) {
1013
+ throw new WikiParseError(
1014
+ `Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
1015
+ { tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
1016
+ );
1017
+ }
1018
+ if (slice === null) {
1019
+ throwNoJsonFound(text, start);
1020
+ }
1021
+ throw new WikiParseError(
1022
+ "No parsable JSON candidate found",
1023
+ { tier: "all", position: start, slice: text }
1024
+ );
1025
+ }
1026
+ function scanJsonSlice(text, start, openChar) {
1027
+ const closeChar = openChar === "{" ? "}" : "]";
965
1028
  let depth = 0;
966
1029
  let inString = false;
967
1030
  let escape = false;
@@ -993,8 +1056,193 @@ function parseJsonResponse(text) {
993
1056
  }
994
1057
  }
995
1058
  }
996
- if (end === -1) throw new SyntaxError("No JSON object/array found in LLM response");
997
- return JSON.parse(text.slice(start, end + 1));
1059
+ if (end === -1) return null;
1060
+ return safeSlice(text, start, end + 1);
1061
+ }
1062
+ function containerAwareRepair(text, start, closeOnEvenParity) {
1063
+ const stack = [];
1064
+ let inString = false;
1065
+ let stringRole = null;
1066
+ let escape = false;
1067
+ let i = start;
1068
+ let out = "";
1069
+ let bareQuoteCount = 0;
1070
+ let bestFailed = null;
1071
+ let escapes = 0;
1072
+ let ambiguous = false;
1073
+ let bestSuccess = null;
1074
+ let attempts = 0;
1075
+ function tryEmitCandidate(candidate) {
1076
+ attempts++;
1077
+ try {
1078
+ bestSuccess = JSON.parse(candidate);
1079
+ return true;
1080
+ } catch (err) {
1081
+ if (bestFailed === null) {
1082
+ bestFailed = {
1083
+ candidate,
1084
+ position: extractParsePosition(err),
1085
+ message: safeErrorToString(err)
1086
+ };
1087
+ }
1088
+ return false;
1089
+ }
1090
+ }
1091
+ while (i < text.length) {
1092
+ const ch = text[i];
1093
+ if (stack.length === 0 && !inString) {
1094
+ if (ch === "{" || ch === "[") {
1095
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
1096
+ out += ch;
1097
+ }
1098
+ i++;
1099
+ continue;
1100
+ }
1101
+ if (escape) {
1102
+ out += ch;
1103
+ escape = false;
1104
+ i++;
1105
+ continue;
1106
+ }
1107
+ if (inString && ch === "\\") {
1108
+ out += ch;
1109
+ escape = true;
1110
+ i++;
1111
+ continue;
1112
+ }
1113
+ if (inString) {
1114
+ const code = ch.charCodeAt(0);
1115
+ if (code < 32) {
1116
+ out += controlCharEscape(code);
1117
+ i++;
1118
+ continue;
1119
+ }
1120
+ }
1121
+ if (ch === '"') {
1122
+ let j = i + 1;
1123
+ while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
1124
+ j++;
1125
+ }
1126
+ const next = j < text.length ? text[j] : "";
1127
+ if (!inString) {
1128
+ if (next === "}" || next === "]") {
1129
+ if (j > i + 1) out += text.slice(i + 1, j);
1130
+ out += next;
1131
+ i = j + 1;
1132
+ stack.pop();
1133
+ if (stack.length === 0) {
1134
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
1135
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
1136
+ out = "";
1137
+ }
1138
+ continue;
1139
+ }
1140
+ const top = stack[stack.length - 1];
1141
+ stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
1142
+ out += ch;
1143
+ inString = true;
1144
+ i++;
1145
+ continue;
1146
+ }
1147
+ const commaOnly = next === ",";
1148
+ const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
1149
+ const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
1150
+ if (commaOnly && bareQuoteCount > 0) ambiguous = true;
1151
+ const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
1152
+ const isClosing = isKeyClose || isValueClose || shouldCommaClose;
1153
+ if (isClosing) {
1154
+ out += ch;
1155
+ inString = false;
1156
+ const topFrame = stack[stack.length - 1];
1157
+ if (topFrame && topFrame.container === "object") {
1158
+ topFrame.expectKey = stringRole === "value";
1159
+ }
1160
+ stringRole = null;
1161
+ bareQuoteCount = 0;
1162
+ i++;
1163
+ continue;
1164
+ }
1165
+ out += "\\" + ch;
1166
+ bareQuoteCount++;
1167
+ escapes++;
1168
+ i++;
1169
+ continue;
1170
+ }
1171
+ if (!inString && (ch === "{" || ch === "[")) {
1172
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
1173
+ out += ch;
1174
+ i++;
1175
+ continue;
1176
+ }
1177
+ if (!inString && ch === ",") {
1178
+ const topFrame = stack[stack.length - 1];
1179
+ if (topFrame && topFrame.container === "object") {
1180
+ topFrame.expectKey = true;
1181
+ }
1182
+ out += ch;
1183
+ i++;
1184
+ continue;
1185
+ }
1186
+ if (!inString && (ch === "}" || ch === "]")) {
1187
+ out += ch;
1188
+ stack.pop();
1189
+ if (stack.length === 0) {
1190
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
1191
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
1192
+ out = "";
1193
+ }
1194
+ i++;
1195
+ continue;
1196
+ }
1197
+ out += ch;
1198
+ i++;
1199
+ }
1200
+ return { success: null, failed: bestFailed, escapes, ambiguous };
1201
+ }
1202
+ function controlCharEscape(code) {
1203
+ switch (code) {
1204
+ case 10:
1205
+ return "\\n";
1206
+ case 9:
1207
+ return "\\t";
1208
+ case 13:
1209
+ return "\\r";
1210
+ case 8:
1211
+ return "\\b";
1212
+ case 12:
1213
+ return "\\f";
1214
+ default:
1215
+ return "\\u" + code.toString(16).padStart(4, "0");
1216
+ }
1217
+ }
1218
+ function extractParsePosition(err) {
1219
+ const match = /position\s+(\d+)/.exec(safeErrorToString(err));
1220
+ return match ? Number(match[1]) : null;
1221
+ }
1222
+ function safeErrorToString(e) {
1223
+ if (e instanceof Error) {
1224
+ const msg = readErrorField(e, "message");
1225
+ if (typeof msg === "string" && msg.length > 0) return msg;
1226
+ const name = readErrorField(e, "name");
1227
+ if (typeof name === "string" && name.length > 0) return name;
1228
+ return "[Error]";
1229
+ }
1230
+ try {
1231
+ return String(e);
1232
+ } catch {
1233
+ try {
1234
+ return Object.prototype.toString.call(e);
1235
+ } catch {
1236
+ return "[unstringifiable error]";
1237
+ }
1238
+ }
1239
+ }
1240
+ function readErrorField(e, key) {
1241
+ try {
1242
+ return e[key];
1243
+ } catch {
1244
+ return void 0;
1245
+ }
998
1246
  }
999
1247
  function sanitizeRankerError(err, sanitizeRankerErrors) {
1000
1248
  if (sanitizeRankerErrors === false) {
@@ -1179,7 +1427,7 @@ Return ONLY a valid JSON object matching this schema:
1179
1427
  {
1180
1428
  "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
1181
1429
  }
1182
- Extract verbatim factual content. Do not return markdown, just raw JSON.`;
1430
+ Extract verbatim factual content. JSON escaping rules: every literal " character in the source must be escaped as \\" inside any JSON string body, and every literal newline as \\n. Source prose containing quotes (e.g. a worked example with "...") must still be reproduced exactly \u2014 re-escape, do not omit. Do not return markdown, just raw JSON.`;
1183
1431
  var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
1184
1432
  Return ONLY a valid JSON object matching this schema:
1185
1433
  {
@@ -1188,7 +1436,7 @@ Return ONLY a valid JSON object matching this schema:
1188
1436
  ]
1189
1437
  }
1190
1438
  If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
1191
- Do not return markdown, just raw JSON.`;
1439
+ When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
1192
1440
 
1193
1441
  // src/services/PromptService.ts
1194
1442
  var PromptService = class {
@@ -1301,6 +1549,16 @@ var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
1301
1549
  var DEFAULT_CHUNK_OVERLAP = 400;
1302
1550
 
1303
1551
  // src/services/IngestionService.ts
1552
+ function zeroChunkResult(duplicateOf) {
1553
+ const result = {
1554
+ truncated: false,
1555
+ chunks: 0,
1556
+ ingestedChunks: 0,
1557
+ failedChunks: 0
1558
+ };
1559
+ if (duplicateOf !== void 0) result.duplicateOf = duplicateOf;
1560
+ return result;
1561
+ }
1304
1562
  var IngestionService = class {
1305
1563
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
1306
1564
  this.db = db;
@@ -1341,113 +1599,110 @@ var IngestionService = class {
1341
1599
  if (onDuplicateHash === "throw") {
1342
1600
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
1343
1601
  }
1344
- return { truncated: false, chunks: 0, duplicateOf: canonical };
1602
+ return zeroChunkResult(canonical);
1345
1603
  }
1346
1604
  }
1347
1605
  const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
1348
- if (chunks.length === 0) return { truncated: false, chunks: 0 };
1606
+ if (chunks.length === 0) return zeroChunkResult();
1607
+ const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
1349
1608
  const chunkResults = await withConcurrency(
1350
- chunks.map((chunk) => async () => {
1351
- const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
1609
+ chunks.map((chunk, chunkIndex) => async () => {
1352
1610
  const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
1353
1611
  chunk,
1354
1612
  params.promptOverride,
1355
1613
  ontologyContext
1356
1614
  );
1357
- const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
1358
- const result = parseJsonResponse(responseText);
1359
- return {
1360
- facts: (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null),
1361
- ontology_updates: result.ontology_updates
1362
- };
1615
+ try {
1616
+ const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
1617
+ const result2 = parseJsonResponse(responseText);
1618
+ return {
1619
+ status: "ok",
1620
+ facts: (Array.isArray(result2.facts) ? result2.facts : []).map(validateFact).filter((f) => f !== null),
1621
+ ontology_updates: result2.ontology_updates
1622
+ };
1623
+ } catch (e) {
1624
+ const failure = e instanceof WikiParseError ? {
1625
+ chunkIndex,
1626
+ sourceRef,
1627
+ source: "parse",
1628
+ tier: e.tier,
1629
+ position: e.position,
1630
+ message: e.message
1631
+ } : {
1632
+ chunkIndex,
1633
+ sourceRef,
1634
+ source: "llm",
1635
+ // `safeErrorToString` is a non-throwing coercion: an LLM
1636
+ // provider may reject with a non-Error value, and even
1637
+ // `String(e)` itself can throw when `e.toString()` throws.
1638
+ // Letting such an exception escape this catch would cause
1639
+ // `withConcurrency` to reject, discarding every sibling
1640
+ // chunk's results — the exact failure mode this per-chunk
1641
+ // try/catch exists to prevent. Keeping the value in the
1642
+ // `ChunkFailure` preserves the typed diagnostic for
1643
+ // callers (see parseFailures) without unwinding the loop.
1644
+ position: null,
1645
+ message: safeErrorToString(e)
1646
+ };
1647
+ const total = chunks.length;
1648
+ const tags = [];
1649
+ if (failure.tier) tags.push(`tier=${failure.tier}`);
1650
+ if (failure.position !== null) tags.push(`position=${failure.position}`);
1651
+ const tagsSuffix = tags.length > 0 ? `; ${tags.join(" ")}` : "";
1652
+ console.warn(
1653
+ `[WikiMemory] ingest chunk ${chunkIndex + 1}/${total} ${failure.source} failed (sourceRef=${sourceRef}${tagsSuffix})`
1654
+ );
1655
+ return { status: "failed", error: failure };
1656
+ }
1363
1657
  }),
1364
1658
  chunkConcurrency
1365
1659
  );
1660
+ let ingestedChunks = 0;
1661
+ let failedChunks = 0;
1662
+ const failures = [];
1366
1663
  const seen = /* @__PURE__ */ new Set();
1367
1664
  const orderedChunkFacts = [];
1368
- for (const chunkResult of chunkResults) {
1665
+ for (const slot of chunkResults) {
1666
+ if (slot.status === "failed") {
1667
+ failedChunks++;
1668
+ failures.push(slot.error);
1669
+ continue;
1670
+ }
1671
+ ingestedChunks++;
1369
1672
  const dedupedFacts = [];
1370
- for (const fact of chunkResult.facts) {
1673
+ for (const fact of slot.facts) {
1371
1674
  const normalizedTitle = normalizeTitleKey(fact.title);
1372
1675
  if (!seen.has(normalizedTitle)) {
1373
1676
  seen.add(normalizedTitle);
1374
1677
  dedupedFacts.push(fact);
1375
1678
  }
1376
1679
  }
1377
- orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: chunkResult.ontology_updates });
1680
+ orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: slot.ontology_updates });
1681
+ }
1682
+ if (failedChunks === chunks.length) {
1683
+ throw new WikiIngestEmptyError({
1684
+ parseFailures: failures,
1685
+ sourceRef,
1686
+ chunks: chunks.length
1687
+ });
1378
1688
  }
1379
- const now = Date.now();
1380
1689
  const insertedFacts = [];
1381
1690
  const deletedSourceFactIds = [];
1382
1691
  try {
1383
- await this.db.withTransactionAsync(async (tx) => {
1384
- deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
1385
- const titleIndex = /* @__PURE__ */ new Map();
1386
- const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
1387
- for (const existing of existingFacts) {
1388
- titleIndex.set(normalizeTitleKey(existing.title), {
1389
- id: existing.id,
1390
- okf_type: existing.okf_type ?? null
1391
- });
1392
- }
1393
- let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
1394
- let { mode, manifest } = ontologyState;
1395
- const hostNodes = [];
1396
- const rawEdgeRequests = [];
1397
- for (const { facts, ontology_updates } of orderedChunkFacts) {
1398
- if (mode === "emergent" && ontology_updates && this.ontologyService) {
1399
- manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
1400
- ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
1401
- mode = ontologyState.mode;
1402
- }
1403
- for (const fact of facts) {
1404
- const ontologyFact = fact;
1405
- const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
1406
- const id = generateId("fact_");
1407
- hostNodes.push({
1408
- id,
1409
- type: ontologyFact.okf_type ?? "",
1410
- title: fact.title,
1411
- body: fact.body,
1412
- // Forward the LLM-extracted tags/confidence through the
1413
- // extracted-shape upsertGraphCore path so the entries row
1414
- // stores them (search filterability, heal-candidate
1415
- // selection, runReembed embedding-text signal). The public
1416
- // WikiMemory.upsertGraph API doesn't accept tags/confidence
1417
- // — host-supplied deterministic nodes default to [] and
1418
- // 'certain' as documented.
1419
- tags: fact.tags,
1420
- confidence: fact.confidence
1421
- });
1422
- insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
1423
- titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
1424
- if (normalized.edges.length > 0) {
1425
- rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
1426
- }
1427
- }
1428
- }
1429
- const hostEdges = [];
1430
- for (const req of rawEdgeRequests) {
1431
- const resolved = this.ontologyService?.resolveEdges(
1432
- entityId,
1433
- req.sourceId,
1434
- req.sourceType,
1435
- req.edges,
1436
- manifest,
1437
- titleIndex,
1438
- now
1439
- ) ?? [];
1440
- for (const e of resolved) {
1441
- hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
1442
- }
1443
- }
1444
- await this.upsertGraphCore(
1445
- entityId,
1446
- { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
1447
- tx,
1448
- { strict: false }
1449
- );
1450
- });
1692
+ if (failedChunks === 0) {
1693
+ const fullResult = await this.db.withTransactionAsync(async (tx) => {
1694
+ return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx);
1695
+ });
1696
+ deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
1697
+ insertedFacts.push(...fullResult.insertedFacts);
1698
+ } else {
1699
+ const partialResult = await this.db.withTransactionAsync(async (tx) => {
1700
+ const flat = [];
1701
+ for (const slot of orderedChunkFacts) flat.push(...slot.facts);
1702
+ return await this.appendPartialFacts(entityId, sourceRef, flat, tx);
1703
+ });
1704
+ insertedFacts.push(...partialResult.insertedDescriptors);
1705
+ }
1451
1706
  } catch (err) {
1452
1707
  const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
1453
1708
  if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
@@ -1456,22 +1711,31 @@ var IngestionService = class {
1456
1711
  if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
1457
1712
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
1458
1713
  }
1459
- return { truncated: false, chunks: 0, duplicateOf: canonical };
1714
+ return zeroChunkResult(canonical);
1460
1715
  }
1461
1716
  await this.searchService.sync(entityId);
1462
- const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
1463
- for (const factId of uniqueDeletedSourceFactIds) {
1464
- try {
1465
- await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
1466
- } catch (hookErr) {
1467
- console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
1717
+ if (failedChunks === 0) {
1718
+ const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
1719
+ for (const factId of uniqueDeletedSourceFactIds) {
1720
+ try {
1721
+ await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
1722
+ } catch (hookErr) {
1723
+ console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
1724
+ }
1468
1725
  }
1469
1726
  }
1470
1727
  for (const fact of insertedFacts) {
1471
1728
  await this.embeddingService.embedFact(fact);
1472
1729
  }
1473
1730
  this.searchService.evictCache(entityId);
1474
- return { truncated, chunks: chunks.length };
1731
+ const result = {
1732
+ truncated,
1733
+ chunks: chunks.length,
1734
+ ingestedChunks,
1735
+ failedChunks
1736
+ };
1737
+ if (failedChunks > 0) result.parseFailures = failures;
1738
+ return result;
1475
1739
  } finally {
1476
1740
  releaseIngestLocks();
1477
1741
  }
@@ -1596,6 +1860,156 @@ var IngestionService = class {
1596
1860
  superseded: deletedFactIds.length + deletedEdgeCount
1597
1861
  };
1598
1862
  }
1863
+ /**
1864
+ * Full supersession + ownership path: identical to the pre-issue-#92 behavior
1865
+ * for the happy path. Called from the `failedChunks === 0` branch of
1866
+ * `ingestDocument`. Runs INSIDE the caller's tx.
1867
+ *
1868
+ * Returns `{ deletedSourceFactIds, insertedFacts }` so the caller can fire
1869
+ * post-commit hooks (embedding lifecycle) AFTER the transaction commits.
1870
+ * On the partial path the caller never invokes this method; the empty
1871
+ * arrays stay empty.
1872
+ */
1873
+ async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx) {
1874
+ const deletedSourceFactIds = [];
1875
+ const insertedFacts = [];
1876
+ deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
1877
+ const titleIndex = /* @__PURE__ */ new Map();
1878
+ const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
1879
+ for (const existing of existingFacts) {
1880
+ titleIndex.set(normalizeTitleKey(existing.title), {
1881
+ id: existing.id,
1882
+ okf_type: existing.okf_type ?? null
1883
+ });
1884
+ }
1885
+ let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
1886
+ let { mode, manifest } = ontologyState;
1887
+ const hostNodes = [];
1888
+ const rawEdgeRequests = [];
1889
+ const now = Date.now();
1890
+ for (const { facts, ontology_updates } of orderedChunkFacts) {
1891
+ if (mode === "emergent" && ontology_updates && this.ontologyService) {
1892
+ manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
1893
+ ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
1894
+ mode = ontologyState.mode;
1895
+ }
1896
+ for (const fact of facts) {
1897
+ const ontologyFact = fact;
1898
+ const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
1899
+ const id = generateId("fact_");
1900
+ hostNodes.push({
1901
+ id,
1902
+ type: ontologyFact.okf_type ?? "",
1903
+ title: fact.title,
1904
+ body: fact.body,
1905
+ // Forward the LLM-extracted tags/confidence through the
1906
+ // extracted-shape upsertGraphCore path so the entries row
1907
+ // stores them (search filterability, heal-candidate
1908
+ // selection, runReembed embedding-text signal). The public
1909
+ // WikiMemory.upsertGraph API doesn't accept tags/confidence
1910
+ // — host-supplied deterministic nodes default to [] and
1911
+ // 'certain' as documented.
1912
+ tags: fact.tags,
1913
+ confidence: fact.confidence
1914
+ });
1915
+ insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
1916
+ titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
1917
+ if (normalized.edges.length > 0) {
1918
+ rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
1919
+ }
1920
+ }
1921
+ }
1922
+ const hostEdges = [];
1923
+ for (const req of rawEdgeRequests) {
1924
+ const resolved = this.ontologyService?.resolveEdges(
1925
+ entityId,
1926
+ req.sourceId,
1927
+ req.sourceType,
1928
+ req.edges,
1929
+ manifest,
1930
+ titleIndex,
1931
+ now
1932
+ ) ?? [];
1933
+ for (const e of resolved) {
1934
+ hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
1935
+ }
1936
+ }
1937
+ await this.upsertGraphCore(
1938
+ entityId,
1939
+ { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
1940
+ tx,
1941
+ { strict: false }
1942
+ );
1943
+ return { deletedSourceFactIds, insertedFacts };
1944
+ }
1945
+ /**
1946
+ * Partial-commit path for `ingestDocument`. Inserts ONLY facts whose
1947
+ * normalized title is not already in the live set for this (entityId,
1948
+ * sourceRef). Does NOT call `entryRepo.softDeleteBySource` (no
1949
+ * supersession) and does NOT call `sourceRefIndexRepo.upsert` (no
1950
+ * ownership). Edges are NOT resolved on this path (no ontology-context
1951
+ * build, no `resolveEdges`, no `mergeEmergentUpdates`). Stale edges from
1952
+ * prior attempts are recovered on the next full run's supersession.
1953
+ *
1954
+ * Source-hash is stored as NULL on partial rows. `findLatestSourceHash`
1955
+ * reads from the most recently updated live row for the sourceRef, so
1956
+ * storing the incoming hash here would cause `hasChanged` to return
1957
+ * `false` on a same-hash retry — the failed chunks would never get a
1958
+ * second chance. With NULL, a retry sees `storedHash === null`, `hasChanged`
1959
+ * returns `true`, and the partial commit's sibling rows remain live
1960
+ * (deduped, not superseded) for the next attempt to extend.
1961
+ *
1962
+ * Runs INSIDE the caller's `tx`. Does not open a nested transaction.
1963
+ * Returns `{ inserted, skippedDuplicate }` for observability.
1964
+ */
1965
+ async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx) {
1966
+ const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
1967
+ const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
1968
+ const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
1969
+ let inserted = 0;
1970
+ let skippedDuplicate = 0;
1971
+ const now = Date.now();
1972
+ const insertedDescriptors = [];
1973
+ for (const fact of dedupedFacts) {
1974
+ const normalizedTitle = normalizeTitleKey(fact.title);
1975
+ if (liveTitles.has(normalizedTitle)) {
1976
+ skippedDuplicate++;
1977
+ continue;
1978
+ }
1979
+ liveTitles.add(normalizedTitle);
1980
+ const id = generateId("fact_");
1981
+ const wikiFact = {
1982
+ id,
1983
+ entity_id: entityId,
1984
+ title: fact.title,
1985
+ body: fact.body,
1986
+ tags: fact.tags,
1987
+ confidence: fact.confidence,
1988
+ source_type: "immutable_document",
1989
+ // source_hash: null on partial rows — see method docstring. The
1990
+ // full path stamps the actual hash; partial rows stay hash-less so
1991
+ // `hasChanged` keeps returning true for retries.
1992
+ source_hash: null,
1993
+ source_ref: sourceRef,
1994
+ created_at: now,
1995
+ updated_at: now,
1996
+ last_accessed_at: null,
1997
+ access_count: 0,
1998
+ deleted_at: null,
1999
+ okf_type: null
2000
+ };
2001
+ await this.entryRepo.upsert(wikiFact, tx);
2002
+ insertedDescriptors.push({
2003
+ id,
2004
+ entity_id: entityId,
2005
+ title: fact.title,
2006
+ body: fact.body,
2007
+ tags: JSON.stringify(fact.tags)
2008
+ });
2009
+ inserted++;
2010
+ }
2011
+ return { inserted, skippedDuplicate, insertedDescriptors };
2012
+ }
1599
2013
  };
1600
2014
 
1601
2015
  // src/repositories/BaseRepository.ts
@@ -1888,6 +2302,25 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
1888
2302
  var HEAL_MAX_PROMPT_CHARS = 4e4;
1889
2303
  var HEAL_BATCH_SIZE = 25;
1890
2304
  var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
2305
+ var SKIP_ERROR_LOG_CHARS = 4096;
2306
+ var formatSkipError = (err) => {
2307
+ let base;
2308
+ if (err instanceof Error || typeof err !== "object" && typeof err !== "function") {
2309
+ base = safeErrorToString(err);
2310
+ } else {
2311
+ try {
2312
+ const json = JSON.stringify(err);
2313
+ base = typeof json === "string" ? json : safeErrorToString(err);
2314
+ } catch {
2315
+ base = safeErrorToString(err);
2316
+ }
2317
+ }
2318
+ if (base.length > SKIP_ERROR_LOG_CHARS) {
2319
+ const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
2320
+ return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
2321
+ }
2322
+ return base;
2323
+ };
1891
2324
  var MaintenanceService = class {
1892
2325
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
1893
2326
  this.db = db;
@@ -2368,8 +2801,7 @@ var MaintenanceService = class {
2368
2801
  maxPromptChars: HEAL_MAX_PROMPT_CHARS,
2369
2802
  onSkip: (fact, err) => {
2370
2803
  console.warn(
2371
- `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded`,
2372
- err
2804
+ `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
2373
2805
  );
2374
2806
  }
2375
2807
  });
@@ -2507,8 +2939,7 @@ var MaintenanceService = class {
2507
2939
  maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
2508
2940
  onSkip: (fact, err) => {
2509
2941
  console.warn(
2510
- `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded`,
2511
- err
2942
+ `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
2512
2943
  );
2513
2944
  }
2514
2945
  });
@@ -2686,8 +3117,6 @@ var MaintenanceService = class {
2686
3117
  return sanitizeRankerError(err, this.options.sanitizeRankerErrors);
2687
3118
  }
2688
3119
  };
2689
-
2690
- // src/services/ImportExportService.ts
2691
3120
  var MAX_EMBEDDING_BLOB_BYTES = 32 * 1024;
2692
3121
  var IMPORT_TITLE_MAX = 500;
2693
3122
  var IMPORT_BODY_MAX = 8e3;
@@ -2907,7 +3336,23 @@ var ImportExportService = class {
2907
3336
  access_count: fact.access_count,
2908
3337
  deleted_at: fact.deleted_at,
2909
3338
  embedding_blob: blobData ?? void 0,
2910
- okf_type: fact.okf_type ?? null
3339
+ okf_type: fact.okf_type ?? null,
3340
+ // OKF v0.2 — forwarded as-is; parseOkfBundle / the caller is responsible for
3341
+ // populating these per spec, importDump does not derive or validate them.
3342
+ lifecycle_status: fact.lifecycle_status ?? "stable",
3343
+ stale_after: fact.stale_after ?? null,
3344
+ generated_by: fact.generated_by ?? null,
3345
+ okf_sources: fact.okf_sources ?? [],
3346
+ okf_verified: fact.okf_verified ?? [],
3347
+ okf_usage_window: fact.okf_usage_window ?? null,
3348
+ // last_verified_at/_by are DERIVED mirrors of `okf_verified` per spec
3349
+ // §5.3. Recomputing here (instead of trusting the payload field) closes
3350
+ // the invariant that a raw importDump cannot carry a
3351
+ // last_verified_by that contradicts its own okf_verified list.
3352
+ ...(() => {
3353
+ const latest = latestVerified(fact.okf_verified ?? [], Date.now());
3354
+ return { last_verified_at: latest?.at ?? null, last_verified_by: latest?.by ?? null };
3355
+ })()
2911
3356
  };
2912
3357
  await this.entryRepo.upsertForImport(factObj, tx);
2913
3358
  if (blobData != null) {
@@ -2957,7 +3402,19 @@ var ImportExportService = class {
2957
3402
  updated_at: safeUpdatedAt,
2958
3403
  resolved_at: task.resolved_at,
2959
3404
  deleted_at: task.deleted_at,
2960
- okf_type: task.okf_type ?? null
3405
+ okf_type: task.okf_type ?? null,
3406
+ // OKF v0.2
3407
+ lifecycle_status: task.lifecycle_status ?? "stable",
3408
+ stale_after: task.stale_after ?? null,
3409
+ generated_by: task.generated_by ?? null,
3410
+ okf_sources: task.okf_sources ?? [],
3411
+ okf_verified: task.okf_verified ?? [],
3412
+ okf_usage_window: task.okf_usage_window ?? null,
3413
+ // Derived mirrors — see fact branch above; same invariant.
3414
+ ...(() => {
3415
+ const latest = latestVerified(task.okf_verified ?? [], Date.now());
3416
+ return { last_verified_at: latest?.at ?? null, last_verified_by: latest?.by ?? null };
3417
+ })()
2961
3418
  },
2962
3419
  tx,
2963
3420
  safeUpdatedAt
@@ -3913,6 +4370,6 @@ var WriteService = class {
3913
4370
  }
3914
4371
  };
3915
4372
 
3916
- export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, validateInlineEdges, validateManifest };
3917
- //# sourceMappingURL=chunk-KIJINNYU.mjs.map
3918
- //# sourceMappingURL=chunk-KIJINNYU.mjs.map
4373
+ export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiIngestEmptyError, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, validateInlineEdges, validateManifest };
4374
+ //# sourceMappingURL=chunk-QVY5DFJV.mjs.map
4375
+ //# sourceMappingURL=chunk-QVY5DFJV.mjs.map