@equationalapplications/core-llm-wiki 5.4.0 → 5.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -287,6 +287,25 @@ var PrunePartialFailureError = class extends Error {
287
287
  }
288
288
  };
289
289
  var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
290
+ var WikiParseError = class extends Error {
291
+ constructor(message, opts) {
292
+ super(message);
293
+ this.name = "WikiParseError";
294
+ this.tier = opts.tier;
295
+ this.position = opts.position ?? null;
296
+ this.slice = opts.slice ?? "";
297
+ }
298
+ };
299
+ var WikiIngestEmptyError = class extends Error {
300
+ constructor(params) {
301
+ const summary = `All ${params.chunks} chunks failed for sourceRef "${params.sourceRef}"; see parseFailures for per-chunk detail`;
302
+ super(summary);
303
+ this.name = "WikiIngestEmptyError";
304
+ this.parseFailures = params.parseFailures;
305
+ this.sourceRef = params.sourceRef;
306
+ this.chunks = params.chunks;
307
+ }
308
+ };
290
309
 
291
310
  // src/db/serializedAdapter.ts
292
311
  function isDriverError(err) {
@@ -3417,24 +3436,67 @@ var JobManager = class {
3417
3436
  };
3418
3437
 
3419
3438
  // src/utils/pure.ts
3439
+ var MAX_REPAIR_CANDIDATES = 5;
3440
+ function throwNoJsonFound(text, start) {
3441
+ throw new WikiParseError(
3442
+ "No JSON object/array found in LLM response",
3443
+ { tier: "strict", position: start, slice: text }
3444
+ );
3445
+ }
3420
3446
  function parseJsonResponse(text) {
3421
3447
  const firstBrace = text.indexOf("{");
3422
3448
  const firstBracket = text.indexOf("[");
3423
3449
  let start;
3424
3450
  let openChar;
3425
- let closeChar;
3426
3451
  const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
3427
3452
  if (useBrace) {
3428
3453
  start = firstBrace;
3429
3454
  openChar = "{";
3430
- closeChar = "}";
3431
3455
  } else if (firstBracket !== -1) {
3432
3456
  start = firstBracket;
3433
3457
  openChar = "[";
3434
- closeChar = "]";
3435
3458
  } else {
3436
- throw new SyntaxError("No JSON object/array found in LLM response");
3459
+ throwNoJsonFound(text, null);
3460
+ }
3461
+ const slice = scanJsonSlice(text, start, openChar);
3462
+ if (slice !== null) {
3463
+ try {
3464
+ return JSON.parse(slice);
3465
+ } catch {
3466
+ }
3437
3467
  }
3468
+ const repairOdd = containerAwareRepair(text, start, false);
3469
+ if (repairOdd.success !== null && !repairOdd.ambiguous) {
3470
+ return repairOdd.success;
3471
+ }
3472
+ const repairEven = containerAwareRepair(text, start, true);
3473
+ if (repairOdd.success !== null && repairEven.success !== null) {
3474
+ const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
3475
+ return pick.success;
3476
+ }
3477
+ if (repairOdd.success !== null) {
3478
+ return repairOdd.success;
3479
+ }
3480
+ if (repairEven.success !== null) {
3481
+ return repairEven.success;
3482
+ }
3483
+ const repair = repairEven.failed !== null ? repairEven : repairOdd;
3484
+ if (repair.failed !== null) {
3485
+ throw new WikiParseError(
3486
+ `Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
3487
+ { tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
3488
+ );
3489
+ }
3490
+ if (slice === null) {
3491
+ throwNoJsonFound(text, start);
3492
+ }
3493
+ throw new WikiParseError(
3494
+ "No parsable JSON candidate found",
3495
+ { tier: "all", position: start, slice: text }
3496
+ );
3497
+ }
3498
+ function scanJsonSlice(text, start, openChar) {
3499
+ const closeChar = openChar === "{" ? "}" : "]";
3438
3500
  let depth = 0;
3439
3501
  let inString = false;
3440
3502
  let escape = false;
@@ -3466,8 +3528,193 @@ function parseJsonResponse(text) {
3466
3528
  }
3467
3529
  }
3468
3530
  }
3469
- if (end === -1) throw new SyntaxError("No JSON object/array found in LLM response");
3470
- return JSON.parse(text.slice(start, end + 1));
3531
+ if (end === -1) return null;
3532
+ return safeSlice(text, start, end + 1);
3533
+ }
3534
+ function containerAwareRepair(text, start, closeOnEvenParity) {
3535
+ const stack = [];
3536
+ let inString = false;
3537
+ let stringRole = null;
3538
+ let escape = false;
3539
+ let i = start;
3540
+ let out = "";
3541
+ let bareQuoteCount = 0;
3542
+ let bestFailed = null;
3543
+ let escapes = 0;
3544
+ let ambiguous = false;
3545
+ let bestSuccess = null;
3546
+ let attempts = 0;
3547
+ function tryEmitCandidate(candidate) {
3548
+ attempts++;
3549
+ try {
3550
+ bestSuccess = JSON.parse(candidate);
3551
+ return true;
3552
+ } catch (err) {
3553
+ if (bestFailed === null) {
3554
+ bestFailed = {
3555
+ candidate,
3556
+ position: extractParsePosition(err),
3557
+ message: safeErrorToString(err)
3558
+ };
3559
+ }
3560
+ return false;
3561
+ }
3562
+ }
3563
+ while (i < text.length) {
3564
+ const ch = text[i];
3565
+ if (stack.length === 0 && !inString) {
3566
+ if (ch === "{" || ch === "[") {
3567
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
3568
+ out += ch;
3569
+ }
3570
+ i++;
3571
+ continue;
3572
+ }
3573
+ if (escape) {
3574
+ out += ch;
3575
+ escape = false;
3576
+ i++;
3577
+ continue;
3578
+ }
3579
+ if (inString && ch === "\\") {
3580
+ out += ch;
3581
+ escape = true;
3582
+ i++;
3583
+ continue;
3584
+ }
3585
+ if (inString) {
3586
+ const code = ch.charCodeAt(0);
3587
+ if (code < 32) {
3588
+ out += controlCharEscape(code);
3589
+ i++;
3590
+ continue;
3591
+ }
3592
+ }
3593
+ if (ch === '"') {
3594
+ let j = i + 1;
3595
+ while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
3596
+ j++;
3597
+ }
3598
+ const next = j < text.length ? text[j] : "";
3599
+ if (!inString) {
3600
+ if (next === "}" || next === "]") {
3601
+ if (j > i + 1) out += text.slice(i + 1, j);
3602
+ out += next;
3603
+ i = j + 1;
3604
+ stack.pop();
3605
+ if (stack.length === 0) {
3606
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
3607
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
3608
+ out = "";
3609
+ }
3610
+ continue;
3611
+ }
3612
+ const top = stack[stack.length - 1];
3613
+ stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
3614
+ out += ch;
3615
+ inString = true;
3616
+ i++;
3617
+ continue;
3618
+ }
3619
+ const commaOnly = next === ",";
3620
+ const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
3621
+ const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
3622
+ if (commaOnly && bareQuoteCount > 0) ambiguous = true;
3623
+ const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
3624
+ const isClosing = isKeyClose || isValueClose || shouldCommaClose;
3625
+ if (isClosing) {
3626
+ out += ch;
3627
+ inString = false;
3628
+ const topFrame = stack[stack.length - 1];
3629
+ if (topFrame && topFrame.container === "object") {
3630
+ topFrame.expectKey = stringRole === "value";
3631
+ }
3632
+ stringRole = null;
3633
+ bareQuoteCount = 0;
3634
+ i++;
3635
+ continue;
3636
+ }
3637
+ out += "\\" + ch;
3638
+ bareQuoteCount++;
3639
+ escapes++;
3640
+ i++;
3641
+ continue;
3642
+ }
3643
+ if (!inString && (ch === "{" || ch === "[")) {
3644
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
3645
+ out += ch;
3646
+ i++;
3647
+ continue;
3648
+ }
3649
+ if (!inString && ch === ",") {
3650
+ const topFrame = stack[stack.length - 1];
3651
+ if (topFrame && topFrame.container === "object") {
3652
+ topFrame.expectKey = true;
3653
+ }
3654
+ out += ch;
3655
+ i++;
3656
+ continue;
3657
+ }
3658
+ if (!inString && (ch === "}" || ch === "]")) {
3659
+ out += ch;
3660
+ stack.pop();
3661
+ if (stack.length === 0) {
3662
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
3663
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
3664
+ out = "";
3665
+ }
3666
+ i++;
3667
+ continue;
3668
+ }
3669
+ out += ch;
3670
+ i++;
3671
+ }
3672
+ return { success: null, failed: bestFailed, escapes, ambiguous };
3673
+ }
3674
+ function controlCharEscape(code) {
3675
+ switch (code) {
3676
+ case 10:
3677
+ return "\\n";
3678
+ case 9:
3679
+ return "\\t";
3680
+ case 13:
3681
+ return "\\r";
3682
+ case 8:
3683
+ return "\\b";
3684
+ case 12:
3685
+ return "\\f";
3686
+ default:
3687
+ return "\\u" + code.toString(16).padStart(4, "0");
3688
+ }
3689
+ }
3690
+ function extractParsePosition(err) {
3691
+ const match = /position\s+(\d+)/.exec(safeErrorToString(err));
3692
+ return match ? Number(match[1]) : null;
3693
+ }
3694
+ function safeErrorToString(e) {
3695
+ if (e instanceof Error) {
3696
+ const msg = readErrorField(e, "message");
3697
+ if (typeof msg === "string" && msg.length > 0) return msg;
3698
+ const name = readErrorField(e, "name");
3699
+ if (typeof name === "string" && name.length > 0) return name;
3700
+ return "[Error]";
3701
+ }
3702
+ try {
3703
+ return String(e);
3704
+ } catch {
3705
+ try {
3706
+ return Object.prototype.toString.call(e);
3707
+ } catch {
3708
+ return "[unstringifiable error]";
3709
+ }
3710
+ }
3711
+ }
3712
+ function readErrorField(e, key) {
3713
+ try {
3714
+ return e[key];
3715
+ } catch {
3716
+ return void 0;
3717
+ }
3471
3718
  }
3472
3719
  function sanitizeRankerError(err, sanitizeRankerErrors) {
3473
3720
  if (sanitizeRankerErrors === false) {
@@ -3652,7 +3899,7 @@ Return ONLY a valid JSON object matching this schema:
3652
3899
  {
3653
3900
  "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
3654
3901
  }
3655
- Extract verbatim factual content. Do not return markdown, just raw JSON.`;
3902
+ Extract verbatim factual content. JSON escaping rules: every literal " character in the source must be escaped as \\" inside any JSON string body, and every literal newline as \\n. Source prose containing quotes (e.g. a worked example with "...") must still be reproduced exactly \u2014 re-escape, do not omit. Do not return markdown, just raw JSON.`;
3656
3903
  var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
3657
3904
  Return ONLY a valid JSON object matching this schema:
3658
3905
  {
@@ -3661,7 +3908,7 @@ Return ONLY a valid JSON object matching this schema:
3661
3908
  ]
3662
3909
  }
3663
3910
  If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
3664
- Do not return markdown, just raw JSON.`;
3911
+ When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
3665
3912
 
3666
3913
  // src/services/PromptService.ts
3667
3914
  var PromptService = class {
@@ -3774,6 +4021,16 @@ var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
3774
4021
  var DEFAULT_CHUNK_OVERLAP = 400;
3775
4022
 
3776
4023
  // src/services/IngestionService.ts
4024
+ function zeroChunkResult(duplicateOf) {
4025
+ const result = {
4026
+ truncated: false,
4027
+ chunks: 0,
4028
+ ingestedChunks: 0,
4029
+ failedChunks: 0
4030
+ };
4031
+ if (duplicateOf !== void 0) result.duplicateOf = duplicateOf;
4032
+ return result;
4033
+ }
3777
4034
  var IngestionService = class {
3778
4035
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
3779
4036
  this.db = db;
@@ -3814,113 +4071,110 @@ var IngestionService = class {
3814
4071
  if (onDuplicateHash === "throw") {
3815
4072
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
3816
4073
  }
3817
- return { truncated: false, chunks: 0, duplicateOf: canonical };
4074
+ return zeroChunkResult(canonical);
3818
4075
  }
3819
4076
  }
3820
4077
  const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
3821
- if (chunks.length === 0) return { truncated: false, chunks: 0 };
4078
+ if (chunks.length === 0) return zeroChunkResult();
4079
+ const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
3822
4080
  const chunkResults = await withConcurrency(
3823
- chunks.map((chunk) => async () => {
3824
- const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
4081
+ chunks.map((chunk, chunkIndex) => async () => {
3825
4082
  const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
3826
4083
  chunk,
3827
4084
  params.promptOverride,
3828
4085
  ontologyContext
3829
4086
  );
3830
- const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
3831
- const result = parseJsonResponse(responseText);
3832
- return {
3833
- facts: (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null),
3834
- ontology_updates: result.ontology_updates
3835
- };
4087
+ try {
4088
+ const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
4089
+ const result2 = parseJsonResponse(responseText);
4090
+ return {
4091
+ status: "ok",
4092
+ facts: (Array.isArray(result2.facts) ? result2.facts : []).map(validateFact).filter((f) => f !== null),
4093
+ ontology_updates: result2.ontology_updates
4094
+ };
4095
+ } catch (e) {
4096
+ const failure = e instanceof WikiParseError ? {
4097
+ chunkIndex,
4098
+ sourceRef,
4099
+ source: "parse",
4100
+ tier: e.tier,
4101
+ position: e.position,
4102
+ message: e.message
4103
+ } : {
4104
+ chunkIndex,
4105
+ sourceRef,
4106
+ source: "llm",
4107
+ // `safeErrorToString` is a non-throwing coercion: an LLM
4108
+ // provider may reject with a non-Error value, and even
4109
+ // `String(e)` itself can throw when `e.toString()` throws.
4110
+ // Letting such an exception escape this catch would cause
4111
+ // `withConcurrency` to reject, discarding every sibling
4112
+ // chunk's results — the exact failure mode this per-chunk
4113
+ // try/catch exists to prevent. Keeping the value in the
4114
+ // `ChunkFailure` preserves the typed diagnostic for
4115
+ // callers (see parseFailures) without unwinding the loop.
4116
+ position: null,
4117
+ message: safeErrorToString(e)
4118
+ };
4119
+ const total = chunks.length;
4120
+ const tags = [];
4121
+ if (failure.tier) tags.push(`tier=${failure.tier}`);
4122
+ if (failure.position !== null) tags.push(`position=${failure.position}`);
4123
+ const tagsSuffix = tags.length > 0 ? `; ${tags.join(" ")}` : "";
4124
+ console.warn(
4125
+ `[WikiMemory] ingest chunk ${chunkIndex + 1}/${total} ${failure.source} failed (sourceRef=${sourceRef}${tagsSuffix})`
4126
+ );
4127
+ return { status: "failed", error: failure };
4128
+ }
3836
4129
  }),
3837
4130
  chunkConcurrency
3838
4131
  );
4132
+ let ingestedChunks = 0;
4133
+ let failedChunks = 0;
4134
+ const failures = [];
3839
4135
  const seen = /* @__PURE__ */ new Set();
3840
4136
  const orderedChunkFacts = [];
3841
- for (const chunkResult of chunkResults) {
4137
+ for (const slot of chunkResults) {
4138
+ if (slot.status === "failed") {
4139
+ failedChunks++;
4140
+ failures.push(slot.error);
4141
+ continue;
4142
+ }
4143
+ ingestedChunks++;
3842
4144
  const dedupedFacts = [];
3843
- for (const fact of chunkResult.facts) {
4145
+ for (const fact of slot.facts) {
3844
4146
  const normalizedTitle = normalizeTitleKey(fact.title);
3845
4147
  if (!seen.has(normalizedTitle)) {
3846
4148
  seen.add(normalizedTitle);
3847
4149
  dedupedFacts.push(fact);
3848
4150
  }
3849
4151
  }
3850
- orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: chunkResult.ontology_updates });
4152
+ orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: slot.ontology_updates });
4153
+ }
4154
+ if (failedChunks === chunks.length) {
4155
+ throw new WikiIngestEmptyError({
4156
+ parseFailures: failures,
4157
+ sourceRef,
4158
+ chunks: chunks.length
4159
+ });
3851
4160
  }
3852
- const now = Date.now();
3853
4161
  const insertedFacts = [];
3854
4162
  const deletedSourceFactIds = [];
3855
4163
  try {
3856
- await this.db.withTransactionAsync(async (tx) => {
3857
- deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
3858
- const titleIndex = /* @__PURE__ */ new Map();
3859
- const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
3860
- for (const existing of existingFacts) {
3861
- titleIndex.set(normalizeTitleKey(existing.title), {
3862
- id: existing.id,
3863
- okf_type: existing.okf_type ?? null
3864
- });
3865
- }
3866
- let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
3867
- let { mode, manifest } = ontologyState;
3868
- const hostNodes = [];
3869
- const rawEdgeRequests = [];
3870
- for (const { facts, ontology_updates } of orderedChunkFacts) {
3871
- if (mode === "emergent" && ontology_updates && this.ontologyService) {
3872
- manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
3873
- ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
3874
- mode = ontologyState.mode;
3875
- }
3876
- for (const fact of facts) {
3877
- const ontologyFact = fact;
3878
- const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
3879
- const id = generateId("fact_");
3880
- hostNodes.push({
3881
- id,
3882
- type: ontologyFact.okf_type ?? "",
3883
- title: fact.title,
3884
- body: fact.body,
3885
- // Forward the LLM-extracted tags/confidence through the
3886
- // extracted-shape upsertGraphCore path so the entries row
3887
- // stores them (search filterability, heal-candidate
3888
- // selection, runReembed embedding-text signal). The public
3889
- // WikiMemory.upsertGraph API doesn't accept tags/confidence
3890
- // — host-supplied deterministic nodes default to [] and
3891
- // 'certain' as documented.
3892
- tags: fact.tags,
3893
- confidence: fact.confidence
3894
- });
3895
- insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
3896
- titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
3897
- if (normalized.edges.length > 0) {
3898
- rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
3899
- }
3900
- }
3901
- }
3902
- const hostEdges = [];
3903
- for (const req of rawEdgeRequests) {
3904
- const resolved = this.ontologyService?.resolveEdges(
3905
- entityId,
3906
- req.sourceId,
3907
- req.sourceType,
3908
- req.edges,
3909
- manifest,
3910
- titleIndex,
3911
- now
3912
- ) ?? [];
3913
- for (const e of resolved) {
3914
- hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
3915
- }
3916
- }
3917
- await this.upsertGraphCore(
3918
- entityId,
3919
- { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
3920
- tx,
3921
- { strict: false }
3922
- );
3923
- });
4164
+ if (failedChunks === 0) {
4165
+ const fullResult = await this.db.withTransactionAsync(async (tx) => {
4166
+ return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx);
4167
+ });
4168
+ deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
4169
+ insertedFacts.push(...fullResult.insertedFacts);
4170
+ } else {
4171
+ const partialResult = await this.db.withTransactionAsync(async (tx) => {
4172
+ const flat = [];
4173
+ for (const slot of orderedChunkFacts) flat.push(...slot.facts);
4174
+ return await this.appendPartialFacts(entityId, sourceRef, flat, tx);
4175
+ });
4176
+ insertedFacts.push(...partialResult.insertedDescriptors);
4177
+ }
3924
4178
  } catch (err) {
3925
4179
  const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
3926
4180
  if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
@@ -3929,22 +4183,31 @@ var IngestionService = class {
3929
4183
  if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
3930
4184
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
3931
4185
  }
3932
- return { truncated: false, chunks: 0, duplicateOf: canonical };
4186
+ return zeroChunkResult(canonical);
3933
4187
  }
3934
4188
  await this.searchService.sync(entityId);
3935
- const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
3936
- for (const factId of uniqueDeletedSourceFactIds) {
3937
- try {
3938
- await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
3939
- } catch (hookErr) {
3940
- console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
4189
+ if (failedChunks === 0) {
4190
+ const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
4191
+ for (const factId of uniqueDeletedSourceFactIds) {
4192
+ try {
4193
+ await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
4194
+ } catch (hookErr) {
4195
+ console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
4196
+ }
3941
4197
  }
3942
4198
  }
3943
4199
  for (const fact of insertedFacts) {
3944
4200
  await this.embeddingService.embedFact(fact);
3945
4201
  }
3946
4202
  this.searchService.evictCache(entityId);
3947
- return { truncated, chunks: chunks.length };
4203
+ const result = {
4204
+ truncated,
4205
+ chunks: chunks.length,
4206
+ ingestedChunks,
4207
+ failedChunks
4208
+ };
4209
+ if (failedChunks > 0) result.parseFailures = failures;
4210
+ return result;
3948
4211
  } finally {
3949
4212
  releaseIngestLocks();
3950
4213
  }
@@ -4069,6 +4332,156 @@ var IngestionService = class {
4069
4332
  superseded: deletedFactIds.length + deletedEdgeCount
4070
4333
  };
4071
4334
  }
4335
+ /**
4336
+ * Full supersession + ownership path: identical to the pre-issue-#92 behavior
4337
+ * for the happy path. Called from the `failedChunks === 0` branch of
4338
+ * `ingestDocument`. Runs INSIDE the caller's tx.
4339
+ *
4340
+ * Returns `{ deletedSourceFactIds, insertedFacts }` so the caller can fire
4341
+ * post-commit hooks (embedding lifecycle) AFTER the transaction commits.
4342
+ * On the partial path the caller never invokes this method; the empty
4343
+ * arrays stay empty.
4344
+ */
4345
+ async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx) {
4346
+ const deletedSourceFactIds = [];
4347
+ const insertedFacts = [];
4348
+ deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
4349
+ const titleIndex = /* @__PURE__ */ new Map();
4350
+ const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
4351
+ for (const existing of existingFacts) {
4352
+ titleIndex.set(normalizeTitleKey(existing.title), {
4353
+ id: existing.id,
4354
+ okf_type: existing.okf_type ?? null
4355
+ });
4356
+ }
4357
+ let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
4358
+ let { mode, manifest } = ontologyState;
4359
+ const hostNodes = [];
4360
+ const rawEdgeRequests = [];
4361
+ const now = Date.now();
4362
+ for (const { facts, ontology_updates } of orderedChunkFacts) {
4363
+ if (mode === "emergent" && ontology_updates && this.ontologyService) {
4364
+ manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
4365
+ ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
4366
+ mode = ontologyState.mode;
4367
+ }
4368
+ for (const fact of facts) {
4369
+ const ontologyFact = fact;
4370
+ const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
4371
+ const id = generateId("fact_");
4372
+ hostNodes.push({
4373
+ id,
4374
+ type: ontologyFact.okf_type ?? "",
4375
+ title: fact.title,
4376
+ body: fact.body,
4377
+ // Forward the LLM-extracted tags/confidence through the
4378
+ // extracted-shape upsertGraphCore path so the entries row
4379
+ // stores them (search filterability, heal-candidate
4380
+ // selection, runReembed embedding-text signal). The public
4381
+ // WikiMemory.upsertGraph API doesn't accept tags/confidence
4382
+ // — host-supplied deterministic nodes default to [] and
4383
+ // 'certain' as documented.
4384
+ tags: fact.tags,
4385
+ confidence: fact.confidence
4386
+ });
4387
+ insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
4388
+ titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
4389
+ if (normalized.edges.length > 0) {
4390
+ rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
4391
+ }
4392
+ }
4393
+ }
4394
+ const hostEdges = [];
4395
+ for (const req of rawEdgeRequests) {
4396
+ const resolved = this.ontologyService?.resolveEdges(
4397
+ entityId,
4398
+ req.sourceId,
4399
+ req.sourceType,
4400
+ req.edges,
4401
+ manifest,
4402
+ titleIndex,
4403
+ now
4404
+ ) ?? [];
4405
+ for (const e of resolved) {
4406
+ hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
4407
+ }
4408
+ }
4409
+ await this.upsertGraphCore(
4410
+ entityId,
4411
+ { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
4412
+ tx,
4413
+ { strict: false }
4414
+ );
4415
+ return { deletedSourceFactIds, insertedFacts };
4416
+ }
4417
+ /**
4418
+ * Partial-commit path for `ingestDocument`. Inserts ONLY facts whose
4419
+ * normalized title is not already in the live set for this (entityId,
4420
+ * sourceRef). Does NOT call `entryRepo.softDeleteBySource` (no
4421
+ * supersession) and does NOT call `sourceRefIndexRepo.upsert` (no
4422
+ * ownership). Edges are NOT resolved on this path (no ontology-context
4423
+ * build, no `resolveEdges`, no `mergeEmergentUpdates`). Stale edges from
4424
+ * prior attempts are recovered on the next full run's supersession.
4425
+ *
4426
+ * Source-hash is stored as NULL on partial rows. `findLatestSourceHash`
4427
+ * reads from the most recently updated live row for the sourceRef, so
4428
+ * storing the incoming hash here would cause `hasChanged` to return
4429
+ * `false` on a same-hash retry — the failed chunks would never get a
4430
+ * second chance. With NULL, a retry sees `storedHash === null`, `hasChanged`
4431
+ * returns `true`, and the partial commit's sibling rows remain live
4432
+ * (deduped, not superseded) for the next attempt to extend.
4433
+ *
4434
+ * Runs INSIDE the caller's `tx`. Does not open a nested transaction.
4435
+ * Returns `{ inserted, skippedDuplicate }` for observability.
4436
+ */
4437
+ async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx) {
4438
+ const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
4439
+ const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
4440
+ const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
4441
+ let inserted = 0;
4442
+ let skippedDuplicate = 0;
4443
+ const now = Date.now();
4444
+ const insertedDescriptors = [];
4445
+ for (const fact of dedupedFacts) {
4446
+ const normalizedTitle = normalizeTitleKey(fact.title);
4447
+ if (liveTitles.has(normalizedTitle)) {
4448
+ skippedDuplicate++;
4449
+ continue;
4450
+ }
4451
+ liveTitles.add(normalizedTitle);
4452
+ const id = generateId("fact_");
4453
+ const wikiFact = {
4454
+ id,
4455
+ entity_id: entityId,
4456
+ title: fact.title,
4457
+ body: fact.body,
4458
+ tags: fact.tags,
4459
+ confidence: fact.confidence,
4460
+ source_type: "immutable_document",
4461
+ // source_hash: null on partial rows — see method docstring. The
4462
+ // full path stamps the actual hash; partial rows stay hash-less so
4463
+ // `hasChanged` keeps returning true for retries.
4464
+ source_hash: null,
4465
+ source_ref: sourceRef,
4466
+ created_at: now,
4467
+ updated_at: now,
4468
+ last_accessed_at: null,
4469
+ access_count: 0,
4470
+ deleted_at: null,
4471
+ okf_type: null
4472
+ };
4473
+ await this.entryRepo.upsert(wikiFact, tx);
4474
+ insertedDescriptors.push({
4475
+ id,
4476
+ entity_id: entityId,
4477
+ title: fact.title,
4478
+ body: fact.body,
4479
+ tags: JSON.stringify(fact.tags)
4480
+ });
4481
+ inserted++;
4482
+ }
4483
+ return { inserted, skippedDuplicate, insertedDescriptors };
4484
+ }
4072
4485
  };
4073
4486
 
4074
4487
  // src/services/BoundedLlmCall.ts
@@ -4189,6 +4602,25 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
4189
4602
  var HEAL_MAX_PROMPT_CHARS = 4e4;
4190
4603
  var HEAL_BATCH_SIZE = 25;
4191
4604
  var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
4605
+ var SKIP_ERROR_LOG_CHARS = 4096;
4606
+ var formatSkipError = (err) => {
4607
+ let base;
4608
+ if (err instanceof Error || typeof err !== "object" && typeof err !== "function") {
4609
+ base = safeErrorToString(err);
4610
+ } else {
4611
+ try {
4612
+ const json = JSON.stringify(err);
4613
+ base = typeof json === "string" ? json : safeErrorToString(err);
4614
+ } catch {
4615
+ base = safeErrorToString(err);
4616
+ }
4617
+ }
4618
+ if (base.length > SKIP_ERROR_LOG_CHARS) {
4619
+ const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
4620
+ return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
4621
+ }
4622
+ return base;
4623
+ };
4192
4624
  var MaintenanceService = class {
4193
4625
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
4194
4626
  this.db = db;
@@ -4669,8 +5101,7 @@ var MaintenanceService = class {
4669
5101
  maxPromptChars: HEAL_MAX_PROMPT_CHARS,
4670
5102
  onSkip: (fact, err) => {
4671
5103
  console.warn(
4672
- `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded`,
4673
- err
5104
+ `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
4674
5105
  );
4675
5106
  }
4676
5107
  });
@@ -4808,8 +5239,7 @@ var MaintenanceService = class {
4808
5239
  maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
4809
5240
  onSkip: (fact, err) => {
4810
5241
  console.warn(
4811
- `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded`,
4812
- err
5242
+ `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
4813
5243
  );
4814
5244
  }
4815
5245
  });
@@ -7989,7 +8419,9 @@ exports.PromptService = PromptService;
7989
8419
  exports.PrunePartialFailureError = PrunePartialFailureError;
7990
8420
  exports.WikiBusyError = WikiBusyError;
7991
8421
  exports.WikiDuplicateHashError = WikiDuplicateHashError;
8422
+ exports.WikiIngestEmptyError = WikiIngestEmptyError;
7992
8423
  exports.WikiMemory = WikiMemory;
8424
+ exports.WikiParseError = WikiParseError;
7993
8425
  exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
7994
8426
  exports.WikiStrictOntologyViolation = WikiStrictOntologyViolation;
7995
8427
  exports.WikiTransactionError = WikiTransactionError;