@equationalapplications/core-llm-wiki 5.4.0 → 5.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -287,6 +287,25 @@ var PrunePartialFailureError = class extends Error {
287
287
  }
288
288
  };
289
289
  var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
290
+ var WikiParseError = class extends Error {
291
+ constructor(message, opts) {
292
+ super(message);
293
+ this.name = "WikiParseError";
294
+ this.tier = opts.tier;
295
+ this.position = opts.position ?? null;
296
+ this.slice = opts.slice ?? "";
297
+ }
298
+ };
299
+ var WikiIngestEmptyError = class extends Error {
300
+ constructor(params) {
301
+ const summary = `All ${params.chunks} chunks failed for sourceRef "${params.sourceRef}"; see parseFailures for per-chunk detail`;
302
+ super(summary);
303
+ this.name = "WikiIngestEmptyError";
304
+ this.parseFailures = params.parseFailures;
305
+ this.sourceRef = params.sourceRef;
306
+ this.chunks = params.chunks;
307
+ }
308
+ };
290
309
 
291
310
  // src/db/serializedAdapter.ts
292
311
  function isDriverError(err) {
@@ -3417,24 +3436,67 @@ var JobManager = class {
3417
3436
  };
3418
3437
 
3419
3438
  // src/utils/pure.ts
3439
+ var MAX_REPAIR_CANDIDATES = 5;
3440
+ function throwNoJsonFound(text, start) {
3441
+ throw new WikiParseError(
3442
+ "No JSON object/array found in LLM response",
3443
+ { tier: "strict", position: start, slice: text }
3444
+ );
3445
+ }
3420
3446
  function parseJsonResponse(text) {
3421
3447
  const firstBrace = text.indexOf("{");
3422
3448
  const firstBracket = text.indexOf("[");
3423
3449
  let start;
3424
3450
  let openChar;
3425
- let closeChar;
3426
3451
  const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
3427
3452
  if (useBrace) {
3428
3453
  start = firstBrace;
3429
3454
  openChar = "{";
3430
- closeChar = "}";
3431
3455
  } else if (firstBracket !== -1) {
3432
3456
  start = firstBracket;
3433
3457
  openChar = "[";
3434
- closeChar = "]";
3435
3458
  } else {
3436
- throw new SyntaxError("No JSON object/array found in LLM response");
3459
+ throwNoJsonFound(text, null);
3460
+ }
3461
+ const slice = scanJsonSlice(text, start, openChar);
3462
+ if (slice !== null) {
3463
+ try {
3464
+ return JSON.parse(slice);
3465
+ } catch {
3466
+ }
3467
+ }
3468
+ const repairOdd = containerAwareRepair(text, start, false);
3469
+ if (repairOdd.success !== null && !repairOdd.ambiguous) {
3470
+ return repairOdd.success;
3471
+ }
3472
+ const repairEven = containerAwareRepair(text, start, true);
3473
+ if (repairOdd.success !== null && repairEven.success !== null) {
3474
+ const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
3475
+ return pick.success;
3437
3476
  }
3477
+ if (repairOdd.success !== null) {
3478
+ return repairOdd.success;
3479
+ }
3480
+ if (repairEven.success !== null) {
3481
+ return repairEven.success;
3482
+ }
3483
+ const repair = repairEven.failed !== null ? repairEven : repairOdd;
3484
+ if (repair.failed !== null) {
3485
+ throw new WikiParseError(
3486
+ `Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
3487
+ { tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
3488
+ );
3489
+ }
3490
+ if (slice === null) {
3491
+ throwNoJsonFound(text, start);
3492
+ }
3493
+ throw new WikiParseError(
3494
+ "No parsable JSON candidate found",
3495
+ { tier: "all", position: start, slice: text }
3496
+ );
3497
+ }
3498
+ function scanJsonSlice(text, start, openChar) {
3499
+ const closeChar = openChar === "{" ? "}" : "]";
3438
3500
  let depth = 0;
3439
3501
  let inString = false;
3440
3502
  let escape = false;
@@ -3466,20 +3528,248 @@ function parseJsonResponse(text) {
3466
3528
  }
3467
3529
  }
3468
3530
  }
3469
- if (end === -1) throw new SyntaxError("No JSON object/array found in LLM response");
3470
- return JSON.parse(text.slice(start, end + 1));
3531
+ if (end === -1) return null;
3532
+ return safeSlice(text, start, end + 1);
3533
+ }
3534
+ function containerAwareRepair(text, start, closeOnEvenParity) {
3535
+ const stack = [];
3536
+ let inString = false;
3537
+ let stringRole = null;
3538
+ let escape = false;
3539
+ let i = start;
3540
+ let out = "";
3541
+ let bareQuoteCount = 0;
3542
+ let bestFailed = null;
3543
+ let escapes = 0;
3544
+ let ambiguous = false;
3545
+ let bestSuccess = null;
3546
+ let attempts = 0;
3547
+ function tryEmitCandidate(candidate) {
3548
+ attempts++;
3549
+ try {
3550
+ bestSuccess = JSON.parse(candidate);
3551
+ return true;
3552
+ } catch (err) {
3553
+ if (bestFailed === null) {
3554
+ bestFailed = {
3555
+ candidate,
3556
+ position: extractParsePosition(err),
3557
+ message: safeErrorToString(err)
3558
+ };
3559
+ }
3560
+ return false;
3561
+ }
3562
+ }
3563
+ while (i < text.length) {
3564
+ const ch = text[i];
3565
+ if (stack.length === 0 && !inString) {
3566
+ if (ch === "{" || ch === "[") {
3567
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
3568
+ out += ch;
3569
+ }
3570
+ i++;
3571
+ continue;
3572
+ }
3573
+ if (escape) {
3574
+ out += ch;
3575
+ escape = false;
3576
+ i++;
3577
+ continue;
3578
+ }
3579
+ if (inString && ch === "\\") {
3580
+ out += ch;
3581
+ escape = true;
3582
+ i++;
3583
+ continue;
3584
+ }
3585
+ if (inString) {
3586
+ const code = ch.charCodeAt(0);
3587
+ if (code < 32) {
3588
+ out += controlCharEscape(code);
3589
+ i++;
3590
+ continue;
3591
+ }
3592
+ }
3593
+ if (ch === '"') {
3594
+ let j = i + 1;
3595
+ while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
3596
+ j++;
3597
+ }
3598
+ const next = j < text.length ? text[j] : "";
3599
+ if (!inString) {
3600
+ if (next === "}" || next === "]") {
3601
+ if (j > i + 1) out += text.slice(i + 1, j);
3602
+ out += next;
3603
+ i = j + 1;
3604
+ stack.pop();
3605
+ if (stack.length === 0) {
3606
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
3607
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
3608
+ out = "";
3609
+ }
3610
+ continue;
3611
+ }
3612
+ const top = stack[stack.length - 1];
3613
+ stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
3614
+ out += ch;
3615
+ inString = true;
3616
+ i++;
3617
+ continue;
3618
+ }
3619
+ const commaOnly = next === ",";
3620
+ const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
3621
+ const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
3622
+ if (commaOnly && bareQuoteCount > 0) ambiguous = true;
3623
+ const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
3624
+ const isClosing = isKeyClose || isValueClose || shouldCommaClose;
3625
+ if (isClosing) {
3626
+ out += ch;
3627
+ inString = false;
3628
+ const topFrame = stack[stack.length - 1];
3629
+ if (topFrame && topFrame.container === "object") {
3630
+ topFrame.expectKey = stringRole === "value";
3631
+ }
3632
+ stringRole = null;
3633
+ bareQuoteCount = 0;
3634
+ i++;
3635
+ continue;
3636
+ }
3637
+ out += "\\" + ch;
3638
+ bareQuoteCount++;
3639
+ escapes++;
3640
+ i++;
3641
+ continue;
3642
+ }
3643
+ if (!inString && (ch === "{" || ch === "[")) {
3644
+ stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
3645
+ out += ch;
3646
+ i++;
3647
+ continue;
3648
+ }
3649
+ if (!inString && ch === ",") {
3650
+ const topFrame = stack[stack.length - 1];
3651
+ if (topFrame && topFrame.container === "object") {
3652
+ topFrame.expectKey = true;
3653
+ }
3654
+ out += ch;
3655
+ i++;
3656
+ continue;
3657
+ }
3658
+ if (!inString && (ch === "}" || ch === "]")) {
3659
+ out += ch;
3660
+ stack.pop();
3661
+ if (stack.length === 0) {
3662
+ if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
3663
+ if (attempts >= MAX_REPAIR_CANDIDATES) break;
3664
+ out = "";
3665
+ }
3666
+ i++;
3667
+ continue;
3668
+ }
3669
+ out += ch;
3670
+ i++;
3671
+ }
3672
+ return { success: null, failed: bestFailed, escapes, ambiguous };
3673
+ }
3674
+ function controlCharEscape(code) {
3675
+ switch (code) {
3676
+ case 10:
3677
+ return "\\n";
3678
+ case 9:
3679
+ return "\\t";
3680
+ case 13:
3681
+ return "\\r";
3682
+ case 8:
3683
+ return "\\b";
3684
+ case 12:
3685
+ return "\\f";
3686
+ default:
3687
+ return "\\u" + code.toString(16).padStart(4, "0");
3688
+ }
3689
+ }
3690
+ function extractParsePosition(err) {
3691
+ const match = /position\s+(\d+)/.exec(safeErrorToString(err));
3692
+ return match ? Number(match[1]) : null;
3693
+ }
3694
+ function safeErrorToString(e) {
3695
+ let isErrorLike = false;
3696
+ try {
3697
+ isErrorLike = e instanceof Error;
3698
+ } catch {
3699
+ }
3700
+ if (isErrorLike) {
3701
+ const msg = readErrorField(e, "message");
3702
+ if (typeof msg === "string" && msg.length > 0) return msg;
3703
+ const name = readErrorField(e, "name");
3704
+ if (typeof name === "string" && name.length > 0) return name;
3705
+ return "[Error]";
3706
+ }
3707
+ try {
3708
+ return String(e);
3709
+ } catch {
3710
+ try {
3711
+ return Object.prototype.toString.call(e);
3712
+ } catch {
3713
+ return "[unstringifiable error]";
3714
+ }
3715
+ }
3716
+ }
3717
+ function readErrorField(e, key) {
3718
+ try {
3719
+ return e[key];
3720
+ } catch {
3721
+ return void 0;
3722
+ }
3471
3723
  }
3472
3724
  function sanitizeRankerError(err, sanitizeRankerErrors) {
3725
+ let isErrorLike = false;
3726
+ try {
3727
+ isErrorLike = err instanceof Error;
3728
+ } catch {
3729
+ }
3473
3730
  if (sanitizeRankerErrors === false) {
3474
- return err instanceof Error ? err : new Error(String(err));
3731
+ return isErrorLike ? err : new Error(safeErrorToString(err));
3732
+ }
3733
+ let errLike = null;
3734
+ if (isErrorLike) errLike = err;
3735
+ let typeName;
3736
+ try {
3737
+ if (errLike) {
3738
+ const rawName = errLike.constructor?.name;
3739
+ typeName = typeof rawName === "string" ? rawName : "Error";
3740
+ } else {
3741
+ typeName = typeof err;
3742
+ }
3743
+ } catch {
3744
+ typeName = isErrorLike ? "Error" : typeof err;
3745
+ }
3746
+ let innerCause;
3747
+ if (errLike) {
3748
+ let cause;
3749
+ try {
3750
+ cause = errLike.cause;
3751
+ } catch {
3752
+ cause = void 0;
3753
+ }
3754
+ if (cause !== void 0) {
3755
+ let causeName;
3756
+ try {
3757
+ const rawCauseName = cause?.constructor?.name;
3758
+ causeName = typeof rawCauseName === "string" ? rawCauseName : typeof cause;
3759
+ } catch {
3760
+ causeName = typeof cause;
3761
+ }
3762
+ innerCause = new Error(`Caused by: ${causeName}`);
3763
+ }
3475
3764
  }
3476
- const typeName = err instanceof Error ? err.constructor?.name ?? "Error" : typeof err;
3477
- const innerCause = err instanceof Error && err.cause !== void 0 ? new Error(`Caused by: ${err.cause?.constructor?.name ?? typeof err.cause}`) : void 0;
3478
3765
  const sanitized = new Error(
3479
3766
  `VectorRanker ${typeName} (message scrubbed for security)`,
3480
3767
  innerCause ? { cause: innerCause } : void 0
3481
3768
  );
3482
- sanitized.name = typeName;
3769
+ try {
3770
+ sanitized.name = typeName;
3771
+ } catch {
3772
+ }
3483
3773
  return sanitized;
3484
3774
  }
3485
3775
  function safeSlice(value, start, end) {
@@ -3652,7 +3942,7 @@ Return ONLY a valid JSON object matching this schema:
3652
3942
  {
3653
3943
  "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
3654
3944
  }
3655
- Extract verbatim factual content. Do not return markdown, just raw JSON.`;
3945
+ Extract verbatim factual content. JSON escaping rules: every literal " character in the source must be escaped as \\" inside any JSON string body, and every literal newline as \\n. Source prose containing quotes (e.g. a worked example with "...") must still be reproduced exactly \u2014 re-escape, do not omit. Do not return markdown, just raw JSON.`;
3656
3946
  var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
3657
3947
  Return ONLY a valid JSON object matching this schema:
3658
3948
  {
@@ -3661,7 +3951,7 @@ Return ONLY a valid JSON object matching this schema:
3661
3951
  ]
3662
3952
  }
3663
3953
  If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
3664
- Do not return markdown, just raw JSON.`;
3954
+ When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
3665
3955
 
3666
3956
  // src/services/PromptService.ts
3667
3957
  var PromptService = class {
@@ -3774,6 +4064,16 @@ var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
3774
4064
  var DEFAULT_CHUNK_OVERLAP = 400;
3775
4065
 
3776
4066
  // src/services/IngestionService.ts
4067
+ function zeroChunkResult(duplicateOf) {
4068
+ const result = {
4069
+ truncated: false,
4070
+ chunks: 0,
4071
+ ingestedChunks: 0,
4072
+ failedChunks: 0
4073
+ };
4074
+ if (duplicateOf !== void 0) result.duplicateOf = duplicateOf;
4075
+ return result;
4076
+ }
3777
4077
  var IngestionService = class {
3778
4078
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
3779
4079
  this.db = db;
@@ -3814,113 +4114,110 @@ var IngestionService = class {
3814
4114
  if (onDuplicateHash === "throw") {
3815
4115
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
3816
4116
  }
3817
- return { truncated: false, chunks: 0, duplicateOf: canonical };
4117
+ return zeroChunkResult(canonical);
3818
4118
  }
3819
4119
  }
3820
4120
  const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
3821
- if (chunks.length === 0) return { truncated: false, chunks: 0 };
4121
+ if (chunks.length === 0) return zeroChunkResult();
4122
+ const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
3822
4123
  const chunkResults = await withConcurrency(
3823
- chunks.map((chunk) => async () => {
3824
- const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
4124
+ chunks.map((chunk, chunkIndex) => async () => {
3825
4125
  const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
3826
4126
  chunk,
3827
4127
  params.promptOverride,
3828
4128
  ontologyContext
3829
4129
  );
3830
- const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
3831
- const result = parseJsonResponse(responseText);
3832
- return {
3833
- facts: (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null),
3834
- ontology_updates: result.ontology_updates
3835
- };
4130
+ try {
4131
+ const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
4132
+ const result2 = parseJsonResponse(responseText);
4133
+ return {
4134
+ status: "ok",
4135
+ facts: (Array.isArray(result2.facts) ? result2.facts : []).map(validateFact).filter((f) => f !== null),
4136
+ ontology_updates: result2.ontology_updates
4137
+ };
4138
+ } catch (e) {
4139
+ const failure = e instanceof WikiParseError ? {
4140
+ chunkIndex,
4141
+ sourceRef,
4142
+ source: "parse",
4143
+ tier: e.tier,
4144
+ position: e.position,
4145
+ message: e.message
4146
+ } : {
4147
+ chunkIndex,
4148
+ sourceRef,
4149
+ source: "llm",
4150
+ // `safeErrorToString` is a non-throwing coercion: an LLM
4151
+ // provider may reject with a non-Error value, and even
4152
+ // `String(e)` itself can throw when `e.toString()` throws.
4153
+ // Letting such an exception escape this catch would cause
4154
+ // `withConcurrency` to reject, discarding every sibling
4155
+ // chunk's results — the exact failure mode this per-chunk
4156
+ // try/catch exists to prevent. Keeping the value in the
4157
+ // `ChunkFailure` preserves the typed diagnostic for
4158
+ // callers (see parseFailures) without unwinding the loop.
4159
+ position: null,
4160
+ message: safeErrorToString(e)
4161
+ };
4162
+ const total = chunks.length;
4163
+ const tags = [];
4164
+ if (failure.tier) tags.push(`tier=${failure.tier}`);
4165
+ if (failure.position !== null) tags.push(`position=${failure.position}`);
4166
+ const tagsSuffix = tags.length > 0 ? `; ${tags.join(" ")}` : "";
4167
+ console.warn(
4168
+ `[WikiMemory] ingest chunk ${chunkIndex + 1}/${total} ${failure.source} failed (sourceRef=${sourceRef}${tagsSuffix})`
4169
+ );
4170
+ return { status: "failed", error: failure };
4171
+ }
3836
4172
  }),
3837
4173
  chunkConcurrency
3838
4174
  );
4175
+ let ingestedChunks = 0;
4176
+ let failedChunks = 0;
4177
+ const failures = [];
3839
4178
  const seen = /* @__PURE__ */ new Set();
3840
4179
  const orderedChunkFacts = [];
3841
- for (const chunkResult of chunkResults) {
4180
+ for (const slot of chunkResults) {
4181
+ if (slot.status === "failed") {
4182
+ failedChunks++;
4183
+ failures.push(slot.error);
4184
+ continue;
4185
+ }
4186
+ ingestedChunks++;
3842
4187
  const dedupedFacts = [];
3843
- for (const fact of chunkResult.facts) {
4188
+ for (const fact of slot.facts) {
3844
4189
  const normalizedTitle = normalizeTitleKey(fact.title);
3845
4190
  if (!seen.has(normalizedTitle)) {
3846
4191
  seen.add(normalizedTitle);
3847
4192
  dedupedFacts.push(fact);
3848
4193
  }
3849
4194
  }
3850
- orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: chunkResult.ontology_updates });
4195
+ orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: slot.ontology_updates });
4196
+ }
4197
+ if (failedChunks === chunks.length) {
4198
+ throw new WikiIngestEmptyError({
4199
+ parseFailures: failures,
4200
+ sourceRef,
4201
+ chunks: chunks.length
4202
+ });
3851
4203
  }
3852
- const now = Date.now();
3853
4204
  const insertedFacts = [];
3854
4205
  const deletedSourceFactIds = [];
3855
4206
  try {
3856
- await this.db.withTransactionAsync(async (tx) => {
3857
- deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
3858
- const titleIndex = /* @__PURE__ */ new Map();
3859
- const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
3860
- for (const existing of existingFacts) {
3861
- titleIndex.set(normalizeTitleKey(existing.title), {
3862
- id: existing.id,
3863
- okf_type: existing.okf_type ?? null
3864
- });
3865
- }
3866
- let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
3867
- let { mode, manifest } = ontologyState;
3868
- const hostNodes = [];
3869
- const rawEdgeRequests = [];
3870
- for (const { facts, ontology_updates } of orderedChunkFacts) {
3871
- if (mode === "emergent" && ontology_updates && this.ontologyService) {
3872
- manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
3873
- ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
3874
- mode = ontologyState.mode;
3875
- }
3876
- for (const fact of facts) {
3877
- const ontologyFact = fact;
3878
- const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
3879
- const id = generateId("fact_");
3880
- hostNodes.push({
3881
- id,
3882
- type: ontologyFact.okf_type ?? "",
3883
- title: fact.title,
3884
- body: fact.body,
3885
- // Forward the LLM-extracted tags/confidence through the
3886
- // extracted-shape upsertGraphCore path so the entries row
3887
- // stores them (search filterability, heal-candidate
3888
- // selection, runReembed embedding-text signal). The public
3889
- // WikiMemory.upsertGraph API doesn't accept tags/confidence
3890
- // — host-supplied deterministic nodes default to [] and
3891
- // 'certain' as documented.
3892
- tags: fact.tags,
3893
- confidence: fact.confidence
3894
- });
3895
- insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
3896
- titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
3897
- if (normalized.edges.length > 0) {
3898
- rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
3899
- }
3900
- }
3901
- }
3902
- const hostEdges = [];
3903
- for (const req of rawEdgeRequests) {
3904
- const resolved = this.ontologyService?.resolveEdges(
3905
- entityId,
3906
- req.sourceId,
3907
- req.sourceType,
3908
- req.edges,
3909
- manifest,
3910
- titleIndex,
3911
- now
3912
- ) ?? [];
3913
- for (const e of resolved) {
3914
- hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
3915
- }
3916
- }
3917
- await this.upsertGraphCore(
3918
- entityId,
3919
- { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
3920
- tx,
3921
- { strict: false }
3922
- );
3923
- });
4207
+ if (failedChunks === 0) {
4208
+ const fullResult = await this.db.withTransactionAsync(async (tx) => {
4209
+ return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx);
4210
+ });
4211
+ deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
4212
+ insertedFacts.push(...fullResult.insertedFacts);
4213
+ } else {
4214
+ const partialResult = await this.db.withTransactionAsync(async (tx) => {
4215
+ const flat = [];
4216
+ for (const slot of orderedChunkFacts) flat.push(...slot.facts);
4217
+ return await this.appendPartialFacts(entityId, sourceRef, flat, tx);
4218
+ });
4219
+ insertedFacts.push(...partialResult.insertedDescriptors);
4220
+ }
3924
4221
  } catch (err) {
3925
4222
  const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
3926
4223
  if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
@@ -3929,22 +4226,31 @@ var IngestionService = class {
3929
4226
  if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
3930
4227
  throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
3931
4228
  }
3932
- return { truncated: false, chunks: 0, duplicateOf: canonical };
4229
+ return zeroChunkResult(canonical);
3933
4230
  }
3934
4231
  await this.searchService.sync(entityId);
3935
- const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
3936
- for (const factId of uniqueDeletedSourceFactIds) {
3937
- try {
3938
- await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
3939
- } catch (hookErr) {
3940
- console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
4232
+ if (failedChunks === 0) {
4233
+ const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
4234
+ for (const factId of uniqueDeletedSourceFactIds) {
4235
+ try {
4236
+ await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
4237
+ } catch (hookErr) {
4238
+ console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
4239
+ }
3941
4240
  }
3942
4241
  }
3943
4242
  for (const fact of insertedFacts) {
3944
4243
  await this.embeddingService.embedFact(fact);
3945
4244
  }
3946
4245
  this.searchService.evictCache(entityId);
3947
- return { truncated, chunks: chunks.length };
4246
+ const result = {
4247
+ truncated,
4248
+ chunks: chunks.length,
4249
+ ingestedChunks,
4250
+ failedChunks
4251
+ };
4252
+ if (failedChunks > 0) result.parseFailures = failures;
4253
+ return result;
3948
4254
  } finally {
3949
4255
  releaseIngestLocks();
3950
4256
  }
@@ -4069,6 +4375,156 @@ var IngestionService = class {
4069
4375
  superseded: deletedFactIds.length + deletedEdgeCount
4070
4376
  };
4071
4377
  }
4378
+ /**
4379
+ * Full supersession + ownership path: identical to the pre-issue-#92 behavior
4380
+ * for the happy path. Called from the `failedChunks === 0` branch of
4381
+ * `ingestDocument`. Runs INSIDE the caller's tx.
4382
+ *
4383
+ * Returns `{ deletedSourceFactIds, insertedFacts }` so the caller can fire
4384
+ * post-commit hooks (embedding lifecycle) AFTER the transaction commits.
4385
+ * On the partial path the caller never invokes this method; the empty
4386
+ * arrays stay empty.
4387
+ */
4388
+ async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx) {
4389
+ const deletedSourceFactIds = [];
4390
+ const insertedFacts = [];
4391
+ deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
4392
+ const titleIndex = /* @__PURE__ */ new Map();
4393
+ const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
4394
+ for (const existing of existingFacts) {
4395
+ titleIndex.set(normalizeTitleKey(existing.title), {
4396
+ id: existing.id,
4397
+ okf_type: existing.okf_type ?? null
4398
+ });
4399
+ }
4400
+ let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
4401
+ let { mode, manifest } = ontologyState;
4402
+ const hostNodes = [];
4403
+ const rawEdgeRequests = [];
4404
+ const now = Date.now();
4405
+ for (const { facts, ontology_updates } of orderedChunkFacts) {
4406
+ if (mode === "emergent" && ontology_updates && this.ontologyService) {
4407
+ manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
4408
+ ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
4409
+ mode = ontologyState.mode;
4410
+ }
4411
+ for (const fact of facts) {
4412
+ const ontologyFact = fact;
4413
+ const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
4414
+ const id = generateId("fact_");
4415
+ hostNodes.push({
4416
+ id,
4417
+ type: ontologyFact.okf_type ?? "",
4418
+ title: fact.title,
4419
+ body: fact.body,
4420
+ // Forward the LLM-extracted tags/confidence through the
4421
+ // extracted-shape upsertGraphCore path so the entries row
4422
+ // stores them (search filterability, heal-candidate
4423
+ // selection, runReembed embedding-text signal). The public
4424
+ // WikiMemory.upsertGraph API doesn't accept tags/confidence
4425
+ // — host-supplied deterministic nodes default to [] and
4426
+ // 'certain' as documented.
4427
+ tags: fact.tags,
4428
+ confidence: fact.confidence
4429
+ });
4430
+ insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
4431
+ titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
4432
+ if (normalized.edges.length > 0) {
4433
+ rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
4434
+ }
4435
+ }
4436
+ }
4437
+ const hostEdges = [];
4438
+ for (const req of rawEdgeRequests) {
4439
+ const resolved = this.ontologyService?.resolveEdges(
4440
+ entityId,
4441
+ req.sourceId,
4442
+ req.sourceType,
4443
+ req.edges,
4444
+ manifest,
4445
+ titleIndex,
4446
+ now
4447
+ ) ?? [];
4448
+ for (const e of resolved) {
4449
+ hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
4450
+ }
4451
+ }
4452
+ await this.upsertGraphCore(
4453
+ entityId,
4454
+ { sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
4455
+ tx,
4456
+ { strict: false }
4457
+ );
4458
+ return { deletedSourceFactIds, insertedFacts };
4459
+ }
4460
+ /**
4461
+ * Partial-commit path for `ingestDocument`. Inserts ONLY facts whose
4462
+ * normalized title is not already in the live set for this (entityId,
4463
+ * sourceRef). Does NOT call `entryRepo.softDeleteBySource` (no
4464
+ * supersession) and does NOT call `sourceRefIndexRepo.upsert` (no
4465
+ * ownership). Edges are NOT resolved on this path (no ontology-context
4466
+ * build, no `resolveEdges`, no `mergeEmergentUpdates`). Stale edges from
4467
+ * prior attempts are recovered on the next full run's supersession.
4468
+ *
4469
+ * Source-hash is stored as NULL on partial rows. `findLatestSourceHash`
4470
+ * reads from the most recently updated live row for the sourceRef, so
4471
+ * storing the incoming hash here would cause `hasChanged` to return
4472
+ * `false` on a same-hash retry — the failed chunks would never get a
4473
+ * second chance. With NULL, a retry sees `storedHash === null`, `hasChanged`
4474
+ * returns `true`, and the partial commit's sibling rows remain live
4475
+ * (deduped, not superseded) for the next attempt to extend.
4476
+ *
4477
+ * Runs INSIDE the caller's `tx`. Does not open a nested transaction.
4478
+ * Returns `{ inserted, skippedDuplicate }` for observability.
4479
+ */
4480
+ async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx) {
4481
+ const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
4482
+ const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
4483
+ const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
4484
+ let inserted = 0;
4485
+ let skippedDuplicate = 0;
4486
+ const now = Date.now();
4487
+ const insertedDescriptors = [];
4488
+ for (const fact of dedupedFacts) {
4489
+ const normalizedTitle = normalizeTitleKey(fact.title);
4490
+ if (liveTitles.has(normalizedTitle)) {
4491
+ skippedDuplicate++;
4492
+ continue;
4493
+ }
4494
+ liveTitles.add(normalizedTitle);
4495
+ const id = generateId("fact_");
4496
+ const wikiFact = {
4497
+ id,
4498
+ entity_id: entityId,
4499
+ title: fact.title,
4500
+ body: fact.body,
4501
+ tags: fact.tags,
4502
+ confidence: fact.confidence,
4503
+ source_type: "immutable_document",
4504
+ // source_hash: null on partial rows — see method docstring. The
4505
+ // full path stamps the actual hash; partial rows stay hash-less so
4506
+ // `hasChanged` keeps returning true for retries.
4507
+ source_hash: null,
4508
+ source_ref: sourceRef,
4509
+ created_at: now,
4510
+ updated_at: now,
4511
+ last_accessed_at: null,
4512
+ access_count: 0,
4513
+ deleted_at: null,
4514
+ okf_type: null
4515
+ };
4516
+ await this.entryRepo.upsert(wikiFact, tx);
4517
+ insertedDescriptors.push({
4518
+ id,
4519
+ entity_id: entityId,
4520
+ title: fact.title,
4521
+ body: fact.body,
4522
+ tags: JSON.stringify(fact.tags)
4523
+ });
4524
+ inserted++;
4525
+ }
4526
+ return { inserted, skippedDuplicate, insertedDescriptors };
4527
+ }
4072
4528
  };
4073
4529
 
4074
4530
  // src/services/BoundedLlmCall.ts
@@ -4085,7 +4541,12 @@ var TRUNCATION_PATTERNS = [
4085
4541
  ];
4086
4542
  var EXCEEDS_LIMIT_PATTERN = /exceed[a-z]*[^.]{0,40}\b(model|context)?[ _-]?limit/i;
4087
4543
  function isTruncationError(err) {
4088
- const message = err instanceof Error ? err.message : String(err ?? "");
4544
+ let message;
4545
+ try {
4546
+ message = err instanceof Error ? err.message : String(err ?? "");
4547
+ } catch {
4548
+ return false;
4549
+ }
4089
4550
  if (EXCEEDS_LIMIT_PATTERN.test(message)) return false;
4090
4551
  return TRUNCATION_PATTERNS.some((pattern) => pattern.test(message));
4091
4552
  }
@@ -4189,6 +4650,30 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
4189
4650
  var HEAL_MAX_PROMPT_CHARS = 4e4;
4190
4651
  var HEAL_BATCH_SIZE = 25;
4191
4652
  var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
4653
+ var SKIP_ERROR_LOG_CHARS = 4096;
4654
+ var formatSkipError = (err) => {
4655
+ let base;
4656
+ let isErrorLike = false;
4657
+ try {
4658
+ isErrorLike = err instanceof Error;
4659
+ } catch {
4660
+ }
4661
+ if (isErrorLike || typeof err !== "object" && typeof err !== "function") {
4662
+ base = safeErrorToString(err);
4663
+ } else {
4664
+ try {
4665
+ const json = JSON.stringify(err);
4666
+ base = typeof json === "string" ? json : safeErrorToString(err);
4667
+ } catch {
4668
+ base = safeErrorToString(err);
4669
+ }
4670
+ }
4671
+ if (base.length > SKIP_ERROR_LOG_CHARS) {
4672
+ const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
4673
+ return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
4674
+ }
4675
+ return base;
4676
+ };
4192
4677
  var MaintenanceService = class {
4193
4678
  constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
4194
4679
  this.db = db;
@@ -4669,8 +5154,7 @@ var MaintenanceService = class {
4669
5154
  maxPromptChars: HEAL_MAX_PROMPT_CHARS,
4670
5155
  onSkip: (fact, err) => {
4671
5156
  console.warn(
4672
- `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded`,
4673
- err
5157
+ `[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
4674
5158
  );
4675
5159
  }
4676
5160
  });
@@ -4808,8 +5292,7 @@ var MaintenanceService = class {
4808
5292
  maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
4809
5293
  onSkip: (fact, err) => {
4810
5294
  console.warn(
4811
- `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded`,
4812
- err
5295
+ `[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
4813
5296
  );
4814
5297
  }
4815
5298
  });
@@ -5867,7 +6350,12 @@ var RetrievalService = class {
5867
6350
  }
5868
6351
  }
5869
6352
  } catch (rankerErr) {
5870
- const rankerError = rankerErr instanceof Error ? rankerErr : new Error(String(rankerErr));
6353
+ let isErrorLike = false;
6354
+ try {
6355
+ isErrorLike = rankerErr instanceof Error;
6356
+ } catch {
6357
+ }
6358
+ const rankerError = isErrorLike ? rankerErr : new Error(safeErrorToString(rankerErr));
5871
6359
  const policy = this.options.vectorRankerFallback ?? "js-cosine";
5872
6360
  this.options.onVectorRankerFallback?.({
5873
6361
  error: this._sanitizeRankerError(rankerError),
@@ -5979,12 +6467,21 @@ var RetrievalService = class {
5979
6467
  }
5980
6468
  }
5981
6469
  } catch (err) {
5982
- const error = err instanceof Error ? err : new Error(String(err));
6470
+ let isErrorLike = false;
6471
+ try {
6472
+ isErrorLike = err instanceof Error;
6473
+ } catch {
6474
+ }
6475
+ const error = isErrorLike ? err : new Error(safeErrorToString(err));
5983
6476
  if (rankerShouldRethrow) {
5984
6477
  throw error;
5985
6478
  }
5986
6479
  if (pendingRankerFallbackError) {
5987
- error.cause = pendingRankerFallbackError;
6480
+ try {
6481
+ error.cause = pendingRankerFallbackError;
6482
+ } catch {
6483
+ this.options.onRetrievalFallback?.(pendingRankerFallbackError);
6484
+ }
5988
6485
  pendingRankerFallbackError = void 0;
5989
6486
  }
5990
6487
  this.options.onRetrievalFallback?.(error);
@@ -7989,7 +8486,9 @@ exports.PromptService = PromptService;
7989
8486
  exports.PrunePartialFailureError = PrunePartialFailureError;
7990
8487
  exports.WikiBusyError = WikiBusyError;
7991
8488
  exports.WikiDuplicateHashError = WikiDuplicateHashError;
8489
+ exports.WikiIngestEmptyError = WikiIngestEmptyError;
7992
8490
  exports.WikiMemory = WikiMemory;
8491
+ exports.WikiParseError = WikiParseError;
7993
8492
  exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
7994
8493
  exports.WikiStrictOntologyViolation = WikiStrictOntologyViolation;
7995
8494
  exports.WikiTransactionError = WikiTransactionError;