@equationalapplications/core-llm-wiki 5.4.0 → 5.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -0
- package/dist/{chunk-N2NXN6PO.mjs → chunk-QVY5DFJV.mjs} +535 -105
- package/dist/chunk-QVY5DFJV.mjs.map +1 -0
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +534 -102
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2 -2
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-C71mnMxq.d.mts → testing-D0RnZjyW.d.mts} +108 -11
- package/dist/{testing-C71mnMxq.d.ts → testing-D0RnZjyW.d.ts} +108 -11
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +532 -102
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-N2NXN6PO.mjs.map +0 -1
package/dist/index.js
CHANGED
|
@@ -287,6 +287,25 @@ var PrunePartialFailureError = class extends Error {
|
|
|
287
287
|
}
|
|
288
288
|
};
|
|
289
289
|
var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
|
|
290
|
+
var WikiParseError = class extends Error {
|
|
291
|
+
constructor(message, opts) {
|
|
292
|
+
super(message);
|
|
293
|
+
this.name = "WikiParseError";
|
|
294
|
+
this.tier = opts.tier;
|
|
295
|
+
this.position = opts.position ?? null;
|
|
296
|
+
this.slice = opts.slice ?? "";
|
|
297
|
+
}
|
|
298
|
+
};
|
|
299
|
+
var WikiIngestEmptyError = class extends Error {
|
|
300
|
+
constructor(params) {
|
|
301
|
+
const summary = `All ${params.chunks} chunks failed for sourceRef "${params.sourceRef}"; see parseFailures for per-chunk detail`;
|
|
302
|
+
super(summary);
|
|
303
|
+
this.name = "WikiIngestEmptyError";
|
|
304
|
+
this.parseFailures = params.parseFailures;
|
|
305
|
+
this.sourceRef = params.sourceRef;
|
|
306
|
+
this.chunks = params.chunks;
|
|
307
|
+
}
|
|
308
|
+
};
|
|
290
309
|
|
|
291
310
|
// src/db/serializedAdapter.ts
|
|
292
311
|
function isDriverError(err) {
|
|
@@ -3417,24 +3436,67 @@ var JobManager = class {
|
|
|
3417
3436
|
};
|
|
3418
3437
|
|
|
3419
3438
|
// src/utils/pure.ts
|
|
3439
|
+
var MAX_REPAIR_CANDIDATES = 5;
|
|
3440
|
+
function throwNoJsonFound(text, start) {
|
|
3441
|
+
throw new WikiParseError(
|
|
3442
|
+
"No JSON object/array found in LLM response",
|
|
3443
|
+
{ tier: "strict", position: start, slice: text }
|
|
3444
|
+
);
|
|
3445
|
+
}
|
|
3420
3446
|
function parseJsonResponse(text) {
|
|
3421
3447
|
const firstBrace = text.indexOf("{");
|
|
3422
3448
|
const firstBracket = text.indexOf("[");
|
|
3423
3449
|
let start;
|
|
3424
3450
|
let openChar;
|
|
3425
|
-
let closeChar;
|
|
3426
3451
|
const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
|
|
3427
3452
|
if (useBrace) {
|
|
3428
3453
|
start = firstBrace;
|
|
3429
3454
|
openChar = "{";
|
|
3430
|
-
closeChar = "}";
|
|
3431
3455
|
} else if (firstBracket !== -1) {
|
|
3432
3456
|
start = firstBracket;
|
|
3433
3457
|
openChar = "[";
|
|
3434
|
-
closeChar = "]";
|
|
3435
3458
|
} else {
|
|
3436
|
-
|
|
3459
|
+
throwNoJsonFound(text, null);
|
|
3460
|
+
}
|
|
3461
|
+
const slice = scanJsonSlice(text, start, openChar);
|
|
3462
|
+
if (slice !== null) {
|
|
3463
|
+
try {
|
|
3464
|
+
return JSON.parse(slice);
|
|
3465
|
+
} catch {
|
|
3466
|
+
}
|
|
3437
3467
|
}
|
|
3468
|
+
const repairOdd = containerAwareRepair(text, start, false);
|
|
3469
|
+
if (repairOdd.success !== null && !repairOdd.ambiguous) {
|
|
3470
|
+
return repairOdd.success;
|
|
3471
|
+
}
|
|
3472
|
+
const repairEven = containerAwareRepair(text, start, true);
|
|
3473
|
+
if (repairOdd.success !== null && repairEven.success !== null) {
|
|
3474
|
+
const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
|
|
3475
|
+
return pick.success;
|
|
3476
|
+
}
|
|
3477
|
+
if (repairOdd.success !== null) {
|
|
3478
|
+
return repairOdd.success;
|
|
3479
|
+
}
|
|
3480
|
+
if (repairEven.success !== null) {
|
|
3481
|
+
return repairEven.success;
|
|
3482
|
+
}
|
|
3483
|
+
const repair = repairEven.failed !== null ? repairEven : repairOdd;
|
|
3484
|
+
if (repair.failed !== null) {
|
|
3485
|
+
throw new WikiParseError(
|
|
3486
|
+
`Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
|
|
3487
|
+
{ tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
|
|
3488
|
+
);
|
|
3489
|
+
}
|
|
3490
|
+
if (slice === null) {
|
|
3491
|
+
throwNoJsonFound(text, start);
|
|
3492
|
+
}
|
|
3493
|
+
throw new WikiParseError(
|
|
3494
|
+
"No parsable JSON candidate found",
|
|
3495
|
+
{ tier: "all", position: start, slice: text }
|
|
3496
|
+
);
|
|
3497
|
+
}
|
|
3498
|
+
function scanJsonSlice(text, start, openChar) {
|
|
3499
|
+
const closeChar = openChar === "{" ? "}" : "]";
|
|
3438
3500
|
let depth = 0;
|
|
3439
3501
|
let inString = false;
|
|
3440
3502
|
let escape = false;
|
|
@@ -3466,8 +3528,193 @@ function parseJsonResponse(text) {
|
|
|
3466
3528
|
}
|
|
3467
3529
|
}
|
|
3468
3530
|
}
|
|
3469
|
-
if (end === -1)
|
|
3470
|
-
return
|
|
3531
|
+
if (end === -1) return null;
|
|
3532
|
+
return safeSlice(text, start, end + 1);
|
|
3533
|
+
}
|
|
3534
|
+
function containerAwareRepair(text, start, closeOnEvenParity) {
|
|
3535
|
+
const stack = [];
|
|
3536
|
+
let inString = false;
|
|
3537
|
+
let stringRole = null;
|
|
3538
|
+
let escape = false;
|
|
3539
|
+
let i = start;
|
|
3540
|
+
let out = "";
|
|
3541
|
+
let bareQuoteCount = 0;
|
|
3542
|
+
let bestFailed = null;
|
|
3543
|
+
let escapes = 0;
|
|
3544
|
+
let ambiguous = false;
|
|
3545
|
+
let bestSuccess = null;
|
|
3546
|
+
let attempts = 0;
|
|
3547
|
+
function tryEmitCandidate(candidate) {
|
|
3548
|
+
attempts++;
|
|
3549
|
+
try {
|
|
3550
|
+
bestSuccess = JSON.parse(candidate);
|
|
3551
|
+
return true;
|
|
3552
|
+
} catch (err) {
|
|
3553
|
+
if (bestFailed === null) {
|
|
3554
|
+
bestFailed = {
|
|
3555
|
+
candidate,
|
|
3556
|
+
position: extractParsePosition(err),
|
|
3557
|
+
message: safeErrorToString(err)
|
|
3558
|
+
};
|
|
3559
|
+
}
|
|
3560
|
+
return false;
|
|
3561
|
+
}
|
|
3562
|
+
}
|
|
3563
|
+
while (i < text.length) {
|
|
3564
|
+
const ch = text[i];
|
|
3565
|
+
if (stack.length === 0 && !inString) {
|
|
3566
|
+
if (ch === "{" || ch === "[") {
|
|
3567
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
3568
|
+
out += ch;
|
|
3569
|
+
}
|
|
3570
|
+
i++;
|
|
3571
|
+
continue;
|
|
3572
|
+
}
|
|
3573
|
+
if (escape) {
|
|
3574
|
+
out += ch;
|
|
3575
|
+
escape = false;
|
|
3576
|
+
i++;
|
|
3577
|
+
continue;
|
|
3578
|
+
}
|
|
3579
|
+
if (inString && ch === "\\") {
|
|
3580
|
+
out += ch;
|
|
3581
|
+
escape = true;
|
|
3582
|
+
i++;
|
|
3583
|
+
continue;
|
|
3584
|
+
}
|
|
3585
|
+
if (inString) {
|
|
3586
|
+
const code = ch.charCodeAt(0);
|
|
3587
|
+
if (code < 32) {
|
|
3588
|
+
out += controlCharEscape(code);
|
|
3589
|
+
i++;
|
|
3590
|
+
continue;
|
|
3591
|
+
}
|
|
3592
|
+
}
|
|
3593
|
+
if (ch === '"') {
|
|
3594
|
+
let j = i + 1;
|
|
3595
|
+
while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
|
|
3596
|
+
j++;
|
|
3597
|
+
}
|
|
3598
|
+
const next = j < text.length ? text[j] : "";
|
|
3599
|
+
if (!inString) {
|
|
3600
|
+
if (next === "}" || next === "]") {
|
|
3601
|
+
if (j > i + 1) out += text.slice(i + 1, j);
|
|
3602
|
+
out += next;
|
|
3603
|
+
i = j + 1;
|
|
3604
|
+
stack.pop();
|
|
3605
|
+
if (stack.length === 0) {
|
|
3606
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
3607
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
3608
|
+
out = "";
|
|
3609
|
+
}
|
|
3610
|
+
continue;
|
|
3611
|
+
}
|
|
3612
|
+
const top = stack[stack.length - 1];
|
|
3613
|
+
stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
|
|
3614
|
+
out += ch;
|
|
3615
|
+
inString = true;
|
|
3616
|
+
i++;
|
|
3617
|
+
continue;
|
|
3618
|
+
}
|
|
3619
|
+
const commaOnly = next === ",";
|
|
3620
|
+
const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
|
|
3621
|
+
const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
|
|
3622
|
+
if (commaOnly && bareQuoteCount > 0) ambiguous = true;
|
|
3623
|
+
const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
|
|
3624
|
+
const isClosing = isKeyClose || isValueClose || shouldCommaClose;
|
|
3625
|
+
if (isClosing) {
|
|
3626
|
+
out += ch;
|
|
3627
|
+
inString = false;
|
|
3628
|
+
const topFrame = stack[stack.length - 1];
|
|
3629
|
+
if (topFrame && topFrame.container === "object") {
|
|
3630
|
+
topFrame.expectKey = stringRole === "value";
|
|
3631
|
+
}
|
|
3632
|
+
stringRole = null;
|
|
3633
|
+
bareQuoteCount = 0;
|
|
3634
|
+
i++;
|
|
3635
|
+
continue;
|
|
3636
|
+
}
|
|
3637
|
+
out += "\\" + ch;
|
|
3638
|
+
bareQuoteCount++;
|
|
3639
|
+
escapes++;
|
|
3640
|
+
i++;
|
|
3641
|
+
continue;
|
|
3642
|
+
}
|
|
3643
|
+
if (!inString && (ch === "{" || ch === "[")) {
|
|
3644
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
3645
|
+
out += ch;
|
|
3646
|
+
i++;
|
|
3647
|
+
continue;
|
|
3648
|
+
}
|
|
3649
|
+
if (!inString && ch === ",") {
|
|
3650
|
+
const topFrame = stack[stack.length - 1];
|
|
3651
|
+
if (topFrame && topFrame.container === "object") {
|
|
3652
|
+
topFrame.expectKey = true;
|
|
3653
|
+
}
|
|
3654
|
+
out += ch;
|
|
3655
|
+
i++;
|
|
3656
|
+
continue;
|
|
3657
|
+
}
|
|
3658
|
+
if (!inString && (ch === "}" || ch === "]")) {
|
|
3659
|
+
out += ch;
|
|
3660
|
+
stack.pop();
|
|
3661
|
+
if (stack.length === 0) {
|
|
3662
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
3663
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
3664
|
+
out = "";
|
|
3665
|
+
}
|
|
3666
|
+
i++;
|
|
3667
|
+
continue;
|
|
3668
|
+
}
|
|
3669
|
+
out += ch;
|
|
3670
|
+
i++;
|
|
3671
|
+
}
|
|
3672
|
+
return { success: null, failed: bestFailed, escapes, ambiguous };
|
|
3673
|
+
}
|
|
3674
|
+
function controlCharEscape(code) {
|
|
3675
|
+
switch (code) {
|
|
3676
|
+
case 10:
|
|
3677
|
+
return "\\n";
|
|
3678
|
+
case 9:
|
|
3679
|
+
return "\\t";
|
|
3680
|
+
case 13:
|
|
3681
|
+
return "\\r";
|
|
3682
|
+
case 8:
|
|
3683
|
+
return "\\b";
|
|
3684
|
+
case 12:
|
|
3685
|
+
return "\\f";
|
|
3686
|
+
default:
|
|
3687
|
+
return "\\u" + code.toString(16).padStart(4, "0");
|
|
3688
|
+
}
|
|
3689
|
+
}
|
|
3690
|
+
function extractParsePosition(err) {
|
|
3691
|
+
const match = /position\s+(\d+)/.exec(safeErrorToString(err));
|
|
3692
|
+
return match ? Number(match[1]) : null;
|
|
3693
|
+
}
|
|
3694
|
+
function safeErrorToString(e) {
|
|
3695
|
+
if (e instanceof Error) {
|
|
3696
|
+
const msg = readErrorField(e, "message");
|
|
3697
|
+
if (typeof msg === "string" && msg.length > 0) return msg;
|
|
3698
|
+
const name = readErrorField(e, "name");
|
|
3699
|
+
if (typeof name === "string" && name.length > 0) return name;
|
|
3700
|
+
return "[Error]";
|
|
3701
|
+
}
|
|
3702
|
+
try {
|
|
3703
|
+
return String(e);
|
|
3704
|
+
} catch {
|
|
3705
|
+
try {
|
|
3706
|
+
return Object.prototype.toString.call(e);
|
|
3707
|
+
} catch {
|
|
3708
|
+
return "[unstringifiable error]";
|
|
3709
|
+
}
|
|
3710
|
+
}
|
|
3711
|
+
}
|
|
3712
|
+
function readErrorField(e, key) {
|
|
3713
|
+
try {
|
|
3714
|
+
return e[key];
|
|
3715
|
+
} catch {
|
|
3716
|
+
return void 0;
|
|
3717
|
+
}
|
|
3471
3718
|
}
|
|
3472
3719
|
function sanitizeRankerError(err, sanitizeRankerErrors) {
|
|
3473
3720
|
if (sanitizeRankerErrors === false) {
|
|
@@ -3652,7 +3899,7 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
3652
3899
|
{
|
|
3653
3900
|
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
3654
3901
|
}
|
|
3655
|
-
Extract verbatim factual content. Do not return markdown, just raw JSON.`;
|
|
3902
|
+
Extract verbatim factual content. JSON escaping rules: every literal " character in the source must be escaped as \\" inside any JSON string body, and every literal newline as \\n. Source prose containing quotes (e.g. a worked example with "...") must still be reproduced exactly \u2014 re-escape, do not omit. Do not return markdown, just raw JSON.`;
|
|
3656
3903
|
var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
|
|
3657
3904
|
Return ONLY a valid JSON object matching this schema:
|
|
3658
3905
|
{
|
|
@@ -3661,7 +3908,7 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
3661
3908
|
]
|
|
3662
3909
|
}
|
|
3663
3910
|
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
3664
|
-
Do not return markdown, just raw JSON.`;
|
|
3911
|
+
When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
|
|
3665
3912
|
|
|
3666
3913
|
// src/services/PromptService.ts
|
|
3667
3914
|
var PromptService = class {
|
|
@@ -3774,6 +4021,16 @@ var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
|
3774
4021
|
var DEFAULT_CHUNK_OVERLAP = 400;
|
|
3775
4022
|
|
|
3776
4023
|
// src/services/IngestionService.ts
|
|
4024
|
+
function zeroChunkResult(duplicateOf) {
|
|
4025
|
+
const result = {
|
|
4026
|
+
truncated: false,
|
|
4027
|
+
chunks: 0,
|
|
4028
|
+
ingestedChunks: 0,
|
|
4029
|
+
failedChunks: 0
|
|
4030
|
+
};
|
|
4031
|
+
if (duplicateOf !== void 0) result.duplicateOf = duplicateOf;
|
|
4032
|
+
return result;
|
|
4033
|
+
}
|
|
3777
4034
|
var IngestionService = class {
|
|
3778
4035
|
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
3779
4036
|
this.db = db;
|
|
@@ -3814,113 +4071,110 @@ var IngestionService = class {
|
|
|
3814
4071
|
if (onDuplicateHash === "throw") {
|
|
3815
4072
|
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
3816
4073
|
}
|
|
3817
|
-
return
|
|
4074
|
+
return zeroChunkResult(canonical);
|
|
3818
4075
|
}
|
|
3819
4076
|
}
|
|
3820
4077
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
3821
|
-
if (chunks.length === 0) return
|
|
4078
|
+
if (chunks.length === 0) return zeroChunkResult();
|
|
4079
|
+
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
3822
4080
|
const chunkResults = await withConcurrency(
|
|
3823
|
-
chunks.map((chunk) => async () => {
|
|
3824
|
-
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
4081
|
+
chunks.map((chunk, chunkIndex) => async () => {
|
|
3825
4082
|
const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
|
|
3826
4083
|
chunk,
|
|
3827
4084
|
params.promptOverride,
|
|
3828
4085
|
ontologyContext
|
|
3829
4086
|
);
|
|
3830
|
-
|
|
3831
|
-
|
|
3832
|
-
|
|
3833
|
-
|
|
3834
|
-
|
|
3835
|
-
|
|
4087
|
+
try {
|
|
4088
|
+
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
4089
|
+
const result2 = parseJsonResponse(responseText);
|
|
4090
|
+
return {
|
|
4091
|
+
status: "ok",
|
|
4092
|
+
facts: (Array.isArray(result2.facts) ? result2.facts : []).map(validateFact).filter((f) => f !== null),
|
|
4093
|
+
ontology_updates: result2.ontology_updates
|
|
4094
|
+
};
|
|
4095
|
+
} catch (e) {
|
|
4096
|
+
const failure = e instanceof WikiParseError ? {
|
|
4097
|
+
chunkIndex,
|
|
4098
|
+
sourceRef,
|
|
4099
|
+
source: "parse",
|
|
4100
|
+
tier: e.tier,
|
|
4101
|
+
position: e.position,
|
|
4102
|
+
message: e.message
|
|
4103
|
+
} : {
|
|
4104
|
+
chunkIndex,
|
|
4105
|
+
sourceRef,
|
|
4106
|
+
source: "llm",
|
|
4107
|
+
// `safeErrorToString` is a non-throwing coercion: an LLM
|
|
4108
|
+
// provider may reject with a non-Error value, and even
|
|
4109
|
+
// `String(e)` itself can throw when `e.toString()` throws.
|
|
4110
|
+
// Letting such an exception escape this catch would cause
|
|
4111
|
+
// `withConcurrency` to reject, discarding every sibling
|
|
4112
|
+
// chunk's results — the exact failure mode this per-chunk
|
|
4113
|
+
// try/catch exists to prevent. Keeping the value in the
|
|
4114
|
+
// `ChunkFailure` preserves the typed diagnostic for
|
|
4115
|
+
// callers (see parseFailures) without unwinding the loop.
|
|
4116
|
+
position: null,
|
|
4117
|
+
message: safeErrorToString(e)
|
|
4118
|
+
};
|
|
4119
|
+
const total = chunks.length;
|
|
4120
|
+
const tags = [];
|
|
4121
|
+
if (failure.tier) tags.push(`tier=${failure.tier}`);
|
|
4122
|
+
if (failure.position !== null) tags.push(`position=${failure.position}`);
|
|
4123
|
+
const tagsSuffix = tags.length > 0 ? `; ${tags.join(" ")}` : "";
|
|
4124
|
+
console.warn(
|
|
4125
|
+
`[WikiMemory] ingest chunk ${chunkIndex + 1}/${total} ${failure.source} failed (sourceRef=${sourceRef}${tagsSuffix})`
|
|
4126
|
+
);
|
|
4127
|
+
return { status: "failed", error: failure };
|
|
4128
|
+
}
|
|
3836
4129
|
}),
|
|
3837
4130
|
chunkConcurrency
|
|
3838
4131
|
);
|
|
4132
|
+
let ingestedChunks = 0;
|
|
4133
|
+
let failedChunks = 0;
|
|
4134
|
+
const failures = [];
|
|
3839
4135
|
const seen = /* @__PURE__ */ new Set();
|
|
3840
4136
|
const orderedChunkFacts = [];
|
|
3841
|
-
for (const
|
|
4137
|
+
for (const slot of chunkResults) {
|
|
4138
|
+
if (slot.status === "failed") {
|
|
4139
|
+
failedChunks++;
|
|
4140
|
+
failures.push(slot.error);
|
|
4141
|
+
continue;
|
|
4142
|
+
}
|
|
4143
|
+
ingestedChunks++;
|
|
3842
4144
|
const dedupedFacts = [];
|
|
3843
|
-
for (const fact of
|
|
4145
|
+
for (const fact of slot.facts) {
|
|
3844
4146
|
const normalizedTitle = normalizeTitleKey(fact.title);
|
|
3845
4147
|
if (!seen.has(normalizedTitle)) {
|
|
3846
4148
|
seen.add(normalizedTitle);
|
|
3847
4149
|
dedupedFacts.push(fact);
|
|
3848
4150
|
}
|
|
3849
4151
|
}
|
|
3850
|
-
orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates:
|
|
4152
|
+
orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: slot.ontology_updates });
|
|
4153
|
+
}
|
|
4154
|
+
if (failedChunks === chunks.length) {
|
|
4155
|
+
throw new WikiIngestEmptyError({
|
|
4156
|
+
parseFailures: failures,
|
|
4157
|
+
sourceRef,
|
|
4158
|
+
chunks: chunks.length
|
|
4159
|
+
});
|
|
3851
4160
|
}
|
|
3852
|
-
const now = Date.now();
|
|
3853
4161
|
const insertedFacts = [];
|
|
3854
4162
|
const deletedSourceFactIds = [];
|
|
3855
4163
|
try {
|
|
3856
|
-
|
|
3857
|
-
|
|
3858
|
-
|
|
3859
|
-
|
|
3860
|
-
|
|
3861
|
-
|
|
3862
|
-
|
|
3863
|
-
|
|
3864
|
-
|
|
3865
|
-
|
|
3866
|
-
|
|
3867
|
-
|
|
3868
|
-
|
|
3869
|
-
|
|
3870
|
-
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
3871
|
-
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
3872
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
3873
|
-
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
3874
|
-
mode = ontologyState.mode;
|
|
3875
|
-
}
|
|
3876
|
-
for (const fact of facts) {
|
|
3877
|
-
const ontologyFact = fact;
|
|
3878
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
3879
|
-
const id = generateId("fact_");
|
|
3880
|
-
hostNodes.push({
|
|
3881
|
-
id,
|
|
3882
|
-
type: ontologyFact.okf_type ?? "",
|
|
3883
|
-
title: fact.title,
|
|
3884
|
-
body: fact.body,
|
|
3885
|
-
// Forward the LLM-extracted tags/confidence through the
|
|
3886
|
-
// extracted-shape upsertGraphCore path so the entries row
|
|
3887
|
-
// stores them (search filterability, heal-candidate
|
|
3888
|
-
// selection, runReembed embedding-text signal). The public
|
|
3889
|
-
// WikiMemory.upsertGraph API doesn't accept tags/confidence
|
|
3890
|
-
// — host-supplied deterministic nodes default to [] and
|
|
3891
|
-
// 'certain' as documented.
|
|
3892
|
-
tags: fact.tags,
|
|
3893
|
-
confidence: fact.confidence
|
|
3894
|
-
});
|
|
3895
|
-
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
3896
|
-
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
3897
|
-
if (normalized.edges.length > 0) {
|
|
3898
|
-
rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
3899
|
-
}
|
|
3900
|
-
}
|
|
3901
|
-
}
|
|
3902
|
-
const hostEdges = [];
|
|
3903
|
-
for (const req of rawEdgeRequests) {
|
|
3904
|
-
const resolved = this.ontologyService?.resolveEdges(
|
|
3905
|
-
entityId,
|
|
3906
|
-
req.sourceId,
|
|
3907
|
-
req.sourceType,
|
|
3908
|
-
req.edges,
|
|
3909
|
-
manifest,
|
|
3910
|
-
titleIndex,
|
|
3911
|
-
now
|
|
3912
|
-
) ?? [];
|
|
3913
|
-
for (const e of resolved) {
|
|
3914
|
-
hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
|
|
3915
|
-
}
|
|
3916
|
-
}
|
|
3917
|
-
await this.upsertGraphCore(
|
|
3918
|
-
entityId,
|
|
3919
|
-
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
3920
|
-
tx,
|
|
3921
|
-
{ strict: false }
|
|
3922
|
-
);
|
|
3923
|
-
});
|
|
4164
|
+
if (failedChunks === 0) {
|
|
4165
|
+
const fullResult = await this.db.withTransactionAsync(async (tx) => {
|
|
4166
|
+
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx);
|
|
4167
|
+
});
|
|
4168
|
+
deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
|
|
4169
|
+
insertedFacts.push(...fullResult.insertedFacts);
|
|
4170
|
+
} else {
|
|
4171
|
+
const partialResult = await this.db.withTransactionAsync(async (tx) => {
|
|
4172
|
+
const flat = [];
|
|
4173
|
+
for (const slot of orderedChunkFacts) flat.push(...slot.facts);
|
|
4174
|
+
return await this.appendPartialFacts(entityId, sourceRef, flat, tx);
|
|
4175
|
+
});
|
|
4176
|
+
insertedFacts.push(...partialResult.insertedDescriptors);
|
|
4177
|
+
}
|
|
3924
4178
|
} catch (err) {
|
|
3925
4179
|
const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
|
|
3926
4180
|
if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
|
|
@@ -3929,22 +4183,31 @@ var IngestionService = class {
|
|
|
3929
4183
|
if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
|
|
3930
4184
|
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
3931
4185
|
}
|
|
3932
|
-
return
|
|
4186
|
+
return zeroChunkResult(canonical);
|
|
3933
4187
|
}
|
|
3934
4188
|
await this.searchService.sync(entityId);
|
|
3935
|
-
|
|
3936
|
-
|
|
3937
|
-
|
|
3938
|
-
|
|
3939
|
-
|
|
3940
|
-
|
|
4189
|
+
if (failedChunks === 0) {
|
|
4190
|
+
const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
|
|
4191
|
+
for (const factId of uniqueDeletedSourceFactIds) {
|
|
4192
|
+
try {
|
|
4193
|
+
await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
|
|
4194
|
+
} catch (hookErr) {
|
|
4195
|
+
console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
|
|
4196
|
+
}
|
|
3941
4197
|
}
|
|
3942
4198
|
}
|
|
3943
4199
|
for (const fact of insertedFacts) {
|
|
3944
4200
|
await this.embeddingService.embedFact(fact);
|
|
3945
4201
|
}
|
|
3946
4202
|
this.searchService.evictCache(entityId);
|
|
3947
|
-
|
|
4203
|
+
const result = {
|
|
4204
|
+
truncated,
|
|
4205
|
+
chunks: chunks.length,
|
|
4206
|
+
ingestedChunks,
|
|
4207
|
+
failedChunks
|
|
4208
|
+
};
|
|
4209
|
+
if (failedChunks > 0) result.parseFailures = failures;
|
|
4210
|
+
return result;
|
|
3948
4211
|
} finally {
|
|
3949
4212
|
releaseIngestLocks();
|
|
3950
4213
|
}
|
|
@@ -4069,6 +4332,156 @@ var IngestionService = class {
|
|
|
4069
4332
|
superseded: deletedFactIds.length + deletedEdgeCount
|
|
4070
4333
|
};
|
|
4071
4334
|
}
|
|
4335
|
+
/**
|
|
4336
|
+
* Full supersession + ownership path: identical to the pre-issue-#92 behavior
|
|
4337
|
+
* for the happy path. Called from the `failedChunks === 0` branch of
|
|
4338
|
+
* `ingestDocument`. Runs INSIDE the caller's tx.
|
|
4339
|
+
*
|
|
4340
|
+
* Returns `{ deletedSourceFactIds, insertedFacts }` so the caller can fire
|
|
4341
|
+
* post-commit hooks (embedding lifecycle) AFTER the transaction commits.
|
|
4342
|
+
* On the partial path the caller never invokes this method; the empty
|
|
4343
|
+
* arrays stay empty.
|
|
4344
|
+
*/
|
|
4345
|
+
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx) {
|
|
4346
|
+
const deletedSourceFactIds = [];
|
|
4347
|
+
const insertedFacts = [];
|
|
4348
|
+
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
4349
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
4350
|
+
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
4351
|
+
for (const existing of existingFacts) {
|
|
4352
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
4353
|
+
id: existing.id,
|
|
4354
|
+
okf_type: existing.okf_type ?? null
|
|
4355
|
+
});
|
|
4356
|
+
}
|
|
4357
|
+
let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
4358
|
+
let { mode, manifest } = ontologyState;
|
|
4359
|
+
const hostNodes = [];
|
|
4360
|
+
const rawEdgeRequests = [];
|
|
4361
|
+
const now = Date.now();
|
|
4362
|
+
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
4363
|
+
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
4364
|
+
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
4365
|
+
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
4366
|
+
mode = ontologyState.mode;
|
|
4367
|
+
}
|
|
4368
|
+
for (const fact of facts) {
|
|
4369
|
+
const ontologyFact = fact;
|
|
4370
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
4371
|
+
const id = generateId("fact_");
|
|
4372
|
+
hostNodes.push({
|
|
4373
|
+
id,
|
|
4374
|
+
type: ontologyFact.okf_type ?? "",
|
|
4375
|
+
title: fact.title,
|
|
4376
|
+
body: fact.body,
|
|
4377
|
+
// Forward the LLM-extracted tags/confidence through the
|
|
4378
|
+
// extracted-shape upsertGraphCore path so the entries row
|
|
4379
|
+
// stores them (search filterability, heal-candidate
|
|
4380
|
+
// selection, runReembed embedding-text signal). The public
|
|
4381
|
+
// WikiMemory.upsertGraph API doesn't accept tags/confidence
|
|
4382
|
+
// — host-supplied deterministic nodes default to [] and
|
|
4383
|
+
// 'certain' as documented.
|
|
4384
|
+
tags: fact.tags,
|
|
4385
|
+
confidence: fact.confidence
|
|
4386
|
+
});
|
|
4387
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
4388
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
4389
|
+
if (normalized.edges.length > 0) {
|
|
4390
|
+
rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
4391
|
+
}
|
|
4392
|
+
}
|
|
4393
|
+
}
|
|
4394
|
+
const hostEdges = [];
|
|
4395
|
+
for (const req of rawEdgeRequests) {
|
|
4396
|
+
const resolved = this.ontologyService?.resolveEdges(
|
|
4397
|
+
entityId,
|
|
4398
|
+
req.sourceId,
|
|
4399
|
+
req.sourceType,
|
|
4400
|
+
req.edges,
|
|
4401
|
+
manifest,
|
|
4402
|
+
titleIndex,
|
|
4403
|
+
now
|
|
4404
|
+
) ?? [];
|
|
4405
|
+
for (const e of resolved) {
|
|
4406
|
+
hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
|
|
4407
|
+
}
|
|
4408
|
+
}
|
|
4409
|
+
await this.upsertGraphCore(
|
|
4410
|
+
entityId,
|
|
4411
|
+
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
4412
|
+
tx,
|
|
4413
|
+
{ strict: false }
|
|
4414
|
+
);
|
|
4415
|
+
return { deletedSourceFactIds, insertedFacts };
|
|
4416
|
+
}
|
|
4417
|
+
/**
|
|
4418
|
+
* Partial-commit path for `ingestDocument`. Inserts ONLY facts whose
|
|
4419
|
+
* normalized title is not already in the live set for this (entityId,
|
|
4420
|
+
* sourceRef). Does NOT call `entryRepo.softDeleteBySource` (no
|
|
4421
|
+
* supersession) and does NOT call `sourceRefIndexRepo.upsert` (no
|
|
4422
|
+
* ownership). Edges are NOT resolved on this path (no ontology-context
|
|
4423
|
+
* build, no `resolveEdges`, no `mergeEmergentUpdates`). Stale edges from
|
|
4424
|
+
* prior attempts are recovered on the next full run's supersession.
|
|
4425
|
+
*
|
|
4426
|
+
* Source-hash is stored as NULL on partial rows. `findLatestSourceHash`
|
|
4427
|
+
* reads from the most recently updated live row for the sourceRef, so
|
|
4428
|
+
* storing the incoming hash here would cause `hasChanged` to return
|
|
4429
|
+
* `false` on a same-hash retry — the failed chunks would never get a
|
|
4430
|
+
* second chance. With NULL, a retry sees `storedHash === null`, `hasChanged`
|
|
4431
|
+
* returns `true`, and the partial commit's sibling rows remain live
|
|
4432
|
+
* (deduped, not superseded) for the next attempt to extend.
|
|
4433
|
+
*
|
|
4434
|
+
* Runs INSIDE the caller's `tx`. Does not open a nested transaction.
|
|
4435
|
+
* Returns `{ inserted, skippedDuplicate }` for observability.
|
|
4436
|
+
*/
|
|
4437
|
+
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx) {
|
|
4438
|
+
const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
|
|
4439
|
+
const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
|
|
4440
|
+
const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
|
|
4441
|
+
let inserted = 0;
|
|
4442
|
+
let skippedDuplicate = 0;
|
|
4443
|
+
const now = Date.now();
|
|
4444
|
+
const insertedDescriptors = [];
|
|
4445
|
+
for (const fact of dedupedFacts) {
|
|
4446
|
+
const normalizedTitle = normalizeTitleKey(fact.title);
|
|
4447
|
+
if (liveTitles.has(normalizedTitle)) {
|
|
4448
|
+
skippedDuplicate++;
|
|
4449
|
+
continue;
|
|
4450
|
+
}
|
|
4451
|
+
liveTitles.add(normalizedTitle);
|
|
4452
|
+
const id = generateId("fact_");
|
|
4453
|
+
const wikiFact = {
|
|
4454
|
+
id,
|
|
4455
|
+
entity_id: entityId,
|
|
4456
|
+
title: fact.title,
|
|
4457
|
+
body: fact.body,
|
|
4458
|
+
tags: fact.tags,
|
|
4459
|
+
confidence: fact.confidence,
|
|
4460
|
+
source_type: "immutable_document",
|
|
4461
|
+
// source_hash: null on partial rows — see method docstring. The
|
|
4462
|
+
// full path stamps the actual hash; partial rows stay hash-less so
|
|
4463
|
+
// `hasChanged` keeps returning true for retries.
|
|
4464
|
+
source_hash: null,
|
|
4465
|
+
source_ref: sourceRef,
|
|
4466
|
+
created_at: now,
|
|
4467
|
+
updated_at: now,
|
|
4468
|
+
last_accessed_at: null,
|
|
4469
|
+
access_count: 0,
|
|
4470
|
+
deleted_at: null,
|
|
4471
|
+
okf_type: null
|
|
4472
|
+
};
|
|
4473
|
+
await this.entryRepo.upsert(wikiFact, tx);
|
|
4474
|
+
insertedDescriptors.push({
|
|
4475
|
+
id,
|
|
4476
|
+
entity_id: entityId,
|
|
4477
|
+
title: fact.title,
|
|
4478
|
+
body: fact.body,
|
|
4479
|
+
tags: JSON.stringify(fact.tags)
|
|
4480
|
+
});
|
|
4481
|
+
inserted++;
|
|
4482
|
+
}
|
|
4483
|
+
return { inserted, skippedDuplicate, insertedDescriptors };
|
|
4484
|
+
}
|
|
4072
4485
|
};
|
|
4073
4486
|
|
|
4074
4487
|
// src/services/BoundedLlmCall.ts
|
|
@@ -4189,6 +4602,25 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
|
|
|
4189
4602
|
var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
4190
4603
|
var HEAL_BATCH_SIZE = 25;
|
|
4191
4604
|
var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
4605
|
+
var SKIP_ERROR_LOG_CHARS = 4096;
|
|
4606
|
+
var formatSkipError = (err) => {
|
|
4607
|
+
let base;
|
|
4608
|
+
if (err instanceof Error || typeof err !== "object" && typeof err !== "function") {
|
|
4609
|
+
base = safeErrorToString(err);
|
|
4610
|
+
} else {
|
|
4611
|
+
try {
|
|
4612
|
+
const json = JSON.stringify(err);
|
|
4613
|
+
base = typeof json === "string" ? json : safeErrorToString(err);
|
|
4614
|
+
} catch {
|
|
4615
|
+
base = safeErrorToString(err);
|
|
4616
|
+
}
|
|
4617
|
+
}
|
|
4618
|
+
if (base.length > SKIP_ERROR_LOG_CHARS) {
|
|
4619
|
+
const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
|
|
4620
|
+
return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
|
|
4621
|
+
}
|
|
4622
|
+
return base;
|
|
4623
|
+
};
|
|
4192
4624
|
var MaintenanceService = class {
|
|
4193
4625
|
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
4194
4626
|
this.db = db;
|
|
@@ -4669,8 +5101,7 @@ var MaintenanceService = class {
|
|
|
4669
5101
|
maxPromptChars: HEAL_MAX_PROMPT_CHARS,
|
|
4670
5102
|
onSkip: (fact, err) => {
|
|
4671
5103
|
console.warn(
|
|
4672
|
-
`[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded
|
|
4673
|
-
err
|
|
5104
|
+
`[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
|
|
4674
5105
|
);
|
|
4675
5106
|
}
|
|
4676
5107
|
});
|
|
@@ -4808,8 +5239,7 @@ var MaintenanceService = class {
|
|
|
4808
5239
|
maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
|
|
4809
5240
|
onSkip: (fact, err) => {
|
|
4810
5241
|
console.warn(
|
|
4811
|
-
`[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded
|
|
4812
|
-
err
|
|
5242
|
+
`[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
|
|
4813
5243
|
);
|
|
4814
5244
|
}
|
|
4815
5245
|
});
|
|
@@ -7989,7 +8419,9 @@ exports.PromptService = PromptService;
|
|
|
7989
8419
|
exports.PrunePartialFailureError = PrunePartialFailureError;
|
|
7990
8420
|
exports.WikiBusyError = WikiBusyError;
|
|
7991
8421
|
exports.WikiDuplicateHashError = WikiDuplicateHashError;
|
|
8422
|
+
exports.WikiIngestEmptyError = WikiIngestEmptyError;
|
|
7992
8423
|
exports.WikiMemory = WikiMemory;
|
|
8424
|
+
exports.WikiParseError = WikiParseError;
|
|
7993
8425
|
exports.WikiSourceRefHashCollision = WikiSourceRefHashCollision;
|
|
7994
8426
|
exports.WikiStrictOntologyViolation = WikiStrictOntologyViolation;
|
|
7995
8427
|
exports.WikiTransactionError = WikiTransactionError;
|