@equationalapplications/core-llm-wiki 5.4.0 → 5.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +162 -0
- package/dist/{chunk-N2NXN6PO.mjs → chunk-JQRA6BYQ.mjs} +610 -113
- package/dist/chunk-JQRA6BYQ.mjs.map +1 -0
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +609 -110
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2 -2
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-C71mnMxq.d.mts → testing-D0RnZjyW.d.mts} +108 -11
- package/dist/{testing-C71mnMxq.d.ts → testing-D0RnZjyW.d.ts} +108 -11
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +607 -110
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-N2NXN6PO.mjs.map +0 -1
|
@@ -125,6 +125,25 @@ var PrunePartialFailureError = class extends Error {
|
|
|
125
125
|
}
|
|
126
126
|
};
|
|
127
127
|
var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
|
|
128
|
+
var WikiParseError = class extends Error {
|
|
129
|
+
constructor(message, opts) {
|
|
130
|
+
super(message);
|
|
131
|
+
this.name = "WikiParseError";
|
|
132
|
+
this.tier = opts.tier;
|
|
133
|
+
this.position = opts.position ?? null;
|
|
134
|
+
this.slice = opts.slice ?? "";
|
|
135
|
+
}
|
|
136
|
+
};
|
|
137
|
+
var WikiIngestEmptyError = class extends Error {
|
|
138
|
+
constructor(params) {
|
|
139
|
+
const summary = `All ${params.chunks} chunks failed for sourceRef "${params.sourceRef}"; see parseFailures for per-chunk detail`;
|
|
140
|
+
super(summary);
|
|
141
|
+
this.name = "WikiIngestEmptyError";
|
|
142
|
+
this.parseFailures = params.parseFailures;
|
|
143
|
+
this.sourceRef = params.sourceRef;
|
|
144
|
+
this.chunks = params.chunks;
|
|
145
|
+
}
|
|
146
|
+
};
|
|
128
147
|
|
|
129
148
|
// src/utils/ids.ts
|
|
130
149
|
var _injectedGetRandomValues = null;
|
|
@@ -945,24 +964,67 @@ var JobManager = class {
|
|
|
945
964
|
};
|
|
946
965
|
|
|
947
966
|
// src/utils/pure.ts
|
|
967
|
+
var MAX_REPAIR_CANDIDATES = 5;
|
|
968
|
+
function throwNoJsonFound(text, start) {
|
|
969
|
+
throw new WikiParseError(
|
|
970
|
+
"No JSON object/array found in LLM response",
|
|
971
|
+
{ tier: "strict", position: start, slice: text }
|
|
972
|
+
);
|
|
973
|
+
}
|
|
948
974
|
function parseJsonResponse(text) {
|
|
949
975
|
const firstBrace = text.indexOf("{");
|
|
950
976
|
const firstBracket = text.indexOf("[");
|
|
951
977
|
let start;
|
|
952
978
|
let openChar;
|
|
953
|
-
let closeChar;
|
|
954
979
|
const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
|
|
955
980
|
if (useBrace) {
|
|
956
981
|
start = firstBrace;
|
|
957
982
|
openChar = "{";
|
|
958
|
-
closeChar = "}";
|
|
959
983
|
} else if (firstBracket !== -1) {
|
|
960
984
|
start = firstBracket;
|
|
961
985
|
openChar = "[";
|
|
962
|
-
closeChar = "]";
|
|
963
986
|
} else {
|
|
964
|
-
|
|
987
|
+
throwNoJsonFound(text, null);
|
|
988
|
+
}
|
|
989
|
+
const slice = scanJsonSlice(text, start, openChar);
|
|
990
|
+
if (slice !== null) {
|
|
991
|
+
try {
|
|
992
|
+
return JSON.parse(slice);
|
|
993
|
+
} catch {
|
|
994
|
+
}
|
|
995
|
+
}
|
|
996
|
+
const repairOdd = containerAwareRepair(text, start, false);
|
|
997
|
+
if (repairOdd.success !== null && !repairOdd.ambiguous) {
|
|
998
|
+
return repairOdd.success;
|
|
965
999
|
}
|
|
1000
|
+
const repairEven = containerAwareRepair(text, start, true);
|
|
1001
|
+
if (repairOdd.success !== null && repairEven.success !== null) {
|
|
1002
|
+
const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
|
|
1003
|
+
return pick.success;
|
|
1004
|
+
}
|
|
1005
|
+
if (repairOdd.success !== null) {
|
|
1006
|
+
return repairOdd.success;
|
|
1007
|
+
}
|
|
1008
|
+
if (repairEven.success !== null) {
|
|
1009
|
+
return repairEven.success;
|
|
1010
|
+
}
|
|
1011
|
+
const repair = repairEven.failed !== null ? repairEven : repairOdd;
|
|
1012
|
+
if (repair.failed !== null) {
|
|
1013
|
+
throw new WikiParseError(
|
|
1014
|
+
`Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
|
|
1015
|
+
{ tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
|
|
1016
|
+
);
|
|
1017
|
+
}
|
|
1018
|
+
if (slice === null) {
|
|
1019
|
+
throwNoJsonFound(text, start);
|
|
1020
|
+
}
|
|
1021
|
+
throw new WikiParseError(
|
|
1022
|
+
"No parsable JSON candidate found",
|
|
1023
|
+
{ tier: "all", position: start, slice: text }
|
|
1024
|
+
);
|
|
1025
|
+
}
|
|
1026
|
+
function scanJsonSlice(text, start, openChar) {
|
|
1027
|
+
const closeChar = openChar === "{" ? "}" : "]";
|
|
966
1028
|
let depth = 0;
|
|
967
1029
|
let inString = false;
|
|
968
1030
|
let escape = false;
|
|
@@ -994,20 +1056,248 @@ function parseJsonResponse(text) {
|
|
|
994
1056
|
}
|
|
995
1057
|
}
|
|
996
1058
|
}
|
|
997
|
-
if (end === -1)
|
|
998
|
-
return
|
|
1059
|
+
if (end === -1) return null;
|
|
1060
|
+
return safeSlice(text, start, end + 1);
|
|
1061
|
+
}
|
|
1062
|
+
function containerAwareRepair(text, start, closeOnEvenParity) {
|
|
1063
|
+
const stack = [];
|
|
1064
|
+
let inString = false;
|
|
1065
|
+
let stringRole = null;
|
|
1066
|
+
let escape = false;
|
|
1067
|
+
let i = start;
|
|
1068
|
+
let out = "";
|
|
1069
|
+
let bareQuoteCount = 0;
|
|
1070
|
+
let bestFailed = null;
|
|
1071
|
+
let escapes = 0;
|
|
1072
|
+
let ambiguous = false;
|
|
1073
|
+
let bestSuccess = null;
|
|
1074
|
+
let attempts = 0;
|
|
1075
|
+
function tryEmitCandidate(candidate) {
|
|
1076
|
+
attempts++;
|
|
1077
|
+
try {
|
|
1078
|
+
bestSuccess = JSON.parse(candidate);
|
|
1079
|
+
return true;
|
|
1080
|
+
} catch (err) {
|
|
1081
|
+
if (bestFailed === null) {
|
|
1082
|
+
bestFailed = {
|
|
1083
|
+
candidate,
|
|
1084
|
+
position: extractParsePosition(err),
|
|
1085
|
+
message: safeErrorToString(err)
|
|
1086
|
+
};
|
|
1087
|
+
}
|
|
1088
|
+
return false;
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
while (i < text.length) {
|
|
1092
|
+
const ch = text[i];
|
|
1093
|
+
if (stack.length === 0 && !inString) {
|
|
1094
|
+
if (ch === "{" || ch === "[") {
|
|
1095
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
1096
|
+
out += ch;
|
|
1097
|
+
}
|
|
1098
|
+
i++;
|
|
1099
|
+
continue;
|
|
1100
|
+
}
|
|
1101
|
+
if (escape) {
|
|
1102
|
+
out += ch;
|
|
1103
|
+
escape = false;
|
|
1104
|
+
i++;
|
|
1105
|
+
continue;
|
|
1106
|
+
}
|
|
1107
|
+
if (inString && ch === "\\") {
|
|
1108
|
+
out += ch;
|
|
1109
|
+
escape = true;
|
|
1110
|
+
i++;
|
|
1111
|
+
continue;
|
|
1112
|
+
}
|
|
1113
|
+
if (inString) {
|
|
1114
|
+
const code = ch.charCodeAt(0);
|
|
1115
|
+
if (code < 32) {
|
|
1116
|
+
out += controlCharEscape(code);
|
|
1117
|
+
i++;
|
|
1118
|
+
continue;
|
|
1119
|
+
}
|
|
1120
|
+
}
|
|
1121
|
+
if (ch === '"') {
|
|
1122
|
+
let j = i + 1;
|
|
1123
|
+
while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
|
|
1124
|
+
j++;
|
|
1125
|
+
}
|
|
1126
|
+
const next = j < text.length ? text[j] : "";
|
|
1127
|
+
if (!inString) {
|
|
1128
|
+
if (next === "}" || next === "]") {
|
|
1129
|
+
if (j > i + 1) out += text.slice(i + 1, j);
|
|
1130
|
+
out += next;
|
|
1131
|
+
i = j + 1;
|
|
1132
|
+
stack.pop();
|
|
1133
|
+
if (stack.length === 0) {
|
|
1134
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
1135
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
1136
|
+
out = "";
|
|
1137
|
+
}
|
|
1138
|
+
continue;
|
|
1139
|
+
}
|
|
1140
|
+
const top = stack[stack.length - 1];
|
|
1141
|
+
stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
|
|
1142
|
+
out += ch;
|
|
1143
|
+
inString = true;
|
|
1144
|
+
i++;
|
|
1145
|
+
continue;
|
|
1146
|
+
}
|
|
1147
|
+
const commaOnly = next === ",";
|
|
1148
|
+
const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
|
|
1149
|
+
const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
|
|
1150
|
+
if (commaOnly && bareQuoteCount > 0) ambiguous = true;
|
|
1151
|
+
const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
|
|
1152
|
+
const isClosing = isKeyClose || isValueClose || shouldCommaClose;
|
|
1153
|
+
if (isClosing) {
|
|
1154
|
+
out += ch;
|
|
1155
|
+
inString = false;
|
|
1156
|
+
const topFrame = stack[stack.length - 1];
|
|
1157
|
+
if (topFrame && topFrame.container === "object") {
|
|
1158
|
+
topFrame.expectKey = stringRole === "value";
|
|
1159
|
+
}
|
|
1160
|
+
stringRole = null;
|
|
1161
|
+
bareQuoteCount = 0;
|
|
1162
|
+
i++;
|
|
1163
|
+
continue;
|
|
1164
|
+
}
|
|
1165
|
+
out += "\\" + ch;
|
|
1166
|
+
bareQuoteCount++;
|
|
1167
|
+
escapes++;
|
|
1168
|
+
i++;
|
|
1169
|
+
continue;
|
|
1170
|
+
}
|
|
1171
|
+
if (!inString && (ch === "{" || ch === "[")) {
|
|
1172
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
1173
|
+
out += ch;
|
|
1174
|
+
i++;
|
|
1175
|
+
continue;
|
|
1176
|
+
}
|
|
1177
|
+
if (!inString && ch === ",") {
|
|
1178
|
+
const topFrame = stack[stack.length - 1];
|
|
1179
|
+
if (topFrame && topFrame.container === "object") {
|
|
1180
|
+
topFrame.expectKey = true;
|
|
1181
|
+
}
|
|
1182
|
+
out += ch;
|
|
1183
|
+
i++;
|
|
1184
|
+
continue;
|
|
1185
|
+
}
|
|
1186
|
+
if (!inString && (ch === "}" || ch === "]")) {
|
|
1187
|
+
out += ch;
|
|
1188
|
+
stack.pop();
|
|
1189
|
+
if (stack.length === 0) {
|
|
1190
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
1191
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
1192
|
+
out = "";
|
|
1193
|
+
}
|
|
1194
|
+
i++;
|
|
1195
|
+
continue;
|
|
1196
|
+
}
|
|
1197
|
+
out += ch;
|
|
1198
|
+
i++;
|
|
1199
|
+
}
|
|
1200
|
+
return { success: null, failed: bestFailed, escapes, ambiguous };
|
|
1201
|
+
}
|
|
1202
|
+
function controlCharEscape(code) {
|
|
1203
|
+
switch (code) {
|
|
1204
|
+
case 10:
|
|
1205
|
+
return "\\n";
|
|
1206
|
+
case 9:
|
|
1207
|
+
return "\\t";
|
|
1208
|
+
case 13:
|
|
1209
|
+
return "\\r";
|
|
1210
|
+
case 8:
|
|
1211
|
+
return "\\b";
|
|
1212
|
+
case 12:
|
|
1213
|
+
return "\\f";
|
|
1214
|
+
default:
|
|
1215
|
+
return "\\u" + code.toString(16).padStart(4, "0");
|
|
1216
|
+
}
|
|
1217
|
+
}
|
|
1218
|
+
function extractParsePosition(err) {
|
|
1219
|
+
const match = /position\s+(\d+)/.exec(safeErrorToString(err));
|
|
1220
|
+
return match ? Number(match[1]) : null;
|
|
1221
|
+
}
|
|
1222
|
+
function safeErrorToString(e) {
|
|
1223
|
+
let isErrorLike = false;
|
|
1224
|
+
try {
|
|
1225
|
+
isErrorLike = e instanceof Error;
|
|
1226
|
+
} catch {
|
|
1227
|
+
}
|
|
1228
|
+
if (isErrorLike) {
|
|
1229
|
+
const msg = readErrorField(e, "message");
|
|
1230
|
+
if (typeof msg === "string" && msg.length > 0) return msg;
|
|
1231
|
+
const name = readErrorField(e, "name");
|
|
1232
|
+
if (typeof name === "string" && name.length > 0) return name;
|
|
1233
|
+
return "[Error]";
|
|
1234
|
+
}
|
|
1235
|
+
try {
|
|
1236
|
+
return String(e);
|
|
1237
|
+
} catch {
|
|
1238
|
+
try {
|
|
1239
|
+
return Object.prototype.toString.call(e);
|
|
1240
|
+
} catch {
|
|
1241
|
+
return "[unstringifiable error]";
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
function readErrorField(e, key) {
|
|
1246
|
+
try {
|
|
1247
|
+
return e[key];
|
|
1248
|
+
} catch {
|
|
1249
|
+
return void 0;
|
|
1250
|
+
}
|
|
999
1251
|
}
|
|
1000
1252
|
function sanitizeRankerError(err, sanitizeRankerErrors) {
|
|
1253
|
+
let isErrorLike = false;
|
|
1254
|
+
try {
|
|
1255
|
+
isErrorLike = err instanceof Error;
|
|
1256
|
+
} catch {
|
|
1257
|
+
}
|
|
1001
1258
|
if (sanitizeRankerErrors === false) {
|
|
1002
|
-
return
|
|
1259
|
+
return isErrorLike ? err : new Error(safeErrorToString(err));
|
|
1260
|
+
}
|
|
1261
|
+
let errLike = null;
|
|
1262
|
+
if (isErrorLike) errLike = err;
|
|
1263
|
+
let typeName;
|
|
1264
|
+
try {
|
|
1265
|
+
if (errLike) {
|
|
1266
|
+
const rawName = errLike.constructor?.name;
|
|
1267
|
+
typeName = typeof rawName === "string" ? rawName : "Error";
|
|
1268
|
+
} else {
|
|
1269
|
+
typeName = typeof err;
|
|
1270
|
+
}
|
|
1271
|
+
} catch {
|
|
1272
|
+
typeName = isErrorLike ? "Error" : typeof err;
|
|
1273
|
+
}
|
|
1274
|
+
let innerCause;
|
|
1275
|
+
if (errLike) {
|
|
1276
|
+
let cause;
|
|
1277
|
+
try {
|
|
1278
|
+
cause = errLike.cause;
|
|
1279
|
+
} catch {
|
|
1280
|
+
cause = void 0;
|
|
1281
|
+
}
|
|
1282
|
+
if (cause !== void 0) {
|
|
1283
|
+
let causeName;
|
|
1284
|
+
try {
|
|
1285
|
+
const rawCauseName = cause?.constructor?.name;
|
|
1286
|
+
causeName = typeof rawCauseName === "string" ? rawCauseName : typeof cause;
|
|
1287
|
+
} catch {
|
|
1288
|
+
causeName = typeof cause;
|
|
1289
|
+
}
|
|
1290
|
+
innerCause = new Error(`Caused by: ${causeName}`);
|
|
1291
|
+
}
|
|
1003
1292
|
}
|
|
1004
|
-
const typeName = err instanceof Error ? err.constructor?.name ?? "Error" : typeof err;
|
|
1005
|
-
const innerCause = err instanceof Error && err.cause !== void 0 ? new Error(`Caused by: ${err.cause?.constructor?.name ?? typeof err.cause}`) : void 0;
|
|
1006
1293
|
const sanitized = new Error(
|
|
1007
1294
|
`VectorRanker ${typeName} (message scrubbed for security)`,
|
|
1008
1295
|
innerCause ? { cause: innerCause } : void 0
|
|
1009
1296
|
);
|
|
1010
|
-
|
|
1297
|
+
try {
|
|
1298
|
+
sanitized.name = typeName;
|
|
1299
|
+
} catch {
|
|
1300
|
+
}
|
|
1011
1301
|
return sanitized;
|
|
1012
1302
|
}
|
|
1013
1303
|
function safeSlice(value, start, end) {
|
|
@@ -1180,7 +1470,7 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
1180
1470
|
{
|
|
1181
1471
|
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
1182
1472
|
}
|
|
1183
|
-
Extract verbatim factual content. Do not return markdown, just raw JSON.`;
|
|
1473
|
+
Extract verbatim factual content. JSON escaping rules: every literal " character in the source must be escaped as \\" inside any JSON string body, and every literal newline as \\n. Source prose containing quotes (e.g. a worked example with "...") must still be reproduced exactly \u2014 re-escape, do not omit. Do not return markdown, just raw JSON.`;
|
|
1184
1474
|
var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
|
|
1185
1475
|
Return ONLY a valid JSON object matching this schema:
|
|
1186
1476
|
{
|
|
@@ -1189,7 +1479,7 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
1189
1479
|
]
|
|
1190
1480
|
}
|
|
1191
1481
|
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
1192
|
-
Do not return markdown, just raw JSON.`;
|
|
1482
|
+
When echoing an existing fact's title verbatim into "target_title", preserve every JSON escape sequence (\\", \\n, \\\\, \\/) exactly as it appeared in the input body \u2014 do not strip backslashes, do not add unescaped quotes. Do not return markdown, just raw JSON.`;
|
|
1193
1483
|
|
|
1194
1484
|
// src/services/PromptService.ts
|
|
1195
1485
|
var PromptService = class {
|
|
@@ -1302,6 +1592,16 @@ var DEFAULT_MAX_CHUNK_LENGTH = 12e3;
|
|
|
1302
1592
|
var DEFAULT_CHUNK_OVERLAP = 400;
|
|
1303
1593
|
|
|
1304
1594
|
// src/services/IngestionService.ts
|
|
1595
|
+
function zeroChunkResult(duplicateOf) {
|
|
1596
|
+
const result = {
|
|
1597
|
+
truncated: false,
|
|
1598
|
+
chunks: 0,
|
|
1599
|
+
ingestedChunks: 0,
|
|
1600
|
+
failedChunks: 0
|
|
1601
|
+
};
|
|
1602
|
+
if (duplicateOf !== void 0) result.duplicateOf = duplicateOf;
|
|
1603
|
+
return result;
|
|
1604
|
+
}
|
|
1305
1605
|
var IngestionService = class {
|
|
1306
1606
|
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, metadataRepo, edgeRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1307
1607
|
this.db = db;
|
|
@@ -1342,113 +1642,110 @@ var IngestionService = class {
|
|
|
1342
1642
|
if (onDuplicateHash === "throw") {
|
|
1343
1643
|
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
1344
1644
|
}
|
|
1345
|
-
return
|
|
1645
|
+
return zeroChunkResult(canonical);
|
|
1346
1646
|
}
|
|
1347
1647
|
}
|
|
1348
1648
|
const { chunks, truncated } = chunkText(params.documentChunk, maxChunkLength, chunkOverlap);
|
|
1349
|
-
if (chunks.length === 0) return
|
|
1649
|
+
if (chunks.length === 0) return zeroChunkResult();
|
|
1650
|
+
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
1350
1651
|
const chunkResults = await withConcurrency(
|
|
1351
|
-
chunks.map((chunk) => async () => {
|
|
1352
|
-
const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
|
|
1652
|
+
chunks.map((chunk, chunkIndex) => async () => {
|
|
1353
1653
|
const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
|
|
1354
1654
|
chunk,
|
|
1355
1655
|
params.promptOverride,
|
|
1356
1656
|
ontologyContext
|
|
1357
1657
|
);
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1658
|
+
try {
|
|
1659
|
+
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
1660
|
+
const result2 = parseJsonResponse(responseText);
|
|
1661
|
+
return {
|
|
1662
|
+
status: "ok",
|
|
1663
|
+
facts: (Array.isArray(result2.facts) ? result2.facts : []).map(validateFact).filter((f) => f !== null),
|
|
1664
|
+
ontology_updates: result2.ontology_updates
|
|
1665
|
+
};
|
|
1666
|
+
} catch (e) {
|
|
1667
|
+
const failure = e instanceof WikiParseError ? {
|
|
1668
|
+
chunkIndex,
|
|
1669
|
+
sourceRef,
|
|
1670
|
+
source: "parse",
|
|
1671
|
+
tier: e.tier,
|
|
1672
|
+
position: e.position,
|
|
1673
|
+
message: e.message
|
|
1674
|
+
} : {
|
|
1675
|
+
chunkIndex,
|
|
1676
|
+
sourceRef,
|
|
1677
|
+
source: "llm",
|
|
1678
|
+
// `safeErrorToString` is a non-throwing coercion: an LLM
|
|
1679
|
+
// provider may reject with a non-Error value, and even
|
|
1680
|
+
// `String(e)` itself can throw when `e.toString()` throws.
|
|
1681
|
+
// Letting such an exception escape this catch would cause
|
|
1682
|
+
// `withConcurrency` to reject, discarding every sibling
|
|
1683
|
+
// chunk's results — the exact failure mode this per-chunk
|
|
1684
|
+
// try/catch exists to prevent. Keeping the value in the
|
|
1685
|
+
// `ChunkFailure` preserves the typed diagnostic for
|
|
1686
|
+
// callers (see parseFailures) without unwinding the loop.
|
|
1687
|
+
position: null,
|
|
1688
|
+
message: safeErrorToString(e)
|
|
1689
|
+
};
|
|
1690
|
+
const total = chunks.length;
|
|
1691
|
+
const tags = [];
|
|
1692
|
+
if (failure.tier) tags.push(`tier=${failure.tier}`);
|
|
1693
|
+
if (failure.position !== null) tags.push(`position=${failure.position}`);
|
|
1694
|
+
const tagsSuffix = tags.length > 0 ? `; ${tags.join(" ")}` : "";
|
|
1695
|
+
console.warn(
|
|
1696
|
+
`[WikiMemory] ingest chunk ${chunkIndex + 1}/${total} ${failure.source} failed (sourceRef=${sourceRef}${tagsSuffix})`
|
|
1697
|
+
);
|
|
1698
|
+
return { status: "failed", error: failure };
|
|
1699
|
+
}
|
|
1364
1700
|
}),
|
|
1365
1701
|
chunkConcurrency
|
|
1366
1702
|
);
|
|
1703
|
+
let ingestedChunks = 0;
|
|
1704
|
+
let failedChunks = 0;
|
|
1705
|
+
const failures = [];
|
|
1367
1706
|
const seen = /* @__PURE__ */ new Set();
|
|
1368
1707
|
const orderedChunkFacts = [];
|
|
1369
|
-
for (const
|
|
1708
|
+
for (const slot of chunkResults) {
|
|
1709
|
+
if (slot.status === "failed") {
|
|
1710
|
+
failedChunks++;
|
|
1711
|
+
failures.push(slot.error);
|
|
1712
|
+
continue;
|
|
1713
|
+
}
|
|
1714
|
+
ingestedChunks++;
|
|
1370
1715
|
const dedupedFacts = [];
|
|
1371
|
-
for (const fact of
|
|
1716
|
+
for (const fact of slot.facts) {
|
|
1372
1717
|
const normalizedTitle = normalizeTitleKey(fact.title);
|
|
1373
1718
|
if (!seen.has(normalizedTitle)) {
|
|
1374
1719
|
seen.add(normalizedTitle);
|
|
1375
1720
|
dedupedFacts.push(fact);
|
|
1376
1721
|
}
|
|
1377
1722
|
}
|
|
1378
|
-
orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates:
|
|
1723
|
+
orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: slot.ontology_updates });
|
|
1724
|
+
}
|
|
1725
|
+
if (failedChunks === chunks.length) {
|
|
1726
|
+
throw new WikiIngestEmptyError({
|
|
1727
|
+
parseFailures: failures,
|
|
1728
|
+
sourceRef,
|
|
1729
|
+
chunks: chunks.length
|
|
1730
|
+
});
|
|
1379
1731
|
}
|
|
1380
|
-
const now = Date.now();
|
|
1381
1732
|
const insertedFacts = [];
|
|
1382
1733
|
const deletedSourceFactIds = [];
|
|
1383
1734
|
try {
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
1399
|
-
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
1400
|
-
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
1401
|
-
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
1402
|
-
mode = ontologyState.mode;
|
|
1403
|
-
}
|
|
1404
|
-
for (const fact of facts) {
|
|
1405
|
-
const ontologyFact = fact;
|
|
1406
|
-
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
1407
|
-
const id = generateId("fact_");
|
|
1408
|
-
hostNodes.push({
|
|
1409
|
-
id,
|
|
1410
|
-
type: ontologyFact.okf_type ?? "",
|
|
1411
|
-
title: fact.title,
|
|
1412
|
-
body: fact.body,
|
|
1413
|
-
// Forward the LLM-extracted tags/confidence through the
|
|
1414
|
-
// extracted-shape upsertGraphCore path so the entries row
|
|
1415
|
-
// stores them (search filterability, heal-candidate
|
|
1416
|
-
// selection, runReembed embedding-text signal). The public
|
|
1417
|
-
// WikiMemory.upsertGraph API doesn't accept tags/confidence
|
|
1418
|
-
// — host-supplied deterministic nodes default to [] and
|
|
1419
|
-
// 'certain' as documented.
|
|
1420
|
-
tags: fact.tags,
|
|
1421
|
-
confidence: fact.confidence
|
|
1422
|
-
});
|
|
1423
|
-
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
1424
|
-
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
1425
|
-
if (normalized.edges.length > 0) {
|
|
1426
|
-
rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
1427
|
-
}
|
|
1428
|
-
}
|
|
1429
|
-
}
|
|
1430
|
-
const hostEdges = [];
|
|
1431
|
-
for (const req of rawEdgeRequests) {
|
|
1432
|
-
const resolved = this.ontologyService?.resolveEdges(
|
|
1433
|
-
entityId,
|
|
1434
|
-
req.sourceId,
|
|
1435
|
-
req.sourceType,
|
|
1436
|
-
req.edges,
|
|
1437
|
-
manifest,
|
|
1438
|
-
titleIndex,
|
|
1439
|
-
now
|
|
1440
|
-
) ?? [];
|
|
1441
|
-
for (const e of resolved) {
|
|
1442
|
-
hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
|
|
1443
|
-
}
|
|
1444
|
-
}
|
|
1445
|
-
await this.upsertGraphCore(
|
|
1446
|
-
entityId,
|
|
1447
|
-
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
1448
|
-
tx,
|
|
1449
|
-
{ strict: false }
|
|
1450
|
-
);
|
|
1451
|
-
});
|
|
1735
|
+
if (failedChunks === 0) {
|
|
1736
|
+
const fullResult = await this.db.withTransactionAsync(async (tx) => {
|
|
1737
|
+
return await this.runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx);
|
|
1738
|
+
});
|
|
1739
|
+
deletedSourceFactIds.push(...fullResult.deletedSourceFactIds);
|
|
1740
|
+
insertedFacts.push(...fullResult.insertedFacts);
|
|
1741
|
+
} else {
|
|
1742
|
+
const partialResult = await this.db.withTransactionAsync(async (tx) => {
|
|
1743
|
+
const flat = [];
|
|
1744
|
+
for (const slot of orderedChunkFacts) flat.push(...slot.facts);
|
|
1745
|
+
return await this.appendPartialFacts(entityId, sourceRef, flat, tx);
|
|
1746
|
+
});
|
|
1747
|
+
insertedFacts.push(...partialResult.insertedDescriptors);
|
|
1748
|
+
}
|
|
1452
1749
|
} catch (err) {
|
|
1453
1750
|
const sqliteCode = err instanceof WikiTransactionError ? err.sqliteErrorCode : extractSqliteCode(err);
|
|
1454
1751
|
if (sqliteCode !== "SQLITE_CONSTRAINT_UNIQUE") throw err;
|
|
@@ -1457,22 +1754,31 @@ var IngestionService = class {
|
|
|
1457
1754
|
if (onDuplicateHash === "throw" || onDuplicateHash === "ingest") {
|
|
1458
1755
|
throw new WikiDuplicateHashError({ canonical, sourceHash, entityId });
|
|
1459
1756
|
}
|
|
1460
|
-
return
|
|
1757
|
+
return zeroChunkResult(canonical);
|
|
1461
1758
|
}
|
|
1462
1759
|
await this.searchService.sync(entityId);
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1760
|
+
if (failedChunks === 0) {
|
|
1761
|
+
const uniqueDeletedSourceFactIds = Array.from(new Set(deletedSourceFactIds));
|
|
1762
|
+
for (const factId of uniqueDeletedSourceFactIds) {
|
|
1763
|
+
try {
|
|
1764
|
+
await this.embeddingService.notifyEmbeddingPersisted(entityId, factId, null);
|
|
1765
|
+
} catch (hookErr) {
|
|
1766
|
+
console.warn(`[WikiMemory] onEmbeddingPersisted hook failed during ingest for ${factId}:`, hookErr);
|
|
1767
|
+
}
|
|
1469
1768
|
}
|
|
1470
1769
|
}
|
|
1471
1770
|
for (const fact of insertedFacts) {
|
|
1472
1771
|
await this.embeddingService.embedFact(fact);
|
|
1473
1772
|
}
|
|
1474
1773
|
this.searchService.evictCache(entityId);
|
|
1475
|
-
|
|
1774
|
+
const result = {
|
|
1775
|
+
truncated,
|
|
1776
|
+
chunks: chunks.length,
|
|
1777
|
+
ingestedChunks,
|
|
1778
|
+
failedChunks
|
|
1779
|
+
};
|
|
1780
|
+
if (failedChunks > 0) result.parseFailures = failures;
|
|
1781
|
+
return result;
|
|
1476
1782
|
} finally {
|
|
1477
1783
|
releaseIngestLocks();
|
|
1478
1784
|
}
|
|
@@ -1597,6 +1903,156 @@ var IngestionService = class {
|
|
|
1597
1903
|
superseded: deletedFactIds.length + deletedEdgeCount
|
|
1598
1904
|
};
|
|
1599
1905
|
}
|
|
1906
|
+
/**
|
|
1907
|
+
* Full supersession + ownership path: identical to the pre-issue-#92 behavior
|
|
1908
|
+
* for the happy path. Called from the `failedChunks === 0` branch of
|
|
1909
|
+
* `ingestDocument`. Runs INSIDE the caller's tx.
|
|
1910
|
+
*
|
|
1911
|
+
* Returns `{ deletedSourceFactIds, insertedFacts }` so the caller can fire
|
|
1912
|
+
* post-commit hooks (embedding lifecycle) AFTER the transaction commits.
|
|
1913
|
+
* On the partial path the caller never invokes this method; the empty
|
|
1914
|
+
* arrays stay empty.
|
|
1915
|
+
*/
|
|
1916
|
+
async runFullUpsertGraph(entityId, sourceRef, sourceHash, orderedChunkFacts, tx) {
|
|
1917
|
+
const deletedSourceFactIds = [];
|
|
1918
|
+
const insertedFacts = [];
|
|
1919
|
+
deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
|
|
1920
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
1921
|
+
const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx, sourceRef);
|
|
1922
|
+
for (const existing of existingFacts) {
|
|
1923
|
+
titleIndex.set(normalizeTitleKey(existing.title), {
|
|
1924
|
+
id: existing.id,
|
|
1925
|
+
okf_type: existing.okf_type ?? null
|
|
1926
|
+
});
|
|
1927
|
+
}
|
|
1928
|
+
let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
1929
|
+
let { mode, manifest } = ontologyState;
|
|
1930
|
+
const hostNodes = [];
|
|
1931
|
+
const rawEdgeRequests = [];
|
|
1932
|
+
const now = Date.now();
|
|
1933
|
+
for (const { facts, ontology_updates } of orderedChunkFacts) {
|
|
1934
|
+
if (mode === "emergent" && ontology_updates && this.ontologyService) {
|
|
1935
|
+
manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
|
|
1936
|
+
ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
|
|
1937
|
+
mode = ontologyState.mode;
|
|
1938
|
+
}
|
|
1939
|
+
for (const fact of facts) {
|
|
1940
|
+
const ontologyFact = fact;
|
|
1941
|
+
const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest, { strict: false }) ?? { okf_type: null, edges: [] };
|
|
1942
|
+
const id = generateId("fact_");
|
|
1943
|
+
hostNodes.push({
|
|
1944
|
+
id,
|
|
1945
|
+
type: ontologyFact.okf_type ?? "",
|
|
1946
|
+
title: fact.title,
|
|
1947
|
+
body: fact.body,
|
|
1948
|
+
// Forward the LLM-extracted tags/confidence through the
|
|
1949
|
+
// extracted-shape upsertGraphCore path so the entries row
|
|
1950
|
+
// stores them (search filterability, heal-candidate
|
|
1951
|
+
// selection, runReembed embedding-text signal). The public
|
|
1952
|
+
// WikiMemory.upsertGraph API doesn't accept tags/confidence
|
|
1953
|
+
// — host-supplied deterministic nodes default to [] and
|
|
1954
|
+
// 'certain' as documented.
|
|
1955
|
+
tags: fact.tags,
|
|
1956
|
+
confidence: fact.confidence
|
|
1957
|
+
});
|
|
1958
|
+
insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
|
|
1959
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
|
|
1960
|
+
if (normalized.edges.length > 0) {
|
|
1961
|
+
rawEdgeRequests.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
1962
|
+
}
|
|
1963
|
+
}
|
|
1964
|
+
}
|
|
1965
|
+
const hostEdges = [];
|
|
1966
|
+
for (const req of rawEdgeRequests) {
|
|
1967
|
+
const resolved = this.ontologyService?.resolveEdges(
|
|
1968
|
+
entityId,
|
|
1969
|
+
req.sourceId,
|
|
1970
|
+
req.sourceType,
|
|
1971
|
+
req.edges,
|
|
1972
|
+
manifest,
|
|
1973
|
+
titleIndex,
|
|
1974
|
+
now
|
|
1975
|
+
) ?? [];
|
|
1976
|
+
for (const e of resolved) {
|
|
1977
|
+
hostEdges.push({ type: e.edge_type, sourceId: e.source_id, targetId: e.target_id });
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
await this.upsertGraphCore(
|
|
1981
|
+
entityId,
|
|
1982
|
+
{ sourceRef, sourceHash, nodes: hostNodes, edges: hostEdges },
|
|
1983
|
+
tx,
|
|
1984
|
+
{ strict: false }
|
|
1985
|
+
);
|
|
1986
|
+
return { deletedSourceFactIds, insertedFacts };
|
|
1987
|
+
}
|
|
1988
|
+
/**
|
|
1989
|
+
* Partial-commit path for `ingestDocument`. Inserts ONLY facts whose
|
|
1990
|
+
* normalized title is not already in the live set for this (entityId,
|
|
1991
|
+
* sourceRef). Does NOT call `entryRepo.softDeleteBySource` (no
|
|
1992
|
+
* supersession) and does NOT call `sourceRefIndexRepo.upsert` (no
|
|
1993
|
+
* ownership). Edges are NOT resolved on this path (no ontology-context
|
|
1994
|
+
* build, no `resolveEdges`, no `mergeEmergentUpdates`). Stale edges from
|
|
1995
|
+
* prior attempts are recovered on the next full run's supersession.
|
|
1996
|
+
*
|
|
1997
|
+
* Source-hash is stored as NULL on partial rows. `findLatestSourceHash`
|
|
1998
|
+
* reads from the most recently updated live row for the sourceRef, so
|
|
1999
|
+
* storing the incoming hash here would cause `hasChanged` to return
|
|
2000
|
+
* `false` on a same-hash retry — the failed chunks would never get a
|
|
2001
|
+
* second chance. With NULL, a retry sees `storedHash === null`, `hasChanged`
|
|
2002
|
+
* returns `true`, and the partial commit's sibling rows remain live
|
|
2003
|
+
* (deduped, not superseded) for the next attempt to extend.
|
|
2004
|
+
*
|
|
2005
|
+
* Runs INSIDE the caller's `tx`. Does not open a nested transaction.
|
|
2006
|
+
* Returns `{ inserted, skippedDuplicate }` for observability.
|
|
2007
|
+
*/
|
|
2008
|
+
async appendPartialFacts(entityId, sourceRef, dedupedFacts, tx) {
|
|
2009
|
+
const liveIds = await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false);
|
|
2010
|
+
const liveFacts = liveIds.length === 0 ? [] : await this.entryRepo.findByIds(liveIds, void 0, tx);
|
|
2011
|
+
const liveTitles = new Set(liveFacts.map((f) => normalizeTitleKey(f.title)));
|
|
2012
|
+
let inserted = 0;
|
|
2013
|
+
let skippedDuplicate = 0;
|
|
2014
|
+
const now = Date.now();
|
|
2015
|
+
const insertedDescriptors = [];
|
|
2016
|
+
for (const fact of dedupedFacts) {
|
|
2017
|
+
const normalizedTitle = normalizeTitleKey(fact.title);
|
|
2018
|
+
if (liveTitles.has(normalizedTitle)) {
|
|
2019
|
+
skippedDuplicate++;
|
|
2020
|
+
continue;
|
|
2021
|
+
}
|
|
2022
|
+
liveTitles.add(normalizedTitle);
|
|
2023
|
+
const id = generateId("fact_");
|
|
2024
|
+
const wikiFact = {
|
|
2025
|
+
id,
|
|
2026
|
+
entity_id: entityId,
|
|
2027
|
+
title: fact.title,
|
|
2028
|
+
body: fact.body,
|
|
2029
|
+
tags: fact.tags,
|
|
2030
|
+
confidence: fact.confidence,
|
|
2031
|
+
source_type: "immutable_document",
|
|
2032
|
+
// source_hash: null on partial rows — see method docstring. The
|
|
2033
|
+
// full path stamps the actual hash; partial rows stay hash-less so
|
|
2034
|
+
// `hasChanged` keeps returning true for retries.
|
|
2035
|
+
source_hash: null,
|
|
2036
|
+
source_ref: sourceRef,
|
|
2037
|
+
created_at: now,
|
|
2038
|
+
updated_at: now,
|
|
2039
|
+
last_accessed_at: null,
|
|
2040
|
+
access_count: 0,
|
|
2041
|
+
deleted_at: null,
|
|
2042
|
+
okf_type: null
|
|
2043
|
+
};
|
|
2044
|
+
await this.entryRepo.upsert(wikiFact, tx);
|
|
2045
|
+
insertedDescriptors.push({
|
|
2046
|
+
id,
|
|
2047
|
+
entity_id: entityId,
|
|
2048
|
+
title: fact.title,
|
|
2049
|
+
body: fact.body,
|
|
2050
|
+
tags: JSON.stringify(fact.tags)
|
|
2051
|
+
});
|
|
2052
|
+
inserted++;
|
|
2053
|
+
}
|
|
2054
|
+
return { inserted, skippedDuplicate, insertedDescriptors };
|
|
2055
|
+
}
|
|
1600
2056
|
};
|
|
1601
2057
|
|
|
1602
2058
|
// src/repositories/BaseRepository.ts
|
|
@@ -1785,7 +2241,12 @@ var TRUNCATION_PATTERNS = [
|
|
|
1785
2241
|
];
|
|
1786
2242
|
var EXCEEDS_LIMIT_PATTERN = /exceed[a-z]*[^.]{0,40}\b(model|context)?[ _-]?limit/i;
|
|
1787
2243
|
function isTruncationError(err) {
|
|
1788
|
-
|
|
2244
|
+
let message;
|
|
2245
|
+
try {
|
|
2246
|
+
message = err instanceof Error ? err.message : String(err ?? "");
|
|
2247
|
+
} catch {
|
|
2248
|
+
return false;
|
|
2249
|
+
}
|
|
1789
2250
|
if (EXCEEDS_LIMIT_PATTERN.test(message)) return false;
|
|
1790
2251
|
return TRUNCATION_PATTERNS.some((pattern) => pattern.test(message));
|
|
1791
2252
|
}
|
|
@@ -1889,6 +2350,30 @@ var HEAL_ANCHOR_SEARCH_OVERFETCH = 4;
|
|
|
1889
2350
|
var HEAL_MAX_PROMPT_CHARS = 4e4;
|
|
1890
2351
|
var HEAL_BATCH_SIZE = 25;
|
|
1891
2352
|
var HEAL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
2353
|
+
var SKIP_ERROR_LOG_CHARS = 4096;
|
|
2354
|
+
var formatSkipError = (err) => {
|
|
2355
|
+
let base;
|
|
2356
|
+
let isErrorLike = false;
|
|
2357
|
+
try {
|
|
2358
|
+
isErrorLike = err instanceof Error;
|
|
2359
|
+
} catch {
|
|
2360
|
+
}
|
|
2361
|
+
if (isErrorLike || typeof err !== "object" && typeof err !== "function") {
|
|
2362
|
+
base = safeErrorToString(err);
|
|
2363
|
+
} else {
|
|
2364
|
+
try {
|
|
2365
|
+
const json = JSON.stringify(err);
|
|
2366
|
+
base = typeof json === "string" ? json : safeErrorToString(err);
|
|
2367
|
+
} catch {
|
|
2368
|
+
base = safeErrorToString(err);
|
|
2369
|
+
}
|
|
2370
|
+
}
|
|
2371
|
+
if (base.length > SKIP_ERROR_LOG_CHARS) {
|
|
2372
|
+
const truncated = safeSlice(base, 0, SKIP_ERROR_LOG_CHARS);
|
|
2373
|
+
return `${truncated}\u2026[+${base.length - SKIP_ERROR_LOG_CHARS} chars truncated]`;
|
|
2374
|
+
}
|
|
2375
|
+
return base;
|
|
2376
|
+
};
|
|
1892
2377
|
var MaintenanceService = class {
|
|
1893
2378
|
constructor(db, prefix, options, entryRepo, sourceRefIndexRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1894
2379
|
this.db = db;
|
|
@@ -2369,8 +2854,7 @@ var MaintenanceService = class {
|
|
|
2369
2854
|
maxPromptChars: HEAL_MAX_PROMPT_CHARS,
|
|
2370
2855
|
onSkip: (fact, err) => {
|
|
2371
2856
|
console.warn(
|
|
2372
|
-
`[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded
|
|
2373
|
-
err
|
|
2857
|
+
`[WikiMemory] heal skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
|
|
2374
2858
|
);
|
|
2375
2859
|
}
|
|
2376
2860
|
});
|
|
@@ -2508,8 +2992,7 @@ var MaintenanceService = class {
|
|
|
2508
2992
|
maxPromptChars: ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS,
|
|
2509
2993
|
onSkip: (fact, err) => {
|
|
2510
2994
|
console.warn(
|
|
2511
|
-
`[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded
|
|
2512
|
-
err
|
|
2995
|
+
`[WikiMemory] ontology backfill skipped ${entityId}/${fact.id}: response could not be bounded: ${formatSkipError(err)}`
|
|
2513
2996
|
);
|
|
2514
2997
|
}
|
|
2515
2998
|
});
|
|
@@ -3567,7 +4050,12 @@ var RetrievalService = class {
|
|
|
3567
4050
|
}
|
|
3568
4051
|
}
|
|
3569
4052
|
} catch (rankerErr) {
|
|
3570
|
-
|
|
4053
|
+
let isErrorLike = false;
|
|
4054
|
+
try {
|
|
4055
|
+
isErrorLike = rankerErr instanceof Error;
|
|
4056
|
+
} catch {
|
|
4057
|
+
}
|
|
4058
|
+
const rankerError = isErrorLike ? rankerErr : new Error(safeErrorToString(rankerErr));
|
|
3571
4059
|
const policy = this.options.vectorRankerFallback ?? "js-cosine";
|
|
3572
4060
|
this.options.onVectorRankerFallback?.({
|
|
3573
4061
|
error: this._sanitizeRankerError(rankerError),
|
|
@@ -3679,12 +4167,21 @@ var RetrievalService = class {
|
|
|
3679
4167
|
}
|
|
3680
4168
|
}
|
|
3681
4169
|
} catch (err) {
|
|
3682
|
-
|
|
4170
|
+
let isErrorLike = false;
|
|
4171
|
+
try {
|
|
4172
|
+
isErrorLike = err instanceof Error;
|
|
4173
|
+
} catch {
|
|
4174
|
+
}
|
|
4175
|
+
const error = isErrorLike ? err : new Error(safeErrorToString(err));
|
|
3683
4176
|
if (rankerShouldRethrow) {
|
|
3684
4177
|
throw error;
|
|
3685
4178
|
}
|
|
3686
4179
|
if (pendingRankerFallbackError) {
|
|
3687
|
-
|
|
4180
|
+
try {
|
|
4181
|
+
error.cause = pendingRankerFallbackError;
|
|
4182
|
+
} catch {
|
|
4183
|
+
this.options.onRetrievalFallback?.(pendingRankerFallbackError);
|
|
4184
|
+
}
|
|
3688
4185
|
pendingRankerFallbackError = void 0;
|
|
3689
4186
|
}
|
|
3690
4187
|
this.options.onRetrievalFallback?.(error);
|
|
@@ -3940,6 +4437,6 @@ var WriteService = class {
|
|
|
3940
4437
|
}
|
|
3941
4438
|
};
|
|
3942
4439
|
|
|
3943
|
-
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, validateInlineEdges, validateManifest };
|
|
3944
|
-
//# sourceMappingURL=chunk-
|
|
3945
|
-
//# sourceMappingURL=chunk-
|
|
4440
|
+
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, EmbeddingService, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiDuplicateHashError, WikiIngestEmptyError, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, safeSlice, validateInlineEdges, validateManifest };
|
|
4441
|
+
//# sourceMappingURL=chunk-JQRA6BYQ.mjs.map
|
|
4442
|
+
//# sourceMappingURL=chunk-JQRA6BYQ.mjs.map
|