@equationalapplications/core-llm-wiki 7.11.1 → 7.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/{chunk-IJFOAWPS.mjs → chunk-KN4XSLLB.mjs} +2103 -1414
- package/dist/chunk-KN4XSLLB.mjs.map +1 -0
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2930 -2631
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +156 -549
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-C-WsBxLh.d.mts → testing-7d76Ptlq.d.mts} +90 -9
- package/dist/{testing-C-WsBxLh.d.ts → testing-7d76Ptlq.d.ts} +90 -9
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +431 -25
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-IJFOAWPS.mjs.map +0 -1
|
@@ -184,1519 +184,1569 @@ var WikiSupersedeError = class _WikiSupersedeError extends Error {
|
|
|
184
184
|
Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
|
|
185
185
|
}
|
|
186
186
|
};
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
187
|
+
var WikiMigrationRequiredError = class _WikiMigrationRequiredError extends Error {
|
|
188
|
+
constructor(message, pendingWork) {
|
|
189
|
+
super(message);
|
|
190
|
+
this.code = "WIKI_MIGRATION_REQUIRED";
|
|
191
|
+
this.name = "WikiMigrationRequiredError";
|
|
192
|
+
this.pendingWork = pendingWork;
|
|
193
|
+
if (typeof Error.captureStackTrace === "function") {
|
|
194
|
+
Error.captureStackTrace(this, _WikiMigrationRequiredError);
|
|
195
|
+
}
|
|
196
196
|
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
197
|
+
};
|
|
198
|
+
var WikiSearchUnavailableError = class extends Error {
|
|
199
|
+
constructor(message = "Keyword search is temporarily unavailable", cause) {
|
|
200
|
+
super(message, cause !== void 0 ? { cause } : void 0);
|
|
201
|
+
this.name = "WikiSearchUnavailableError";
|
|
201
202
|
}
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
return
|
|
203
|
+
};
|
|
204
|
+
function isWikiSearchUnavailableError(err) {
|
|
205
|
+
try {
|
|
206
|
+
return err instanceof WikiSearchUnavailableError;
|
|
207
|
+
} catch {
|
|
208
|
+
return false;
|
|
206
209
|
}
|
|
207
|
-
throw new Error(
|
|
208
|
-
"generateId: no cryptographically secure random source available (crypto.randomUUID and crypto.getRandomValues are both missing, and no configureRandomSource() injection was provided)."
|
|
209
|
-
);
|
|
210
210
|
}
|
|
211
211
|
|
|
212
|
-
// src/utils/
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
}
|
|
219
|
-
function resolveNodeType(raw, manifest) {
|
|
220
|
-
const slug = raw.trim();
|
|
221
|
-
if (!slug) return null;
|
|
222
|
-
const hit = manifest.node_types.find((n) => n.type.toLowerCase() === slug.toLowerCase());
|
|
223
|
-
return hit?.type ?? null;
|
|
224
|
-
}
|
|
225
|
-
function resolveEdgeDefinitions(rawEdgeType, manifest) {
|
|
226
|
-
const slug = rawEdgeType.trim();
|
|
227
|
-
if (!slug) return [];
|
|
228
|
-
return manifest.edge_types.filter((e) => e.type.toLowerCase() === slug.toLowerCase());
|
|
229
|
-
}
|
|
230
|
-
function edgeTripleKey(type, sourceType, targetType) {
|
|
231
|
-
return `${type.trim().toLowerCase()}|${sourceType.trim().toLowerCase()}|${targetType.trim().toLowerCase()}`;
|
|
232
|
-
}
|
|
233
|
-
function typeSatisfies(declaredType, concreteType, manifest) {
|
|
234
|
-
const concrete = concreteType.trim().toLowerCase();
|
|
235
|
-
const declared = declaredType.trim().toLowerCase();
|
|
236
|
-
if (!concrete || !declared) return false;
|
|
237
|
-
if (declared === concrete) return true;
|
|
238
|
-
const def = (manifest.node_types ?? []).find(
|
|
239
|
-
(n) => typeof n?.type === "string" && n.type.trim().toLowerCase() === concrete
|
|
212
|
+
// src/utils/pure.ts
|
|
213
|
+
var MAX_REPAIR_CANDIDATES = 5;
|
|
214
|
+
function throwNoJsonFound(text, start) {
|
|
215
|
+
throw new WikiParseError(
|
|
216
|
+
"No JSON object/array found in LLM response",
|
|
217
|
+
{ tier: "strict", position: start, slice: text }
|
|
240
218
|
);
|
|
241
|
-
const parent = typeof def?.parent_type === "string" ? def.parent_type.trim().toLowerCase() : "";
|
|
242
|
-
return parent !== "" && parent === declared;
|
|
243
|
-
}
|
|
244
|
-
function edgeTripleAllowed(manifest, edgeType, sourceType, targetType) {
|
|
245
|
-
const wanted = edgeType.trim().toLowerCase();
|
|
246
|
-
return (manifest.edge_types ?? []).some((d) => typeof d?.type === "string" && d.type.trim().toLowerCase() === wanted && typeof d.source_type === "string" && typeSatisfies(d.source_type, sourceType, manifest) && typeof d.target_type === "string" && typeSatisfies(d.target_type, targetType, manifest));
|
|
247
219
|
}
|
|
248
|
-
function
|
|
249
|
-
const
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
220
|
+
function parseJsonResponse(text) {
|
|
221
|
+
const firstBrace = text.indexOf("{");
|
|
222
|
+
const firstBracket = text.indexOf("[");
|
|
223
|
+
let start;
|
|
224
|
+
let openChar;
|
|
225
|
+
const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
|
|
226
|
+
if (useBrace) {
|
|
227
|
+
start = firstBrace;
|
|
228
|
+
openChar = "{";
|
|
229
|
+
} else if (firstBracket !== -1) {
|
|
230
|
+
start = firstBracket;
|
|
231
|
+
openChar = "[";
|
|
232
|
+
} else {
|
|
233
|
+
throwNoJsonFound(text, null);
|
|
261
234
|
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
}
|
|
267
|
-
const parentSlug = node.parent_type.trim().toLowerCase();
|
|
268
|
-
if (parentSlug === node.type.trim().toLowerCase()) {
|
|
269
|
-
throw new Error(`Self-parent: ${node.type}`);
|
|
270
|
-
}
|
|
271
|
-
if (!nodeSlugs.has(parentSlug)) {
|
|
272
|
-
throw new Error(`Parent type not found: ${node.parent_type}`);
|
|
273
|
-
}
|
|
274
|
-
const grandparent = parentOf.get(parentSlug);
|
|
275
|
-
if (grandparent) {
|
|
276
|
-
throw new Error(`Parent chain too deep: ${node.type} \u2192 ${node.parent_type} \u2192 ${grandparent}`);
|
|
235
|
+
const slice = scanJsonSlice(text, start, openChar);
|
|
236
|
+
if (slice !== null) {
|
|
237
|
+
try {
|
|
238
|
+
return JSON.parse(slice);
|
|
239
|
+
} catch {
|
|
277
240
|
}
|
|
278
241
|
}
|
|
279
|
-
const
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
const edgeType = edge.type?.trim();
|
|
283
|
-
const sourceType = edge.source_type?.trim();
|
|
284
|
-
const targetType = edge.target_type?.trim();
|
|
285
|
-
if (!edgeType) throw new Error("Ontology edge type slug must be non-empty");
|
|
286
|
-
if (!sourceType || !targetType || !nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) {
|
|
287
|
-
throw new Error(`Edge type ${edgeType} references unknown node type`);
|
|
288
|
-
}
|
|
289
|
-
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
290
|
-
if (edgeKeys.has(edgeKey)) {
|
|
291
|
-
throw new Error(`Duplicate edge definition: ${edgeType} (${sourceType} \u2192 ${targetType})`);
|
|
292
|
-
}
|
|
293
|
-
edgeKeys.add(edgeKey);
|
|
294
|
-
const canonical = edgeNames.get(edgeType.toLowerCase());
|
|
295
|
-
if (canonical === void 0) {
|
|
296
|
-
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
297
|
-
} else if (canonical !== edgeType) {
|
|
298
|
-
throw new Error(`Inconsistent casing for edge type: ${edgeType} conflicts with ${canonical}`);
|
|
299
|
-
}
|
|
242
|
+
const repairOdd = containerAwareRepair(text, start, false);
|
|
243
|
+
if (repairOdd.success !== null && !repairOdd.ambiguous) {
|
|
244
|
+
return repairOdd.success;
|
|
300
245
|
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
const slug = typeof n?.type === "string" ? n.type.trim().toLowerCase() : "";
|
|
306
|
-
if (!slug || declaresParent.has(slug)) continue;
|
|
307
|
-
declaresParent.set(slug, n?.parent_type !== void 0);
|
|
246
|
+
const repairEven = containerAwareRepair(text, start, true);
|
|
247
|
+
if (repairOdd.success !== null && repairEven.success !== null) {
|
|
248
|
+
const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
|
|
249
|
+
return pick.success;
|
|
308
250
|
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
function mergeOntologyUpdates(current, updates) {
|
|
312
|
-
const node_types = [...current.node_types];
|
|
313
|
-
const edge_types = [...current.edge_types];
|
|
314
|
-
const nodeSlugs = new Set(node_types.map((n) => n.type.trim().toLowerCase()));
|
|
315
|
-
const edgeKeys = new Set(edge_types.map((e) => edgeTripleKey(e.type, e.source_type, e.target_type)));
|
|
316
|
-
const edgeNames = new Map(edge_types.map((e) => [e.type.trim().toLowerCase(), e.type.trim()]));
|
|
317
|
-
const declaresParent = buildDeclaresParentIndex([
|
|
318
|
-
...current.node_types,
|
|
319
|
-
...updates.node_types ?? []
|
|
320
|
-
]);
|
|
321
|
-
for (const node of updates.node_types ?? []) {
|
|
322
|
-
const type = node?.type?.trim();
|
|
323
|
-
if (!type) continue;
|
|
324
|
-
const key = type.toLowerCase();
|
|
325
|
-
if (nodeSlugs.has(key)) continue;
|
|
326
|
-
const rawParent = typeof node?.parent_type === "string" ? node.parent_type.trim() : "";
|
|
327
|
-
const parentSlug = rawParent.toLowerCase();
|
|
328
|
-
const keepParent = parentSlug !== "" && parentSlug !== key && declaresParent.get(parentSlug) === false;
|
|
329
|
-
node_types.push({
|
|
330
|
-
type,
|
|
331
|
-
description: String(node.description ?? ""),
|
|
332
|
-
...keepParent ? { parent_type: rawParent } : {}
|
|
333
|
-
});
|
|
334
|
-
nodeSlugs.add(key);
|
|
251
|
+
if (repairOdd.success !== null) {
|
|
252
|
+
return repairOdd.success;
|
|
335
253
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
const sourceType = edge?.source_type?.trim();
|
|
339
|
-
const targetType = edge?.target_type?.trim();
|
|
340
|
-
if (!rawEdgeType || !sourceType || !targetType) continue;
|
|
341
|
-
const edgeType = edgeNames.get(rawEdgeType.toLowerCase()) ?? rawEdgeType;
|
|
342
|
-
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
343
|
-
if (edgeKeys.has(edgeKey)) continue;
|
|
344
|
-
if (!nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) continue;
|
|
345
|
-
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
346
|
-
edge_types.push({
|
|
347
|
-
type: edgeType,
|
|
348
|
-
source_type: sourceType,
|
|
349
|
-
target_type: targetType,
|
|
350
|
-
description: String(edge.description ?? "")
|
|
351
|
-
});
|
|
352
|
-
edgeKeys.add(edgeKey);
|
|
254
|
+
if (repairEven.success !== null) {
|
|
255
|
+
return repairEven.success;
|
|
353
256
|
}
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
const drop = (reason, edgeType) => {
|
|
361
|
-
drops?.push({ reason, sourceId: null, edgeType, sourceNodeType: sourceType, targetNodeType: null });
|
|
362
|
-
};
|
|
363
|
-
if (!Array.isArray(edges)) {
|
|
364
|
-
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", "");
|
|
365
|
-
drop("invalid_shape", null);
|
|
366
|
-
return [];
|
|
257
|
+
const repair = repairEven.failed !== null ? repairEven : repairOdd;
|
|
258
|
+
if (repair.failed !== null) {
|
|
259
|
+
throw new WikiParseError(
|
|
260
|
+
`Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
|
|
261
|
+
{ tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
|
|
262
|
+
);
|
|
367
263
|
}
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
264
|
+
if (slice === null) {
|
|
265
|
+
throwNoJsonFound(text, start);
|
|
266
|
+
}
|
|
267
|
+
throw new WikiParseError(
|
|
268
|
+
"No parsable JSON candidate found",
|
|
269
|
+
{ tier: "all", position: start, slice: text }
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
function scanJsonSlice(text, start, openChar) {
|
|
273
|
+
const closeChar = openChar === "{" ? "}" : "]";
|
|
274
|
+
let depth = 0;
|
|
275
|
+
let inString = false;
|
|
276
|
+
let escape = false;
|
|
277
|
+
let end = -1;
|
|
278
|
+
for (let i = start; i < text.length; i++) {
|
|
279
|
+
const ch = text[i];
|
|
280
|
+
if (escape) {
|
|
281
|
+
escape = false;
|
|
373
282
|
continue;
|
|
374
283
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
if (!match) {
|
|
378
|
-
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", edge.edge_type);
|
|
379
|
-
drop("type_not_in_manifest", edge.edge_type);
|
|
284
|
+
if (ch === "\\" && inString) {
|
|
285
|
+
escape = true;
|
|
380
286
|
continue;
|
|
381
287
|
}
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
// src/utils/embedding.ts
|
|
388
|
-
function parseEmbedding(blob, text) {
|
|
389
|
-
if (blob && blob.byteLength > 0) {
|
|
390
|
-
if (blob.byteLength % 4 !== 0) return null;
|
|
391
|
-
const copy = new ArrayBuffer(blob.byteLength);
|
|
392
|
-
new Uint8Array(copy).set(blob);
|
|
393
|
-
const vector = new Float32Array(copy);
|
|
394
|
-
for (const value of vector) {
|
|
395
|
-
if (!Number.isFinite(value)) return null;
|
|
288
|
+
if (ch === '"') {
|
|
289
|
+
inString = !inString;
|
|
290
|
+
continue;
|
|
396
291
|
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
292
|
+
if (inString) continue;
|
|
293
|
+
if (ch === openChar) {
|
|
294
|
+
depth++;
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
if (ch === closeChar) {
|
|
298
|
+
depth--;
|
|
299
|
+
if (depth === 0) {
|
|
300
|
+
end = i;
|
|
301
|
+
break;
|
|
406
302
|
}
|
|
407
|
-
return vector;
|
|
408
|
-
} catch {
|
|
409
|
-
return null;
|
|
410
303
|
}
|
|
411
304
|
}
|
|
412
|
-
return null;
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
// src/utils/cosine.ts
|
|
416
|
-
function cosineSimilarity(a, b) {
|
|
417
|
-
let dot = 0, normA = 0, normB = 0;
|
|
418
|
-
const len = Math.min(a.length, b.length);
|
|
419
|
-
for (let i = 0; i < len; i++) {
|
|
420
|
-
dot += a[i] * b[i];
|
|
421
|
-
normA += a[i] * a[i];
|
|
422
|
-
normB += b[i] * b[i];
|
|
423
|
-
}
|
|
424
|
-
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
425
|
-
return denom === 0 ? 0 : dot / denom;
|
|
305
|
+
if (end === -1) return null;
|
|
306
|
+
return safeSlice(text, start, end + 1);
|
|
426
307
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
308
|
+
function containerAwareRepair(text, start, closeOnEvenParity) {
|
|
309
|
+
const stack = [];
|
|
310
|
+
let inString = false;
|
|
311
|
+
let stringRole = null;
|
|
312
|
+
let escape = false;
|
|
313
|
+
let i = start;
|
|
314
|
+
let out = "";
|
|
315
|
+
let bareQuoteCount = 0;
|
|
316
|
+
let bestFailed = null;
|
|
317
|
+
let escapes = 0;
|
|
318
|
+
let ambiguous = false;
|
|
319
|
+
let bestSuccess = null;
|
|
320
|
+
let attempts = 0;
|
|
321
|
+
function tryEmitCandidate(candidate) {
|
|
322
|
+
attempts++;
|
|
323
|
+
try {
|
|
324
|
+
bestSuccess = JSON.parse(candidate);
|
|
325
|
+
return true;
|
|
326
|
+
} catch (err) {
|
|
327
|
+
if (bestFailed === null) {
|
|
328
|
+
bestFailed = {
|
|
329
|
+
candidate,
|
|
330
|
+
position: extractParsePosition(err),
|
|
331
|
+
message: safeErrorToString(err)
|
|
332
|
+
};
|
|
441
333
|
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
// src/services/SearchService.ts
|
|
447
|
-
var _SearchService = class _SearchService {
|
|
448
|
-
constructor(entryRepo, indexStrategy) {
|
|
449
|
-
this.entryRepo = entryRepo;
|
|
450
|
-
this.vectorCache = /* @__PURE__ */ new Map();
|
|
451
|
-
/**
|
|
452
|
-
* Serializes rebuilds. `rebuildIndex` awaits a repository read between
|
|
453
|
-
* snapshotting the previous id set and discarding it, so two concurrent
|
|
454
|
-
* sync() calls for one entity can interleave: a slow, stale read lands last
|
|
455
|
-
* and discards documents the fresh read just added. Chaining also keeps
|
|
456
|
-
* discard()/addAll() out of each other's way, which is what accrued the
|
|
457
|
-
* auto-vacuum debt behind the TypeError in #64.
|
|
458
|
-
*/
|
|
459
|
-
this.syncChain = Promise.resolve();
|
|
460
|
-
/**
|
|
461
|
-
* Entities whose index may have drifted from SQLite: an incremental update
|
|
462
|
-
* failed part-way, or core wrote rows it could not index (upsertGraph runs in
|
|
463
|
-
* the host's transaction). Their next syncEntries rebuilds the entity in full.
|
|
464
|
-
* See spec 2026-09-28 §5.
|
|
465
|
-
*/
|
|
466
|
-
this.staleEntities = /* @__PURE__ */ new Set();
|
|
467
|
-
/**
|
|
468
|
-
* Per-entity count of markStale() calls. A rebuild clears an entity's stale
|
|
469
|
-
* flag only when the count it snapshotted before its read still holds
|
|
470
|
-
* afterwards: markStale() runs inside the host's still-open transaction, so
|
|
471
|
-
* a call landing mid-read belongs to rows the read cannot have seen, and
|
|
472
|
-
* its flag must outlive the turn (#233 review).
|
|
473
|
-
*/
|
|
474
|
-
this.staleEpochs = /* @__PURE__ */ new Map();
|
|
475
|
-
this.indexStrategy = indexStrategy;
|
|
476
|
-
}
|
|
477
|
-
/**
|
|
478
|
-
* Install the strategy resolved by `WikiMemory.setup()`. Before setup, the
|
|
479
|
-
* service runs on an empty MiniSearch placeholder, as it always has.
|
|
480
|
-
*/
|
|
481
|
-
setIndexStrategy(strategy) {
|
|
482
|
-
this.indexStrategy = strategy;
|
|
334
|
+
return false;
|
|
335
|
+
}
|
|
483
336
|
}
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
const work = this.syncChain.then(async () => {
|
|
491
|
-
try {
|
|
492
|
-
try {
|
|
493
|
-
await this.indexStrategy.drain();
|
|
494
|
-
} finally {
|
|
495
|
-
this.evictCache(entityId);
|
|
496
|
-
}
|
|
497
|
-
} catch (err) {
|
|
498
|
-
console.warn(`[WikiMemory] search index drain failed for ${entityId ?? "*"}:`, err);
|
|
337
|
+
while (i < text.length) {
|
|
338
|
+
const ch = text[i];
|
|
339
|
+
if (stack.length === 0 && !inString) {
|
|
340
|
+
if (ch === "{" || ch === "[") {
|
|
341
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
342
|
+
out += ch;
|
|
499
343
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
for (const id of [...this.staleEntities]) {
|
|
522
|
-
if ((this.staleEpochs.get(id) ?? 0) === (epochsBefore.get(id) ?? 0)) {
|
|
523
|
-
this.staleEntities.delete(id);
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
} else {
|
|
527
|
-
await this.rebuildIndex(entityId);
|
|
528
|
-
if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
|
|
529
|
-
this.staleEntities.delete(entityId);
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
await this.indexStrategy.vacuum?.();
|
|
533
|
-
} finally {
|
|
534
|
-
this.evictCache(entityId);
|
|
535
|
-
}
|
|
536
|
-
} catch (err) {
|
|
537
|
-
console.warn(`[WikiMemory] search index rebuild failed for ${entityId ?? "*"}:`, err);
|
|
344
|
+
i++;
|
|
345
|
+
continue;
|
|
346
|
+
}
|
|
347
|
+
if (escape) {
|
|
348
|
+
out += ch;
|
|
349
|
+
escape = false;
|
|
350
|
+
i++;
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
if (inString && ch === "\\") {
|
|
354
|
+
out += ch;
|
|
355
|
+
escape = true;
|
|
356
|
+
i++;
|
|
357
|
+
continue;
|
|
358
|
+
}
|
|
359
|
+
if (inString) {
|
|
360
|
+
const code = ch.charCodeAt(0);
|
|
361
|
+
if (code < 32) {
|
|
362
|
+
out += controlCharEscape(code);
|
|
363
|
+
i++;
|
|
364
|
+
continue;
|
|
538
365
|
}
|
|
539
|
-
});
|
|
540
|
-
this.syncChain = work;
|
|
541
|
-
return work;
|
|
542
|
-
}
|
|
543
|
-
/**
|
|
544
|
-
* Re-indexes only `ids` for `entityId`: drops each from the index, then
|
|
545
|
-
* re-adds the ones still live in SQLite, so soft-deleted or missing ids end
|
|
546
|
-
* up absent. Costs O(ids), where sync(entityId) costs O(entity) — callers that
|
|
547
|
-
* know what a write touched use this so chunked imports stay linear (#232).
|
|
548
|
-
*
|
|
549
|
-
* Serialized with sync() on the same chain and, like it, never rejects. An
|
|
550
|
-
* entity that is stale or has never been indexed gets a full rebuild instead
|
|
551
|
-
* — including on an empty `ids` list, which is a no-op only for an entity
|
|
552
|
-
* the index already tracks, and even then one that waits for rebuilds
|
|
553
|
-
* already queued on the chain.
|
|
554
|
-
*/
|
|
555
|
-
async syncEntries(entityId, ids) {
|
|
556
|
-
if (this.indexStrategy.drain) return this.drainTurn(entityId);
|
|
557
|
-
const uniqueIds = [...new Set(ids)];
|
|
558
|
-
const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
|
|
559
|
-
if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
|
|
560
|
-
return this.syncChain;
|
|
561
366
|
}
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
const needsRebuild = () => this.staleEntities.has(entityId) || this.indexStrategy.hasIndexedEntity?.(entityId) !== true;
|
|
567
|
-
if (!needsRebuild()) {
|
|
568
|
-
const rows = await this.entryRepo.findMiniSearchRowsByIds(entityId, uniqueIds);
|
|
569
|
-
if (!this.staleEntities.has(entityId)) {
|
|
570
|
-
await this.indexStrategy.replace(entityId, uniqueIds, rows.map(toIndexDoc));
|
|
571
|
-
await this.indexStrategy.vacuum?.();
|
|
572
|
-
return;
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
await this.rebuildIndex(entityId);
|
|
576
|
-
if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
|
|
577
|
-
this.staleEntities.delete(entityId);
|
|
578
|
-
}
|
|
579
|
-
await this.indexStrategy.vacuum?.();
|
|
580
|
-
} finally {
|
|
581
|
-
this.evictCache(entityId);
|
|
582
|
-
}
|
|
583
|
-
} catch (err) {
|
|
584
|
-
this.staleEntities.add(entityId);
|
|
585
|
-
console.warn(`[WikiMemory] search index incremental sync failed for ${entityId}:`, err);
|
|
367
|
+
if (ch === '"') {
|
|
368
|
+
let j = i + 1;
|
|
369
|
+
while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
|
|
370
|
+
j++;
|
|
586
371
|
}
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
/**
|
|
601
|
-
* Runs syncEntries(id, []) for each entity marked stale when this is called,
|
|
602
|
-
* so only those entities are rebuilt. Each rebuild still takes its own turn
|
|
603
|
-
* on the sync chain, and like syncEntries this never rejects. With nothing
|
|
604
|
-
* stale it resolves at once, without waiting on work already on the chain.
|
|
605
|
-
*/
|
|
606
|
-
async syncStale() {
|
|
607
|
-
if (this.indexStrategy.drain) return this.drainTurn();
|
|
608
|
-
const ids = [...this.staleEntities];
|
|
609
|
-
await Promise.all(ids.map((id) => this.syncEntries(id, [])));
|
|
610
|
-
}
|
|
611
|
-
/**
|
|
612
|
-
* Clears the parsed vector cache. Useful for mid-loop flush guarantees
|
|
613
|
-
* or memory pressure evictions.
|
|
614
|
-
*/
|
|
615
|
-
evictCache(entityId) {
|
|
616
|
-
if (entityId) {
|
|
617
|
-
this.vectorCache.delete(entityId);
|
|
618
|
-
} else {
|
|
619
|
-
this.vectorCache.clear();
|
|
620
|
-
}
|
|
621
|
-
}
|
|
622
|
-
/**
|
|
623
|
-
* Fully resets the search service.
|
|
624
|
-
*/
|
|
625
|
-
async clearAll() {
|
|
626
|
-
this.vectorCache.clear();
|
|
627
|
-
await this.indexStrategy.replaceAll([]);
|
|
628
|
-
this.staleEntities.clear();
|
|
629
|
-
this.staleEpochs.clear();
|
|
630
|
-
}
|
|
631
|
-
/**
|
|
632
|
-
* Executes a keyword search against the active index strategy.
|
|
633
|
-
*/
|
|
634
|
-
async searchKeyword(query, entityIds, limit) {
|
|
635
|
-
const results = finiteResults(await this.indexStrategy.search(query, { entityIds, limit }));
|
|
636
|
-
return results.sort((a, b) => this._compareSearchResults(a, b)).slice(0, limit);
|
|
637
|
-
}
|
|
638
|
-
/**
|
|
639
|
-
* Pre-fetches keyword scores for candidate hydration, used during hybrid weighting.
|
|
640
|
-
*/
|
|
641
|
-
async getKeywordScores(query, entityIds, preFilterLimit) {
|
|
642
|
-
const opts = preFilterLimit !== void 0 ? { entityIds, limit: preFilterLimit, preFilterLimit } : { entityIds, limit: Number.MAX_SAFE_INTEGER };
|
|
643
|
-
const results = finiteResults(await this.indexStrategy.search(query, opts));
|
|
644
|
-
if (results.length === 0) return /* @__PURE__ */ new Map();
|
|
645
|
-
const maxScore = Math.max(1, results[0]?.score ?? 1);
|
|
646
|
-
return new Map(results.map((r) => [r.id, r.score / maxScore]));
|
|
647
|
-
}
|
|
648
|
-
/**
|
|
649
|
-
* Score candidate rows using in-process JS cosine similarity.
|
|
650
|
-
* Applies hybrid blending (if weight set) and tie-break sorting before returning.
|
|
651
|
-
*/
|
|
652
|
-
async rankSemantic(args) {
|
|
653
|
-
const queryVec = args.queryVec instanceof Float32Array ? args.queryVec.slice() : Array.from(args.queryVec);
|
|
654
|
-
const { entityId, candidateRows, weight, miniSearchScores, populateCache, limit, skipSort } = args;
|
|
655
|
-
let entityCache = this.vectorCache.get(entityId);
|
|
656
|
-
const tooLarge = populateCache && candidateRows.length > _SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY;
|
|
657
|
-
if (tooLarge && entityCache) {
|
|
658
|
-
this.vectorCache.delete(entityId);
|
|
659
|
-
entityCache = void 0;
|
|
660
|
-
}
|
|
661
|
-
const canCache = populateCache && !tooLarge;
|
|
662
|
-
if (canCache && !entityCache) {
|
|
663
|
-
entityCache = /* @__PURE__ */ new Map();
|
|
664
|
-
}
|
|
665
|
-
const scored = candidateRows.map((row) => {
|
|
666
|
-
let vector = entityCache?.get(row.id) ?? parseEmbedding(row.embedding_blob, row.embedding);
|
|
667
|
-
if (vector && canCache && entityCache && !entityCache.has(row.id)) {
|
|
668
|
-
entityCache.set(row.id, vector);
|
|
669
|
-
}
|
|
670
|
-
let score = 0;
|
|
671
|
-
if (vector && vector.length === queryVec.length) {
|
|
672
|
-
const cosSim = cosineSimilarity(queryVec, vector);
|
|
673
|
-
if (weight !== void 0) {
|
|
674
|
-
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
675
|
-
score = weight * Math.max(0, cosSim) + (1 - weight) * kwScore;
|
|
676
|
-
} else {
|
|
677
|
-
score = cosSim;
|
|
372
|
+
const next = j < text.length ? text[j] : "";
|
|
373
|
+
if (!inString) {
|
|
374
|
+
if (next === "}" || next === "]") {
|
|
375
|
+
if (j > i + 1) out += text.slice(i + 1, j);
|
|
376
|
+
out += next;
|
|
377
|
+
i = j + 1;
|
|
378
|
+
stack.pop();
|
|
379
|
+
if (stack.length === 0) {
|
|
380
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
381
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
382
|
+
out = "";
|
|
383
|
+
}
|
|
384
|
+
continue;
|
|
678
385
|
}
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
386
|
+
const top = stack[stack.length - 1];
|
|
387
|
+
stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
|
|
388
|
+
out += ch;
|
|
389
|
+
inString = true;
|
|
390
|
+
i++;
|
|
391
|
+
continue;
|
|
684
392
|
}
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
if (
|
|
696
|
-
|
|
697
|
-
if (oldestKey !== void 0) this.vectorCache.delete(oldestKey);
|
|
393
|
+
const commaOnly = next === ",";
|
|
394
|
+
const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
|
|
395
|
+
const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
|
|
396
|
+
if (commaOnly && bareQuoteCount > 0) ambiguous = true;
|
|
397
|
+
const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
|
|
398
|
+
const isClosing = isKeyClose || isValueClose || shouldCommaClose;
|
|
399
|
+
if (isClosing) {
|
|
400
|
+
out += ch;
|
|
401
|
+
inString = false;
|
|
402
|
+
const topFrame = stack[stack.length - 1];
|
|
403
|
+
if (topFrame && topFrame.container === "object") {
|
|
404
|
+
topFrame.expectKey = stringRole === "value";
|
|
698
405
|
}
|
|
699
|
-
|
|
406
|
+
stringRole = null;
|
|
407
|
+
bareQuoteCount = 0;
|
|
408
|
+
i++;
|
|
409
|
+
continue;
|
|
700
410
|
}
|
|
411
|
+
out += "\\" + ch;
|
|
412
|
+
bareQuoteCount++;
|
|
413
|
+
escapes++;
|
|
414
|
+
i++;
|
|
415
|
+
continue;
|
|
701
416
|
}
|
|
702
|
-
if (!
|
|
703
|
-
|
|
417
|
+
if (!inString && (ch === "{" || ch === "[")) {
|
|
418
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
419
|
+
out += ch;
|
|
420
|
+
i++;
|
|
421
|
+
continue;
|
|
704
422
|
}
|
|
705
|
-
|
|
423
|
+
if (!inString && ch === ",") {
|
|
424
|
+
const topFrame = stack[stack.length - 1];
|
|
425
|
+
if (topFrame && topFrame.container === "object") {
|
|
426
|
+
topFrame.expectKey = true;
|
|
427
|
+
}
|
|
428
|
+
out += ch;
|
|
429
|
+
i++;
|
|
430
|
+
continue;
|
|
431
|
+
}
|
|
432
|
+
if (!inString && (ch === "}" || ch === "]")) {
|
|
433
|
+
out += ch;
|
|
434
|
+
stack.pop();
|
|
435
|
+
if (stack.length === 0) {
|
|
436
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
437
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
438
|
+
out = "";
|
|
439
|
+
}
|
|
440
|
+
i++;
|
|
441
|
+
continue;
|
|
442
|
+
}
|
|
443
|
+
out += ch;
|
|
444
|
+
i++;
|
|
706
445
|
}
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
446
|
+
return { success: null, failed: bestFailed, escapes, ambiguous };
|
|
447
|
+
}
|
|
448
|
+
function controlCharEscape(code) {
|
|
449
|
+
switch (code) {
|
|
450
|
+
case 10:
|
|
451
|
+
return "\\n";
|
|
452
|
+
case 9:
|
|
453
|
+
return "\\t";
|
|
454
|
+
case 13:
|
|
455
|
+
return "\\r";
|
|
456
|
+
case 8:
|
|
457
|
+
return "\\b";
|
|
458
|
+
case 12:
|
|
459
|
+
return "\\f";
|
|
460
|
+
default:
|
|
461
|
+
return "\\u" + code.toString(16).padStart(4, "0");
|
|
718
462
|
}
|
|
719
|
-
|
|
720
|
-
|
|
463
|
+
}
|
|
464
|
+
function extractParsePosition(err) {
|
|
465
|
+
const match = /position\s+(\d+)/.exec(safeErrorToString(err));
|
|
466
|
+
return match ? Number(match[1]) : null;
|
|
467
|
+
}
|
|
468
|
+
function safeErrorToString(e) {
|
|
469
|
+
let isErrorLike = false;
|
|
470
|
+
try {
|
|
471
|
+
isErrorLike = e instanceof Error;
|
|
472
|
+
} catch {
|
|
721
473
|
}
|
|
722
|
-
|
|
723
|
-
const
|
|
724
|
-
if (
|
|
725
|
-
const
|
|
726
|
-
if (
|
|
727
|
-
|
|
728
|
-
if (updatedAtDiff !== 0) return updatedAtDiff;
|
|
729
|
-
return a.id.localeCompare(b.id);
|
|
474
|
+
if (isErrorLike) {
|
|
475
|
+
const msg = readErrorField(e, "message");
|
|
476
|
+
if (typeof msg === "string" && msg.length > 0) return msg;
|
|
477
|
+
const name = readErrorField(e, "name");
|
|
478
|
+
if (typeof name === "string" && name.length > 0) return name;
|
|
479
|
+
return "[Error]";
|
|
730
480
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
const scoreDiff = b.score - a.score;
|
|
740
|
-
if (!Number.isNaN(scoreDiff) && scoreDiff !== 0) return scoreDiff;
|
|
741
|
-
return a.id.localeCompare(b.id);
|
|
481
|
+
try {
|
|
482
|
+
return String(e);
|
|
483
|
+
} catch {
|
|
484
|
+
try {
|
|
485
|
+
return Object.prototype.toString.call(e);
|
|
486
|
+
} catch {
|
|
487
|
+
return "[unstringifiable error]";
|
|
488
|
+
}
|
|
742
489
|
}
|
|
743
|
-
};
|
|
744
|
-
/**
|
|
745
|
-
* Maximum number of entities whose parsed embedding vectors are held in
|
|
746
|
-
* memory. This cap is intentionally conservative so the cache remains safe
|
|
747
|
-
* on memory-constrained runtimes (e.g., mobile/Expo).
|
|
748
|
-
*/
|
|
749
|
-
_SearchService.MAX_VECTOR_CACHE_ENTITIES = 16;
|
|
750
|
-
/**
|
|
751
|
-
* Maximum number of fact vectors cached per entity. Keep this high enough to
|
|
752
|
-
* preserve the parsed-embedding reuse optimization for common mid-sized
|
|
753
|
-
* entities while still maintaining a bounded memory footprint.
|
|
754
|
-
*/
|
|
755
|
-
_SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY = 500;
|
|
756
|
-
var SearchService = _SearchService;
|
|
757
|
-
function finiteResults(results) {
|
|
758
|
-
return results.every((r) => Number.isFinite(r.score)) ? results : results.filter((r) => Number.isFinite(r.score));
|
|
759
490
|
}
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
this.activeMaintenanceJobs = /* @__PURE__ */ new Set();
|
|
766
|
-
this.activeIngestJobs = /* @__PURE__ */ new Map();
|
|
767
|
-
/**
|
|
768
|
-
* Per-(entityId, sourceHash) promise-chain lock. Serializes ingest races
|
|
769
|
-
* at the application level (the v9 partial UNIQUE index enforces it at
|
|
770
|
-
* the DB level as defense-in-depth). Two callers holding the same hash
|
|
771
|
-
* see FIFO order; different hashes never block one another.
|
|
772
|
-
*
|
|
773
|
-
* Keyed `${entityId}\0${sourceHash}` so distinct (entityId, sourceHash)
|
|
774
|
-
* pairs can never share a lock. A raw concatenation would let
|
|
775
|
-
* ("ab", "c…") collide with ("a", "bc…"); the NUL separator makes that
|
|
776
|
-
* impossible because neither component can contain NUL (entityIds are
|
|
777
|
-
* user-supplied identifiers; sourceHashes are hex digests).
|
|
778
|
-
*/
|
|
779
|
-
this.hashLocks = /* @__PURE__ */ new Map();
|
|
780
|
-
this.statusSubscribers = /* @__PURE__ */ new Map();
|
|
781
|
-
/**
|
|
782
|
-
* Lookup table for acquireLock/releaseLock's dynamic-dispatch branch.
|
|
783
|
-
* Excludes 'ingest' | 'global_reembed' | 'global_import', which those
|
|
784
|
-
* methods already handle via explicit if/else branches before reaching
|
|
785
|
-
* this table.
|
|
786
|
-
*/
|
|
787
|
-
this.lockKeyFns = {
|
|
788
|
-
prune: (id) => this._pruneKey(id),
|
|
789
|
-
librarian: (id) => this._librarianKey(id),
|
|
790
|
-
heal: (id) => this._healKey(id),
|
|
791
|
-
reembed: (id) => this._reembedKey(id),
|
|
792
|
-
import: (id) => this._importKey(id),
|
|
793
|
-
forget: (id) => this._forgetKey(id),
|
|
794
|
-
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
795
|
-
};
|
|
796
|
-
}
|
|
797
|
-
_hashLockKey(entityId, sourceHash) {
|
|
798
|
-
return `${entityId}\0${sourceHash}`;
|
|
799
|
-
}
|
|
800
|
-
/**
|
|
801
|
-
* Acquire a promise-chain lock scoped to one `(entityId, sourceHash)` pair.
|
|
802
|
-
* Returns a zero-argument release closure that removes the current tail
|
|
803
|
-
* from the chain. Releasing is idempotent so a holder that double-fires
|
|
804
|
-
* (e.g. on a finally that races an explicit release) cannot unblock
|
|
805
|
-
* the next holder prematurely.
|
|
806
|
-
*
|
|
807
|
-
* Documented ordering: ingest acquires the hash lock FIRST, then the
|
|
808
|
-
* synchronous sourceRef ingest lock, and the release closure unwinds in
|
|
809
|
-
* the opposite order (sourceRef first, hash second). Hash-then-sourceRef
|
|
810
|
-
* is required because the v9 partial UNIQUE index conflicts on the
|
|
811
|
-
* (entity_id, source_hash) pair regardless of source_ref, so two callers
|
|
812
|
-
* with DIFFERENT source_refs racing the same hash are the exact case the
|
|
813
|
-
* TOCTOU race fix needs to serialize.
|
|
814
|
-
*
|
|
815
|
-
* Mechanics: each holder owns a private `released` promise that resolves
|
|
816
|
-
* only when THIS holder calls release(). The map stores the holder's
|
|
817
|
-
* `released` promise as the "tail" — the NEXT caller awaits that tail
|
|
818
|
-
* via `previous = this.hashLocks.get(key)`. The current caller gets
|
|
819
|
-
* their unique release function once `previous` (the prior holder's
|
|
820
|
-
* release signal) settles, so each holder is guaranteed a UNIQUE release
|
|
821
|
-
* closure bound to their own `released` promise.
|
|
822
|
-
*/
|
|
823
|
-
acquireHashLock(entityId, sourceHash) {
|
|
824
|
-
const key = this._hashLockKey(entityId, sourceHash);
|
|
825
|
-
const previous = this.hashLocks.get(key) ?? Promise.resolve();
|
|
826
|
-
let resolveReleased;
|
|
827
|
-
const released = new Promise((resolve) => {
|
|
828
|
-
resolveReleased = resolve;
|
|
829
|
-
});
|
|
830
|
-
let fired = false;
|
|
831
|
-
const release = () => {
|
|
832
|
-
if (fired) return;
|
|
833
|
-
fired = true;
|
|
834
|
-
if (this.hashLocks.get(key) === released) {
|
|
835
|
-
this.hashLocks.delete(key);
|
|
836
|
-
}
|
|
837
|
-
resolveReleased();
|
|
838
|
-
};
|
|
839
|
-
this.hashLocks.set(key, released);
|
|
840
|
-
return previous.then(() => release);
|
|
841
|
-
}
|
|
842
|
-
_pruneKey(entityId) {
|
|
843
|
-
return `${this.prefix}:${entityId}:prune`;
|
|
844
|
-
}
|
|
845
|
-
_reembedKey(entityId) {
|
|
846
|
-
return `${this.prefix}:${entityId}:reembed`;
|
|
847
|
-
}
|
|
848
|
-
_globalReembedKey() {
|
|
849
|
-
return `${this.prefix}:reembed`;
|
|
491
|
+
function readErrorField(e, key) {
|
|
492
|
+
try {
|
|
493
|
+
return e[key];
|
|
494
|
+
} catch {
|
|
495
|
+
return void 0;
|
|
850
496
|
}
|
|
851
|
-
|
|
852
|
-
|
|
497
|
+
}
|
|
498
|
+
function sanitizeRankerError(err, sanitizeRankerErrors) {
|
|
499
|
+
let isErrorLike = false;
|
|
500
|
+
try {
|
|
501
|
+
isErrorLike = err instanceof Error;
|
|
502
|
+
} catch {
|
|
853
503
|
}
|
|
854
|
-
|
|
855
|
-
return
|
|
504
|
+
if (sanitizeRankerErrors === false) {
|
|
505
|
+
return isErrorLike ? err : new Error(safeErrorToString(err));
|
|
856
506
|
}
|
|
857
|
-
|
|
858
|
-
|
|
507
|
+
let errLike = null;
|
|
508
|
+
if (isErrorLike) errLike = err;
|
|
509
|
+
let typeName;
|
|
510
|
+
try {
|
|
511
|
+
if (errLike) {
|
|
512
|
+
const rawName = errLike.constructor?.name;
|
|
513
|
+
typeName = typeof rawName === "string" ? rawName : "Error";
|
|
514
|
+
} else {
|
|
515
|
+
typeName = typeof err;
|
|
516
|
+
}
|
|
517
|
+
} catch {
|
|
518
|
+
typeName = isErrorLike ? "Error" : typeof err;
|
|
859
519
|
}
|
|
860
|
-
|
|
861
|
-
|
|
520
|
+
let innerCause;
|
|
521
|
+
if (errLike) {
|
|
522
|
+
let cause;
|
|
523
|
+
try {
|
|
524
|
+
cause = errLike.cause;
|
|
525
|
+
} catch {
|
|
526
|
+
cause = void 0;
|
|
527
|
+
}
|
|
528
|
+
if (cause !== void 0) {
|
|
529
|
+
let causeName;
|
|
530
|
+
try {
|
|
531
|
+
const rawCauseName = cause?.constructor?.name;
|
|
532
|
+
causeName = typeof rawCauseName === "string" ? rawCauseName : typeof cause;
|
|
533
|
+
} catch {
|
|
534
|
+
causeName = typeof cause;
|
|
535
|
+
}
|
|
536
|
+
innerCause = new Error(`Caused by: ${causeName}`);
|
|
537
|
+
}
|
|
862
538
|
}
|
|
863
|
-
|
|
864
|
-
|
|
539
|
+
const sanitized = new Error(
|
|
540
|
+
`VectorRanker ${typeName} (message scrubbed for security)`,
|
|
541
|
+
innerCause ? { cause: innerCause } : void 0
|
|
542
|
+
);
|
|
543
|
+
try {
|
|
544
|
+
sanitized.name = typeName;
|
|
545
|
+
} catch {
|
|
865
546
|
}
|
|
866
|
-
|
|
867
|
-
|
|
547
|
+
return sanitized;
|
|
548
|
+
}
|
|
549
|
+
function safeSlice(value, start, end) {
|
|
550
|
+
const length = value.length;
|
|
551
|
+
let safeStart = start < 0 ? Math.max(length + start, 0) : Math.min(start, length);
|
|
552
|
+
let safeEnd = end === void 0 ? length : end < 0 ? Math.max(length + end, 0) : Math.min(end, length);
|
|
553
|
+
if (safeStart > safeEnd) {
|
|
554
|
+
[safeStart, safeEnd] = [safeEnd, safeStart];
|
|
868
555
|
}
|
|
869
|
-
|
|
870
|
-
|
|
556
|
+
if (safeStart > 0 && safeStart < length && value.charCodeAt(safeStart) >= 56320 && value.charCodeAt(safeStart) <= 57343 && value.charCodeAt(safeStart - 1) >= 55296 && value.charCodeAt(safeStart - 1) <= 56319) {
|
|
557
|
+
safeStart--;
|
|
871
558
|
}
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
*
|
|
875
|
-
* Read-only; acquires nothing. Callers that mutate embedding failure
|
|
876
|
-
* markers across ALL entities (the dimension-promotion clear) must gate on
|
|
877
|
-
* this and defer, rather than resurrect rows an in-flight sweep already
|
|
878
|
-
* classified. See spec 2026-09-05-reembed-lock-scope-design.md §2.
|
|
879
|
-
*
|
|
880
|
-
* The global key `${prefix}:reembed` also ends in ':reembed', so the suffix
|
|
881
|
-
* scan alone would already match it. The explicit check is kept because it
|
|
882
|
-
* documents the intent and survives any future change to the global key's
|
|
883
|
-
* format.
|
|
884
|
-
*/
|
|
885
|
-
isAnyReembedActive() {
|
|
886
|
-
return this.activeMaintenanceJobs.has(this._globalReembedKey()) || this._isAnyMaintenanceActiveWithSuffix(":reembed");
|
|
559
|
+
if (safeEnd > 0 && safeEnd < length && value.charCodeAt(safeEnd - 1) >= 55296 && value.charCodeAt(safeEnd - 1) <= 56319 && value.charCodeAt(safeEnd) >= 56320 && value.charCodeAt(safeEnd) <= 57343) {
|
|
560
|
+
safeEnd--;
|
|
887
561
|
}
|
|
888
|
-
|
|
889
|
-
|
|
562
|
+
return value.slice(safeStart, safeEnd);
|
|
563
|
+
}
|
|
564
|
+
function chunkText(input, maxChunkLength, overlap) {
|
|
565
|
+
const text = input.trim();
|
|
566
|
+
if (text.length === 0) return { chunks: [], truncated: false };
|
|
567
|
+
if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
|
|
568
|
+
throw new Error("maxChunkLength must be an integer >= 2");
|
|
890
569
|
}
|
|
891
|
-
|
|
892
|
-
|
|
570
|
+
if (!Number.isInteger(overlap) || overlap < 0 || overlap >= maxChunkLength) {
|
|
571
|
+
throw new Error("overlap must be a non-negative integer < maxChunkLength");
|
|
893
572
|
}
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
573
|
+
const chunks = [];
|
|
574
|
+
let truncated = false;
|
|
575
|
+
let cursor = 0;
|
|
576
|
+
const halfMax = Math.floor(maxChunkLength / 2);
|
|
577
|
+
while (cursor < text.length) {
|
|
578
|
+
const remaining = text.length - cursor;
|
|
579
|
+
if (remaining <= maxChunkLength) {
|
|
580
|
+
chunks.push(safeSlice(text, cursor, text.length));
|
|
581
|
+
break;
|
|
898
582
|
}
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
const sourceKey = sourceRef ?? "";
|
|
906
|
-
let refs = this.activeIngestJobs.get(entityId);
|
|
907
|
-
if (!refs) {
|
|
908
|
-
refs = /* @__PURE__ */ new Set();
|
|
909
|
-
this.activeIngestJobs.set(entityId, refs);
|
|
583
|
+
const windowEnd = cursor + maxChunkLength;
|
|
584
|
+
const minSplit = cursor + halfMax;
|
|
585
|
+
let splitPoint = -1;
|
|
586
|
+
const paraIdx = text.lastIndexOf("\n\n", windowEnd);
|
|
587
|
+
if (paraIdx >= minSplit && paraIdx + 2 <= windowEnd) {
|
|
588
|
+
splitPoint = paraIdx + 2;
|
|
910
589
|
}
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
590
|
+
if (splitPoint === -1) {
|
|
591
|
+
let lastTerm = -1;
|
|
592
|
+
for (let i = minSplit; i < windowEnd - 1; i++) {
|
|
593
|
+
const ch = text[i];
|
|
594
|
+
if ((ch === "." || ch === "!" || ch === "?") && /\s/.test(text[i + 1])) {
|
|
595
|
+
lastTerm = i + 2;
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
if (lastTerm !== -1 && lastTerm <= windowEnd) splitPoint = lastTerm;
|
|
920
599
|
}
|
|
600
|
+
if (splitPoint === -1) {
|
|
601
|
+
for (let i = windowEnd - 1; i >= minSplit; i--) {
|
|
602
|
+
if (/\s/.test(text[i])) {
|
|
603
|
+
splitPoint = i + 1;
|
|
604
|
+
break;
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
if (splitPoint === -1) {
|
|
609
|
+
truncated = true;
|
|
610
|
+
splitPoint = windowEnd;
|
|
611
|
+
}
|
|
612
|
+
chunks.push(safeSlice(text, cursor, splitPoint));
|
|
613
|
+
const next = Math.max(splitPoint - overlap, cursor + 1);
|
|
614
|
+
cursor = next;
|
|
921
615
|
}
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
616
|
+
return { chunks, truncated };
|
|
617
|
+
}
|
|
618
|
+
async function withConcurrency(tasks, limit) {
|
|
619
|
+
const results = new Array(tasks.length);
|
|
620
|
+
let index = 0;
|
|
621
|
+
let failed = false;
|
|
622
|
+
let firstError;
|
|
623
|
+
async function worker() {
|
|
624
|
+
while (index < tasks.length && !failed) {
|
|
625
|
+
const i = index++;
|
|
626
|
+
try {
|
|
627
|
+
results[i] = await tasks[i]();
|
|
628
|
+
} catch (e) {
|
|
629
|
+
if (!failed) {
|
|
630
|
+
failed = true;
|
|
631
|
+
firstError = e;
|
|
632
|
+
}
|
|
633
|
+
return;
|
|
634
|
+
}
|
|
929
635
|
}
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
636
|
+
}
|
|
637
|
+
const workerCount = tasks.length === 0 ? 0 : Math.min(Math.max(limit, 1), tasks.length);
|
|
638
|
+
await Promise.allSettled(Array.from({ length: workerCount }, worker));
|
|
639
|
+
if (failed) throw firstError;
|
|
640
|
+
return results;
|
|
641
|
+
}
|
|
642
|
+
function clip(value, max) {
|
|
643
|
+
if (typeof value !== "string") return "";
|
|
644
|
+
const s = value.trim();
|
|
645
|
+
return s.length <= max ? s : safeSlice(s, 0, max).trimEnd();
|
|
646
|
+
}
|
|
647
|
+
function validateTags(tags) {
|
|
648
|
+
if (!Array.isArray(tags)) return [];
|
|
649
|
+
return tags.filter((t) => typeof t === "string").map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0 && t.length <= 40).slice(0, 6);
|
|
650
|
+
}
|
|
651
|
+
var MAX_EVIDENCE_QUOTES = 10;
|
|
652
|
+
function normalizeEvidence(raw) {
|
|
653
|
+
if (!Array.isArray(raw)) return void 0;
|
|
654
|
+
const out = [];
|
|
655
|
+
for (const entry of raw) {
|
|
656
|
+
if (typeof entry !== "string") continue;
|
|
657
|
+
const trimmed = entry.trim();
|
|
658
|
+
if (!trimmed) continue;
|
|
659
|
+
out.push(trimmed);
|
|
660
|
+
if (out.length > MAX_EVIDENCE_QUOTES) break;
|
|
661
|
+
}
|
|
662
|
+
return out;
|
|
663
|
+
}
|
|
664
|
+
function validateFact(fact) {
|
|
665
|
+
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
|
|
666
|
+
const title = clip(fact.title, 80);
|
|
667
|
+
const body = clip(fact.body, 800);
|
|
668
|
+
if (!title || !body) return null;
|
|
669
|
+
let confidence = fact.confidence;
|
|
670
|
+
if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
|
|
671
|
+
const valid = {
|
|
672
|
+
...fact,
|
|
673
|
+
title,
|
|
674
|
+
body,
|
|
675
|
+
confidence,
|
|
676
|
+
tags: validateTags(fact.tags)
|
|
677
|
+
};
|
|
678
|
+
const evidence = normalizeEvidence(fact.evidence);
|
|
679
|
+
if (evidence) valid.evidence = evidence;
|
|
680
|
+
else delete valid.evidence;
|
|
681
|
+
return valid;
|
|
682
|
+
}
|
|
683
|
+
function validateTask(task) {
|
|
684
|
+
if (typeof task?.description !== "string") return null;
|
|
685
|
+
const description = clip(task.description, 200);
|
|
686
|
+
if (!description) return null;
|
|
687
|
+
let priority = task.priority;
|
|
688
|
+
if (typeof priority !== "number" || !isFinite(priority)) priority = 0;
|
|
689
|
+
priority = Math.max(0, Math.min(10, Math.round(priority)));
|
|
690
|
+
return {
|
|
691
|
+
...task,
|
|
692
|
+
description,
|
|
693
|
+
priority
|
|
694
|
+
};
|
|
695
|
+
}
|
|
696
|
+
function factRejectionReason(raw) {
|
|
697
|
+
const fact = raw;
|
|
698
|
+
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return "invalid_shape";
|
|
699
|
+
if (!clip(fact.title, 80)) return "missing_title";
|
|
700
|
+
return "missing_body";
|
|
701
|
+
}
|
|
702
|
+
function taskRejectionReason(raw) {
|
|
703
|
+
const task = raw;
|
|
704
|
+
if (typeof task?.description !== "string") return "invalid_shape";
|
|
705
|
+
return "missing_description";
|
|
706
|
+
}
|
|
707
|
+
function normalizeSourceRef(value) {
|
|
708
|
+
if (typeof value !== "string") return null;
|
|
709
|
+
const cleaned = value.replace(/[^A-Za-z0-9._\- ]/g, "").trim().slice(0, 255).trimEnd();
|
|
710
|
+
return cleaned.length > 0 ? cleaned : null;
|
|
711
|
+
}
|
|
712
|
+
function normalizeSourceHash(value) {
|
|
713
|
+
if (typeof value !== "string") return null;
|
|
714
|
+
return /^[0-9a-f]{64}$/i.test(value) ? value.toLowerCase() : null;
|
|
715
|
+
}
|
|
716
|
+
function titleTokens(title) {
|
|
717
|
+
return new Set(title.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3));
|
|
718
|
+
}
|
|
719
|
+
function jaccardScore(a, b) {
|
|
720
|
+
if (a.size === 0 && b.size === 0) return 0;
|
|
721
|
+
const intersection = new Set([...a].filter((x) => b.has(x)));
|
|
722
|
+
const union = /* @__PURE__ */ new Set([...a, ...b]);
|
|
723
|
+
return intersection.size / union.size;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
// src/utils/ids.ts
|
|
727
|
+
var _injectedGetRandomValues = null;
|
|
728
|
+
function configureRandomSource(fn) {
|
|
729
|
+
_injectedGetRandomValues = fn;
|
|
730
|
+
}
|
|
731
|
+
function generateId(prefix = "") {
|
|
732
|
+
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
|
|
733
|
+
return prefix + crypto.randomUUID().replace(/-/g, "").substring(0, 24);
|
|
734
|
+
}
|
|
735
|
+
if (typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function") {
|
|
736
|
+
const bytes = new Uint8Array(16);
|
|
737
|
+
crypto.getRandomValues(bytes);
|
|
738
|
+
return prefix + Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("").substring(0, 24);
|
|
739
|
+
}
|
|
740
|
+
if (_injectedGetRandomValues) {
|
|
741
|
+
const bytes = new Uint8Array(16);
|
|
742
|
+
_injectedGetRandomValues(bytes);
|
|
743
|
+
return prefix + Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("").substring(0, 24);
|
|
744
|
+
}
|
|
745
|
+
throw new Error(
|
|
746
|
+
"generateId: no cryptographically secure random source available (crypto.randomUUID and crypto.getRandomValues are both missing, and no configureRandomSource() injection was provided)."
|
|
747
|
+
);
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
// src/utils/ontology.ts
|
|
751
|
+
function emptyManifest() {
|
|
752
|
+
return { node_types: [], edge_types: [] };
|
|
753
|
+
}
|
|
754
|
+
function normalizeTitleKey(title) {
|
|
755
|
+
return title.trim().toLowerCase().replace(/\s+/g, " ");
|
|
756
|
+
}
|
|
757
|
+
function resolveNodeType(raw, manifest) {
|
|
758
|
+
const slug = raw.trim();
|
|
759
|
+
if (!slug) return null;
|
|
760
|
+
const hit = manifest.node_types.find((n) => n.type.toLowerCase() === slug.toLowerCase());
|
|
761
|
+
return hit?.type ?? null;
|
|
762
|
+
}
|
|
763
|
+
function resolveEdgeDefinitions(rawEdgeType, manifest) {
|
|
764
|
+
const slug = rawEdgeType.trim();
|
|
765
|
+
if (!slug) return [];
|
|
766
|
+
return manifest.edge_types.filter((e) => e.type.toLowerCase() === slug.toLowerCase());
|
|
767
|
+
}
|
|
768
|
+
function edgeTripleKey(type, sourceType, targetType) {
|
|
769
|
+
return `${type.trim().toLowerCase()}|${sourceType.trim().toLowerCase()}|${targetType.trim().toLowerCase()}`;
|
|
770
|
+
}
|
|
771
|
+
function typeSatisfies(declaredType, concreteType, manifest) {
|
|
772
|
+
const concrete = concreteType.trim().toLowerCase();
|
|
773
|
+
const declared = declaredType.trim().toLowerCase();
|
|
774
|
+
if (!concrete || !declared) return false;
|
|
775
|
+
if (declared === concrete) return true;
|
|
776
|
+
const def = (manifest.node_types ?? []).find(
|
|
777
|
+
(n) => typeof n?.type === "string" && n.type.trim().toLowerCase() === concrete
|
|
778
|
+
);
|
|
779
|
+
const parent = typeof def?.parent_type === "string" ? def.parent_type.trim().toLowerCase() : "";
|
|
780
|
+
return parent !== "" && parent === declared;
|
|
781
|
+
}
|
|
782
|
+
function edgeTripleAllowed(manifest, edgeType, sourceType, targetType) {
|
|
783
|
+
const wanted = edgeType.trim().toLowerCase();
|
|
784
|
+
return (manifest.edge_types ?? []).some((d) => typeof d?.type === "string" && d.type.trim().toLowerCase() === wanted && typeof d.source_type === "string" && typeSatisfies(d.source_type, sourceType, manifest) && typeof d.target_type === "string" && typeSatisfies(d.target_type, targetType, manifest));
|
|
785
|
+
}
|
|
786
|
+
function validateManifest(manifest) {
|
|
787
|
+
const nodeSlugs = /* @__PURE__ */ new Set();
|
|
788
|
+
for (const node of manifest.node_types ?? []) {
|
|
789
|
+
const type = node.type?.trim();
|
|
790
|
+
if (!type) throw new Error("Ontology node type slug must be non-empty");
|
|
791
|
+
const key = type.toLowerCase();
|
|
792
|
+
if (nodeSlugs.has(key)) throw new Error(`Duplicate node type: ${type}`);
|
|
793
|
+
nodeSlugs.add(key);
|
|
794
|
+
}
|
|
795
|
+
const parentOf = /* @__PURE__ */ new Map();
|
|
796
|
+
for (const node of manifest.node_types ?? []) {
|
|
797
|
+
const parent = typeof node.parent_type === "string" ? node.parent_type.trim().toLowerCase() : void 0;
|
|
798
|
+
parentOf.set(node.type.trim().toLowerCase(), parent);
|
|
799
|
+
}
|
|
800
|
+
for (const node of manifest.node_types ?? []) {
|
|
801
|
+
if (node.parent_type === void 0) continue;
|
|
802
|
+
if (typeof node.parent_type !== "string" || !node.parent_type.trim()) {
|
|
803
|
+
throw new Error(`Ontology parent_type must be a non-empty string when present: ${node.type}`);
|
|
804
|
+
}
|
|
805
|
+
const parentSlug = node.parent_type.trim().toLowerCase();
|
|
806
|
+
if (parentSlug === node.type.trim().toLowerCase()) {
|
|
807
|
+
throw new Error(`Self-parent: ${node.type}`);
|
|
808
|
+
}
|
|
809
|
+
if (!nodeSlugs.has(parentSlug)) {
|
|
810
|
+
throw new Error(`Parent type not found: ${node.parent_type}`);
|
|
811
|
+
}
|
|
812
|
+
const grandparent = parentOf.get(parentSlug);
|
|
813
|
+
if (grandparent) {
|
|
814
|
+
throw new Error(`Parent chain too deep: ${node.type} \u2192 ${node.parent_type} \u2192 ${grandparent}`);
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
const edgeKeys = /* @__PURE__ */ new Set();
|
|
818
|
+
const edgeNames = /* @__PURE__ */ new Map();
|
|
819
|
+
for (const edge of manifest.edge_types ?? []) {
|
|
820
|
+
const edgeType = edge.type?.trim();
|
|
821
|
+
const sourceType = edge.source_type?.trim();
|
|
822
|
+
const targetType = edge.target_type?.trim();
|
|
823
|
+
if (!edgeType) throw new Error("Ontology edge type slug must be non-empty");
|
|
824
|
+
if (!sourceType || !targetType || !nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) {
|
|
825
|
+
throw new Error(`Edge type ${edgeType} references unknown node type`);
|
|
826
|
+
}
|
|
827
|
+
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
828
|
+
if (edgeKeys.has(edgeKey)) {
|
|
829
|
+
throw new Error(`Duplicate edge definition: ${edgeType} (${sourceType} \u2192 ${targetType})`);
|
|
830
|
+
}
|
|
831
|
+
edgeKeys.add(edgeKey);
|
|
832
|
+
const canonical = edgeNames.get(edgeType.toLowerCase());
|
|
833
|
+
if (canonical === void 0) {
|
|
834
|
+
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
835
|
+
} else if (canonical !== edgeType) {
|
|
836
|
+
throw new Error(`Inconsistent casing for edge type: ${edgeType} conflicts with ${canonical}`);
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
function buildDeclaresParentIndex(nodes) {
|
|
841
|
+
const declaresParent = /* @__PURE__ */ new Map();
|
|
842
|
+
for (const n of nodes) {
|
|
843
|
+
const slug = typeof n?.type === "string" ? n.type.trim().toLowerCase() : "";
|
|
844
|
+
if (!slug || declaresParent.has(slug)) continue;
|
|
845
|
+
declaresParent.set(slug, n?.parent_type !== void 0);
|
|
846
|
+
}
|
|
847
|
+
return declaresParent;
|
|
848
|
+
}
|
|
849
|
+
function mergeOntologyUpdates(current, updates) {
|
|
850
|
+
const node_types = [...current.node_types];
|
|
851
|
+
const edge_types = [...current.edge_types];
|
|
852
|
+
const nodeSlugs = new Set(node_types.map((n) => n.type.trim().toLowerCase()));
|
|
853
|
+
const edgeKeys = new Set(edge_types.map((e) => edgeTripleKey(e.type, e.source_type, e.target_type)));
|
|
854
|
+
const edgeNames = new Map(edge_types.map((e) => [e.type.trim().toLowerCase(), e.type.trim()]));
|
|
855
|
+
const declaresParent = buildDeclaresParentIndex([
|
|
856
|
+
...current.node_types,
|
|
857
|
+
...updates.node_types ?? []
|
|
858
|
+
]);
|
|
859
|
+
for (const node of updates.node_types ?? []) {
|
|
860
|
+
const type = node?.type?.trim();
|
|
861
|
+
if (!type) continue;
|
|
862
|
+
const key = type.toLowerCase();
|
|
863
|
+
if (nodeSlugs.has(key)) continue;
|
|
864
|
+
const rawParent = typeof node?.parent_type === "string" ? node.parent_type.trim() : "";
|
|
865
|
+
const parentSlug = rawParent.toLowerCase();
|
|
866
|
+
const keepParent = parentSlug !== "" && parentSlug !== key && declaresParent.get(parentSlug) === false;
|
|
867
|
+
node_types.push({
|
|
868
|
+
type,
|
|
869
|
+
description: String(node.description ?? ""),
|
|
870
|
+
...keepParent ? { parent_type: rawParent } : {}
|
|
871
|
+
});
|
|
872
|
+
nodeSlugs.add(key);
|
|
873
|
+
}
|
|
874
|
+
for (const edge of updates.edge_types ?? []) {
|
|
875
|
+
const rawEdgeType = edge?.type?.trim();
|
|
876
|
+
const sourceType = edge?.source_type?.trim();
|
|
877
|
+
const targetType = edge?.target_type?.trim();
|
|
878
|
+
if (!rawEdgeType || !sourceType || !targetType) continue;
|
|
879
|
+
const edgeType = edgeNames.get(rawEdgeType.toLowerCase()) ?? rawEdgeType;
|
|
880
|
+
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
881
|
+
if (edgeKeys.has(edgeKey)) continue;
|
|
882
|
+
if (!nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) continue;
|
|
883
|
+
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
884
|
+
edge_types.push({
|
|
885
|
+
type: edgeType,
|
|
886
|
+
source_type: sourceType,
|
|
887
|
+
target_type: targetType,
|
|
888
|
+
description: String(edge.description ?? "")
|
|
889
|
+
});
|
|
890
|
+
edgeKeys.add(edgeKey);
|
|
891
|
+
}
|
|
892
|
+
return { node_types, edge_types };
|
|
893
|
+
}
|
|
894
|
+
function validateInlineEdges(sourceType, _targetType, edges, manifest, opts) {
|
|
895
|
+
const strict = opts?.strict === true;
|
|
896
|
+
const entityId = opts?.entityId ?? "";
|
|
897
|
+
const drops = opts?.drops;
|
|
898
|
+
const drop = (reason, edgeType) => {
|
|
899
|
+
drops?.push({ reason, sourceId: null, edgeType, sourceNodeType: sourceType, targetNodeType: null });
|
|
900
|
+
};
|
|
901
|
+
if (!Array.isArray(edges)) {
|
|
902
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", "");
|
|
903
|
+
drop("invalid_shape", null);
|
|
904
|
+
return [];
|
|
905
|
+
}
|
|
906
|
+
const valid = [];
|
|
907
|
+
for (const edge of edges) {
|
|
908
|
+
if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string") {
|
|
909
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", String(edge?.edge_type ?? ""));
|
|
910
|
+
drop("invalid_shape", typeof edge?.edge_type === "string" ? edge.edge_type : null);
|
|
911
|
+
continue;
|
|
912
|
+
}
|
|
913
|
+
const defs = resolveEdgeDefinitions(edge.edge_type, manifest);
|
|
914
|
+
const match = defs.find((d) => typeSatisfies(d.source_type, sourceType, manifest));
|
|
915
|
+
if (!match) {
|
|
916
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", edge.edge_type);
|
|
917
|
+
drop("type_not_in_manifest", edge.edge_type);
|
|
918
|
+
continue;
|
|
919
|
+
}
|
|
920
|
+
valid.push({ edge_type: match.type, target_title: edge.target_title });
|
|
921
|
+
}
|
|
922
|
+
return valid;
|
|
923
|
+
}
|
|
924
|
+
|
|
925
|
+
// src/utils/embedding.ts
|
|
926
|
+
function parseEmbedding(blob, text) {
|
|
927
|
+
if (blob && blob.byteLength > 0) {
|
|
928
|
+
if (blob.byteLength % 4 !== 0) return null;
|
|
929
|
+
const copy = new ArrayBuffer(blob.byteLength);
|
|
930
|
+
new Uint8Array(copy).set(blob);
|
|
931
|
+
const vector = new Float32Array(copy);
|
|
932
|
+
for (const value of vector) {
|
|
933
|
+
if (!Number.isFinite(value)) return null;
|
|
934
|
+
}
|
|
935
|
+
return vector;
|
|
936
|
+
}
|
|
937
|
+
if (text) {
|
|
938
|
+
try {
|
|
939
|
+
const arr = JSON.parse(text);
|
|
940
|
+
if (!Array.isArray(arr) || !arr.every((v) => typeof v === "number" && isFinite(v))) return null;
|
|
941
|
+
const vector = new Float32Array(arr);
|
|
942
|
+
for (const value of vector) {
|
|
943
|
+
if (!Number.isFinite(value)) return null;
|
|
944
|
+
}
|
|
945
|
+
return vector;
|
|
946
|
+
} catch {
|
|
947
|
+
return null;
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
return null;
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
// src/utils/cosine.ts
|
|
954
|
+
function cosineSimilarity(a, b) {
|
|
955
|
+
let dot = 0, normA = 0, normB = 0;
|
|
956
|
+
const len = Math.min(a.length, b.length);
|
|
957
|
+
for (let i = 0; i < len; i++) {
|
|
958
|
+
dot += a[i] * b[i];
|
|
959
|
+
normA += a[i] * a[i];
|
|
960
|
+
normB += b[i] * b[i];
|
|
961
|
+
}
|
|
962
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
963
|
+
return denom === 0 ? 0 : dot / denom;
|
|
964
|
+
}
|
|
965
|
+
|
|
966
|
+
// src/utils/indexDoc.ts
|
|
967
|
+
function toIndexDoc(row) {
|
|
968
|
+
return {
|
|
969
|
+
id: row.id,
|
|
970
|
+
entity_id: row.entity_id,
|
|
971
|
+
title: row.title,
|
|
972
|
+
body: row.body,
|
|
973
|
+
tags: (() => {
|
|
974
|
+
try {
|
|
975
|
+
const parsed = JSON.parse(row.tags);
|
|
976
|
+
return Array.isArray(parsed) ? parsed.join(" ") : row.tags;
|
|
977
|
+
} catch {
|
|
978
|
+
return row.tags;
|
|
951
979
|
}
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
980
|
+
})()
|
|
981
|
+
};
|
|
982
|
+
}
|
|
983
|
+
|
|
984
|
+
// src/services/search/UnavailableIndexStrategy.ts
|
|
985
|
+
var UnavailableIndexStrategy = class {
|
|
986
|
+
constructor(cause) {
|
|
987
|
+
this.cause = cause;
|
|
988
|
+
}
|
|
989
|
+
async search(_query, options) {
|
|
990
|
+
if (options.entityIds.length === 0) return [];
|
|
991
|
+
throw new WikiSearchUnavailableError(void 0, this.cause);
|
|
992
|
+
}
|
|
993
|
+
async replace(_entityId, _ids, _documents) {
|
|
994
|
+
}
|
|
995
|
+
async replaceEntity(_entityId, _documents) {
|
|
996
|
+
}
|
|
997
|
+
async replaceAll(_documents) {
|
|
998
|
+
}
|
|
999
|
+
hasIndexedEntity(_entityId) {
|
|
1000
|
+
return false;
|
|
1001
|
+
}
|
|
1002
|
+
async drain() {
|
|
1003
|
+
}
|
|
1004
|
+
};
|
|
1005
|
+
|
|
1006
|
+
// src/services/SearchService.ts
|
|
1007
|
+
var _SearchService = class _SearchService {
|
|
1008
|
+
constructor(entryRepo, indexStrategy) {
|
|
1009
|
+
this.entryRepo = entryRepo;
|
|
1010
|
+
this.vectorCache = /* @__PURE__ */ new Map();
|
|
1011
|
+
/**
|
|
1012
|
+
* Serializes rebuilds. `rebuildIndex` awaits a repository read between
|
|
1013
|
+
* snapshotting the previous id set and discarding it, so two concurrent
|
|
1014
|
+
* sync() calls for one entity can interleave: a slow, stale read lands last
|
|
1015
|
+
* and discards documents the fresh read just added. Chaining also keeps
|
|
1016
|
+
* discard()/addAll() out of each other's way, which is what accrued the
|
|
1017
|
+
* auto-vacuum debt behind the TypeError in #64.
|
|
1018
|
+
*/
|
|
1019
|
+
this.syncChain = Promise.resolve();
|
|
1020
|
+
/**
|
|
1021
|
+
* Entities whose index may have drifted from SQLite: an incremental update
|
|
1022
|
+
* failed part-way, or core wrote rows it could not index (upsertGraph runs in
|
|
1023
|
+
* the host's transaction). Their next syncEntries rebuilds the entity in full.
|
|
1024
|
+
* See spec 2026-09-28 §5.
|
|
1025
|
+
*/
|
|
1026
|
+
this.staleEntities = /* @__PURE__ */ new Set();
|
|
1027
|
+
/**
|
|
1028
|
+
* Per-entity count of markStale() calls. A rebuild clears an entity's stale
|
|
1029
|
+
* flag only when the count it snapshotted before its read still holds
|
|
1030
|
+
* afterwards: markStale() runs inside the host's still-open transaction, so
|
|
1031
|
+
* a call landing mid-read belongs to rows the read cannot have seen, and
|
|
1032
|
+
* its flag must outlive the turn (#233 review).
|
|
1033
|
+
*/
|
|
1034
|
+
this.staleEpochs = /* @__PURE__ */ new Map();
|
|
1035
|
+
this.indexStrategy = indexStrategy;
|
|
1036
|
+
}
|
|
1037
|
+
/**
|
|
1038
|
+
* Install the strategy resolved by `WikiMemory.setup()`. Before setup, the
|
|
1039
|
+
* service runs on an empty MiniSearch placeholder, as it always has.
|
|
1040
|
+
*/
|
|
1041
|
+
setIndexStrategy(strategy) {
|
|
1042
|
+
this.indexStrategy = strategy;
|
|
1043
|
+
}
|
|
1044
|
+
/** False while the installed strategy is the degraded UnavailableIndexStrategy (spec #280 Change 4). */
|
|
1045
|
+
isKeywordAvailable() {
|
|
1046
|
+
return !(this.indexStrategy instanceof UnavailableIndexStrategy);
|
|
1047
|
+
}
|
|
1048
|
+
/**
|
|
1049
|
+
* One serialized drain turn for strategies with a durable ledger. Like
|
|
1050
|
+
* sync(), it never rejects: the keyword index is rebuildable, and a failed
|
|
1051
|
+
* drain leaves the ledger rows in place for the next turn.
|
|
1052
|
+
*/
|
|
1053
|
+
drainTurn(entityId) {
|
|
1054
|
+
const work = this.syncChain.then(async () => {
|
|
1055
|
+
try {
|
|
1056
|
+
try {
|
|
1057
|
+
await this.indexStrategy.drain();
|
|
1058
|
+
} finally {
|
|
1059
|
+
this.evictCache(entityId);
|
|
1060
|
+
}
|
|
1061
|
+
} catch (err) {
|
|
1062
|
+
console.warn(`[WikiMemory] search index drain failed for ${entityId ?? "*"}:`, err);
|
|
987
1063
|
}
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
1064
|
+
});
|
|
1065
|
+
this.syncChain = work;
|
|
1066
|
+
return work;
|
|
1067
|
+
}
|
|
1068
|
+
/**
|
|
1069
|
+
* Rebuilds the search index and clears the vector cache for a given entity.
|
|
1070
|
+
* A direct replacement for manually syncing state after a DB transaction.
|
|
1071
|
+
*
|
|
1072
|
+
* Rebuilds are serialized per instance and never reject: the keyword index
|
|
1073
|
+
* is a rebuildable cache over SQLite, so degraded keyword search is the
|
|
1074
|
+
* correct failure mode and killing the host process is not.
|
|
1075
|
+
*/
|
|
1076
|
+
async sync(entityId) {
|
|
1077
|
+
if (this.indexStrategy.drain) return this.drainTurn(entityId);
|
|
1078
|
+
const work = this.syncChain.then(async () => {
|
|
1079
|
+
try {
|
|
1080
|
+
try {
|
|
1081
|
+
const epochsBefore = new Map(this.staleEpochs);
|
|
1082
|
+
if (entityId === void 0) {
|
|
1083
|
+
const rows = await this.entryRepo.findMiniSearchRows();
|
|
1084
|
+
await this.indexStrategy.replaceAll(rows.map(toIndexDoc));
|
|
1085
|
+
for (const id of [...this.staleEntities]) {
|
|
1086
|
+
if ((this.staleEpochs.get(id) ?? 0) === (epochsBefore.get(id) ?? 0)) {
|
|
1087
|
+
this.staleEntities.delete(id);
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
} else {
|
|
1091
|
+
await this.rebuildIndex(entityId);
|
|
1092
|
+
if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
|
|
1093
|
+
this.staleEntities.delete(entityId);
|
|
1094
|
+
}
|
|
1095
|
+
}
|
|
1096
|
+
await this.indexStrategy.vacuum?.();
|
|
1097
|
+
} finally {
|
|
1098
|
+
this.evictCache(entityId);
|
|
1099
|
+
}
|
|
1100
|
+
} catch (err) {
|
|
1101
|
+
console.warn(`[WikiMemory] search index rebuild failed for ${entityId ?? "*"}:`, err);
|
|
999
1102
|
}
|
|
1103
|
+
});
|
|
1104
|
+
this.syncChain = work;
|
|
1105
|
+
return work;
|
|
1106
|
+
}
|
|
1107
|
+
/**
|
|
1108
|
+
* Re-indexes only `ids` for `entityId`: drops each from the index, then
|
|
1109
|
+
* re-adds the ones still live in SQLite, so soft-deleted or missing ids end
|
|
1110
|
+
* up absent. Costs O(ids), where sync(entityId) costs O(entity) — callers that
|
|
1111
|
+
* know what a write touched use this so chunked imports stay linear (#232).
|
|
1112
|
+
*
|
|
1113
|
+
* Serialized with sync() on the same chain and, like it, never rejects. An
|
|
1114
|
+
* entity that is stale or has never been indexed gets a full rebuild instead
|
|
1115
|
+
* — including on an empty `ids` list, which is a no-op only for an entity
|
|
1116
|
+
* the index already tracks, and even then one that waits for rebuilds
|
|
1117
|
+
* already queued on the chain.
|
|
1118
|
+
*/
|
|
1119
|
+
async syncEntries(entityId, ids) {
|
|
1120
|
+
if (this.indexStrategy.drain) return this.drainTurn(entityId);
|
|
1121
|
+
const uniqueIds = [...new Set(ids)];
|
|
1122
|
+
const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
|
|
1123
|
+
if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
|
|
1124
|
+
return this.syncChain;
|
|
1000
1125
|
}
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1126
|
+
const work = this.syncChain.then(async () => {
|
|
1127
|
+
try {
|
|
1128
|
+
try {
|
|
1129
|
+
const epochsBefore = new Map(this.staleEpochs);
|
|
1130
|
+
const needsRebuild = () => this.staleEntities.has(entityId) || this.indexStrategy.hasIndexedEntity?.(entityId) !== true;
|
|
1131
|
+
if (!needsRebuild()) {
|
|
1132
|
+
const rows = await this.entryRepo.findMiniSearchRowsByIds(entityId, uniqueIds);
|
|
1133
|
+
if (!this.staleEntities.has(entityId)) {
|
|
1134
|
+
await this.indexStrategy.replace(entityId, uniqueIds, rows.map(toIndexDoc));
|
|
1135
|
+
await this.indexStrategy.vacuum?.();
|
|
1136
|
+
return;
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
await this.rebuildIndex(entityId);
|
|
1140
|
+
if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
|
|
1141
|
+
this.staleEntities.delete(entityId);
|
|
1142
|
+
}
|
|
1143
|
+
await this.indexStrategy.vacuum?.();
|
|
1144
|
+
} finally {
|
|
1145
|
+
this.evictCache(entityId);
|
|
1146
|
+
}
|
|
1147
|
+
} catch (err) {
|
|
1148
|
+
this.staleEntities.add(entityId);
|
|
1149
|
+
console.warn(`[WikiMemory] search index incremental sync failed for ${entityId}:`, err);
|
|
1150
|
+
}
|
|
1151
|
+
});
|
|
1152
|
+
this.syncChain = work;
|
|
1153
|
+
return work;
|
|
1154
|
+
}
|
|
1155
|
+
/**
|
|
1156
|
+
* Forces the entity's next syncEntries to rebuild it in full. For writes core
|
|
1157
|
+
* cannot index itself, such as upsertGraph inside a host transaction.
|
|
1158
|
+
*/
|
|
1159
|
+
markStale(entityId) {
|
|
1160
|
+
if (this.indexStrategy.drain) return;
|
|
1161
|
+
this.staleEntities.add(entityId);
|
|
1162
|
+
this.staleEpochs.set(entityId, (this.staleEpochs.get(entityId) ?? 0) + 1);
|
|
1017
1163
|
}
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1164
|
+
/**
|
|
1165
|
+
* Runs syncEntries(id, []) for each entity marked stale when this is called,
|
|
1166
|
+
* so only those entities are rebuilt. Each rebuild still takes its own turn
|
|
1167
|
+
* on the sync chain, and like syncEntries this never rejects. With nothing
|
|
1168
|
+
* stale it resolves at once, without waiting on work already on the chain.
|
|
1169
|
+
*/
|
|
1170
|
+
async syncStale() {
|
|
1171
|
+
if (this.indexStrategy.drain) return this.drainTurn();
|
|
1172
|
+
const ids = [...this.staleEntities];
|
|
1173
|
+
await Promise.all(ids.map((id) => this.syncEntries(id, [])));
|
|
1174
|
+
}
|
|
1175
|
+
/**
|
|
1176
|
+
* Clears the parsed vector cache. Useful for mid-loop flush guarantees
|
|
1177
|
+
* or memory pressure evictions.
|
|
1178
|
+
*/
|
|
1179
|
+
evictCache(entityId) {
|
|
1180
|
+
if (entityId) {
|
|
1181
|
+
this.vectorCache.delete(entityId);
|
|
1025
1182
|
} else {
|
|
1026
|
-
this.
|
|
1183
|
+
this.vectorCache.clear();
|
|
1027
1184
|
}
|
|
1028
|
-
this._notifyStatusSubscribers(entityId);
|
|
1029
1185
|
}
|
|
1030
1186
|
/**
|
|
1031
|
-
*
|
|
1032
|
-
* Use for non-throwing conflict checks (e.g. auto-trigger gating in write()).
|
|
1187
|
+
* Fully resets the search service.
|
|
1033
1188
|
*/
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
case "ontologyBackfill":
|
|
1040
|
-
return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1041
|
-
case "prune":
|
|
1042
|
-
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1043
|
-
case "reembed":
|
|
1044
|
-
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey()) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1045
|
-
case "global_reembed":
|
|
1046
|
-
return this.activeMaintenanceJobs.has(this._globalReembedKey()) || this._isAnyMaintenanceActiveWithSuffix(":reembed") || this._isAnyMaintenanceActiveWithSuffix(":prune") || this._isAnyMaintenanceActiveWithSuffix(":librarian") || this._isAnyMaintenanceActiveWithSuffix(":heal") || this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill") || this.activeIngestJobs.size > 0 || this._isAnyMaintenanceActiveWithSuffix(":import") || this._isAnyMaintenanceActiveWithSuffix(":forget");
|
|
1047
|
-
case "import":
|
|
1048
|
-
case "forget": {
|
|
1049
|
-
const selfKey = operation === "import" ? this._importKey(entityId) : this._forgetKey(entityId);
|
|
1050
|
-
return this.activeMaintenanceJobs.has(selfKey) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1051
|
-
}
|
|
1052
|
-
case "global_import":
|
|
1053
|
-
return this.activeMaintenanceJobs.has(this._globalImportKey());
|
|
1054
|
-
case "ingest":
|
|
1055
|
-
return this._hasIngestJob(entityId) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1056
|
-
default:
|
|
1057
|
-
return false;
|
|
1058
|
-
}
|
|
1189
|
+
async clearAll() {
|
|
1190
|
+
this.vectorCache.clear();
|
|
1191
|
+
await this.indexStrategy.replaceAll([]);
|
|
1192
|
+
this.staleEntities.clear();
|
|
1193
|
+
this.staleEpochs.clear();
|
|
1059
1194
|
}
|
|
1060
1195
|
/**
|
|
1061
|
-
*
|
|
1062
|
-
* permissiveness for write() auto-trigger paths — stricter checks stay in
|
|
1063
|
-
* acquireLock() — with one exception: reembed sweeps, which are excluded
|
|
1064
|
-
* structurally rather than by accident (see below).
|
|
1196
|
+
* Executes a keyword search against the active index strategy.
|
|
1065
1197
|
*/
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
if (this.activeMaintenanceJobs.has(healKey)) return false;
|
|
1070
|
-
this.activeMaintenanceJobs.add(healKey);
|
|
1071
|
-
this._notifyStatusSubscribers(entityId);
|
|
1072
|
-
return true;
|
|
1198
|
+
async searchKeyword(query, entityIds, limit) {
|
|
1199
|
+
const results = finiteResults(await this.indexStrategy.search(query, { entityIds, limit }));
|
|
1200
|
+
return results.sort((a, b) => this._compareSearchResults(a, b)).slice(0, limit);
|
|
1073
1201
|
}
|
|
1074
1202
|
/**
|
|
1075
|
-
*
|
|
1076
|
-
* (FIFO across callers racing the same hash), then the synchronous
|
|
1077
|
-
* sourceRef ingest lock (rejects cross-entity conflicts and busy
|
|
1078
|
-
* operations). Returns a single zero-argument release closure that
|
|
1079
|
-
* unwinds in the opposite order — sourceRef first, hash second — so
|
|
1080
|
-
* the sourceRef lock is freed the instant its work is done and only
|
|
1081
|
-
* the hash lock keeps serializing the duplicate-content race window.
|
|
1082
|
-
*
|
|
1083
|
-
* On a sourceRef conflict (WikiBusyError thrown synchronously by the
|
|
1084
|
-
* underlying acquireLock) the hash lock is released BEFORE the error
|
|
1085
|
-
* propagates, so the next caller in the hash chain is not held up by
|
|
1086
|
-
* a request that never reached the DB.
|
|
1203
|
+
* Pre-fetches keyword scores for candidate hydration, used during hybrid weighting.
|
|
1087
1204
|
*/
|
|
1088
|
-
async
|
|
1089
|
-
const
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
} catch (err) {
|
|
1095
|
-
releaseHash();
|
|
1096
|
-
throw err;
|
|
1097
|
-
}
|
|
1098
|
-
return () => {
|
|
1099
|
-
try {
|
|
1100
|
-
if (sourceLockAcquired) this.releaseLock("ingest", entityId, sourceRef);
|
|
1101
|
-
} finally {
|
|
1102
|
-
releaseHash();
|
|
1103
|
-
}
|
|
1104
|
-
};
|
|
1205
|
+
async getKeywordScores(query, entityIds, preFilterLimit) {
|
|
1206
|
+
const opts = preFilterLimit !== void 0 ? { entityIds, limit: preFilterLimit, preFilterLimit } : { entityIds, limit: Number.MAX_SAFE_INTEGER };
|
|
1207
|
+
const results = finiteResults(await this.indexStrategy.search(query, opts));
|
|
1208
|
+
if (results.length === 0) return /* @__PURE__ */ new Map();
|
|
1209
|
+
const maxScore = Math.max(1, results[0]?.score ?? 1);
|
|
1210
|
+
return new Map(results.map((r) => [r.id, r.score / maxScore]));
|
|
1105
1211
|
}
|
|
1106
1212
|
/**
|
|
1107
|
-
*
|
|
1108
|
-
*
|
|
1213
|
+
* Score candidate rows using in-process JS cosine similarity.
|
|
1214
|
+
* Applies hybrid blending (if weight set) and tie-break sorting before returning.
|
|
1109
1215
|
*/
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);
|
|
1119
|
-
if (this._isForgetActiveFor(entityId)) throw new WikiBusyError("forget", entityId);
|
|
1216
|
+
async rankSemantic(args) {
|
|
1217
|
+
const queryVec = args.queryVec instanceof Float32Array ? args.queryVec.slice() : Array.from(args.queryVec);
|
|
1218
|
+
const { entityId, candidateRows, weight, miniSearchScores, populateCache, limit, skipSort } = args;
|
|
1219
|
+
let entityCache = this.vectorCache.get(entityId);
|
|
1220
|
+
const tooLarge = populateCache && candidateRows.length > _SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY;
|
|
1221
|
+
if (tooLarge && entityCache) {
|
|
1222
|
+
this.vectorCache.delete(entityId);
|
|
1223
|
+
entityCache = void 0;
|
|
1120
1224
|
}
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
this.activeMaintenanceJobs.add(this._importKey(entityId));
|
|
1225
|
+
const canCache = populateCache && !tooLarge;
|
|
1226
|
+
if (canCache && !entityCache) {
|
|
1227
|
+
entityCache = /* @__PURE__ */ new Map();
|
|
1125
1228
|
}
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1229
|
+
const scored = candidateRows.map((row) => {
|
|
1230
|
+
let vector = entityCache?.get(row.id) ?? parseEmbedding(row.embedding_blob, row.embedding);
|
|
1231
|
+
if (vector && canCache && entityCache && !entityCache.has(row.id)) {
|
|
1232
|
+
entityCache.set(row.id, vector);
|
|
1233
|
+
}
|
|
1234
|
+
let score = 0;
|
|
1235
|
+
if (vector && vector.length === queryVec.length) {
|
|
1236
|
+
const cosSim = cosineSimilarity(queryVec, vector);
|
|
1237
|
+
if (weight !== void 0) {
|
|
1238
|
+
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
1239
|
+
score = weight * Math.max(0, cosSim) + (1 - weight) * kwScore;
|
|
1240
|
+
} else {
|
|
1241
|
+
score = cosSim;
|
|
1242
|
+
}
|
|
1243
|
+
} else if (weight !== void 0 && weight < 1) {
|
|
1244
|
+
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
1245
|
+
score = (1 - weight) * kwScore;
|
|
1246
|
+
} else {
|
|
1247
|
+
score = -2;
|
|
1248
|
+
}
|
|
1249
|
+
return {
|
|
1250
|
+
id: row.id,
|
|
1251
|
+
entity_id: row.entity_id,
|
|
1252
|
+
score,
|
|
1253
|
+
updated_at: row.updated_at,
|
|
1254
|
+
access_count: row.access_count
|
|
1255
|
+
};
|
|
1256
|
+
});
|
|
1257
|
+
if (canCache && entityCache && entityCache.size > 0) {
|
|
1258
|
+
if (!this.vectorCache.has(entityId)) {
|
|
1259
|
+
if (this.vectorCache.size >= _SearchService.MAX_VECTOR_CACHE_ENTITIES) {
|
|
1260
|
+
const oldestKey = this.vectorCache.keys().next().value;
|
|
1261
|
+
if (oldestKey !== void 0) this.vectorCache.delete(oldestKey);
|
|
1262
|
+
}
|
|
1263
|
+
this.vectorCache.set(entityId, entityCache);
|
|
1264
|
+
}
|
|
1131
1265
|
}
|
|
1266
|
+
if (!skipSort) {
|
|
1267
|
+
this._tieBreakSort(scored);
|
|
1268
|
+
}
|
|
1269
|
+
return scored.slice(0, limit);
|
|
1132
1270
|
}
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1271
|
+
// --- Internal Index Management ---
|
|
1272
|
+
/**
|
|
1273
|
+
* Rebuild a single entity's index from scratch. Reads every row for
|
|
1274
|
+
* `entityId` from the repository (already filtered by `deleted_at IS NULL`)
|
|
1275
|
+
* and hands the lot to `indexStrategy.replaceEntity`, which owns the
|
|
1276
|
+
* drop-then-add bookkeeping so callers don't need to know the strategy's
|
|
1277
|
+
* id-tracking shape.
|
|
1278
|
+
*/
|
|
1279
|
+
async rebuildIndex(entityId) {
|
|
1280
|
+
const rows = await this.entryRepo.findMiniSearchRows(entityId);
|
|
1281
|
+
await this.indexStrategy.replaceEntity(entityId, rows.map(toIndexDoc));
|
|
1139
1282
|
}
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1283
|
+
_tieBreakSort(items) {
|
|
1284
|
+
items.sort((a, b) => this._compareScoredRows(a, b));
|
|
1285
|
+
}
|
|
1286
|
+
_compareScoredRows(a, b) {
|
|
1287
|
+
const scoreDiff = b.score - a.score;
|
|
1288
|
+
if (!Number.isNaN(scoreDiff) && scoreDiff !== 0) return scoreDiff;
|
|
1289
|
+
const accessCountDiff = (b.access_count ?? 0) - (a.access_count ?? 0);
|
|
1290
|
+
if (accessCountDiff !== 0) return accessCountDiff;
|
|
1291
|
+
const updatedAtDiff = (b.updated_at ?? 0) - (a.updated_at ?? 0);
|
|
1292
|
+
if (updatedAtDiff !== 0) return updatedAtDiff;
|
|
1293
|
+
return a.id.localeCompare(b.id);
|
|
1294
|
+
}
|
|
1295
|
+
/**
|
|
1296
|
+
* The keyword index breaks equal-score ties by internal insertion order,
|
|
1297
|
+
* which syncEntries changes: a discarded-and-re-added id moves to the end.
|
|
1298
|
+
* Every caller truncates these results to a limit, so which ids survive a
|
|
1299
|
+
* tie at the boundary must not depend on write history. Re-sort exact
|
|
1300
|
+
* score ties by id — the same final tie-break _compareScoredRows applies.
|
|
1301
|
+
*/
|
|
1302
|
+
_compareSearchResults(a, b) {
|
|
1303
|
+
const scoreDiff = b.score - a.score;
|
|
1304
|
+
if (!Number.isNaN(scoreDiff) && scoreDiff !== 0) return scoreDiff;
|
|
1305
|
+
return a.id.localeCompare(b.id);
|
|
1306
|
+
}
|
|
1307
|
+
};
|
|
1308
|
+
/**
|
|
1309
|
+
* Maximum number of entities whose parsed embedding vectors are held in
|
|
1310
|
+
* memory. This cap is intentionally conservative so the cache remains safe
|
|
1311
|
+
* on memory-constrained runtimes (e.g., mobile/Expo).
|
|
1312
|
+
*/
|
|
1313
|
+
_SearchService.MAX_VECTOR_CACHE_ENTITIES = 16;
|
|
1314
|
+
/**
|
|
1315
|
+
* Maximum number of fact vectors cached per entity. Keep this high enough to
|
|
1316
|
+
* preserve the parsed-embedding reuse optimization for common mid-sized
|
|
1317
|
+
* entities while still maintaining a bounded memory footprint.
|
|
1318
|
+
*/
|
|
1319
|
+
_SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY = 500;
|
|
1320
|
+
var SearchService = _SearchService;
|
|
1321
|
+
function finiteResults(results) {
|
|
1322
|
+
return results.every((r) => Number.isFinite(r.score)) ? results : results.filter((r) => Number.isFinite(r.score));
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
// src/services/JobManager.ts
|
|
1326
|
+
var JobManager = class {
|
|
1327
|
+
constructor(prefix) {
|
|
1328
|
+
this.prefix = prefix;
|
|
1329
|
+
this.activeMaintenanceJobs = /* @__PURE__ */ new Set();
|
|
1330
|
+
this.activeIngestJobs = /* @__PURE__ */ new Map();
|
|
1331
|
+
/**
|
|
1332
|
+
* Per-(entityId, sourceHash) promise-chain lock. Serializes ingest races
|
|
1333
|
+
* at the application level (the v9 partial UNIQUE index enforces it at
|
|
1334
|
+
* the DB level as defense-in-depth). Two callers holding the same hash
|
|
1335
|
+
* see FIFO order; different hashes never block one another.
|
|
1336
|
+
*
|
|
1337
|
+
* Keyed `${entityId}\0${sourceHash}` so distinct (entityId, sourceHash)
|
|
1338
|
+
* pairs can never share a lock. A raw concatenation would let
|
|
1339
|
+
* ("ab", "c…") collide with ("a", "bc…"); the NUL separator makes that
|
|
1340
|
+
* impossible because neither component can contain NUL (entityIds are
|
|
1341
|
+
* user-supplied identifiers; sourceHashes are hex digests).
|
|
1342
|
+
*/
|
|
1343
|
+
this.hashLocks = /* @__PURE__ */ new Map();
|
|
1344
|
+
this.statusSubscribers = /* @__PURE__ */ new Map();
|
|
1345
|
+
/**
|
|
1346
|
+
* Lookup table for acquireLock/releaseLock's dynamic-dispatch branch.
|
|
1347
|
+
* Excludes 'ingest' | 'global_reembed' | 'global_import', which those
|
|
1348
|
+
* methods already handle via explicit if/else branches before reaching
|
|
1349
|
+
* this table.
|
|
1350
|
+
*/
|
|
1351
|
+
this.lockKeyFns = {
|
|
1352
|
+
prune: (id) => this._pruneKey(id),
|
|
1353
|
+
librarian: (id) => this._librarianKey(id),
|
|
1354
|
+
heal: (id) => this._healKey(id),
|
|
1355
|
+
reembed: (id) => this._reembedKey(id),
|
|
1356
|
+
import: (id) => this._importKey(id),
|
|
1357
|
+
forget: (id) => this._forgetKey(id),
|
|
1358
|
+
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
1162
1359
|
};
|
|
1163
1360
|
}
|
|
1164
|
-
|
|
1165
|
-
return {
|
|
1361
|
+
_hashLockKey(entityId, sourceHash) {
|
|
1362
|
+
return `${entityId}\0${sourceHash}`;
|
|
1166
1363
|
}
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1364
|
+
/**
|
|
1365
|
+
* Acquire a promise-chain lock scoped to one `(entityId, sourceHash)` pair.
|
|
1366
|
+
* Returns a zero-argument release closure that removes the current tail
|
|
1367
|
+
* from the chain. Releasing is idempotent so a holder that double-fires
|
|
1368
|
+
* (e.g. on a finally that races an explicit release) cannot unblock
|
|
1369
|
+
* the next holder prematurely.
|
|
1370
|
+
*
|
|
1371
|
+
* Documented ordering: ingest acquires the hash lock FIRST, then the
|
|
1372
|
+
* synchronous sourceRef ingest lock, and the release closure unwinds in
|
|
1373
|
+
* the opposite order (sourceRef first, hash second). Hash-then-sourceRef
|
|
1374
|
+
* is required because the v9 partial UNIQUE index conflicts on the
|
|
1375
|
+
* (entity_id, source_hash) pair regardless of source_ref, so two callers
|
|
1376
|
+
* with DIFFERENT source_refs racing the same hash are the exact case the
|
|
1377
|
+
* TOCTOU race fix needs to serialize.
|
|
1378
|
+
*
|
|
1379
|
+
* Mechanics: each holder owns a private `released` promise that resolves
|
|
1380
|
+
* only when THIS holder calls release(). The map stores the holder's
|
|
1381
|
+
* `released` promise as the "tail" — the NEXT caller awaits that tail
|
|
1382
|
+
* via `previous = this.hashLocks.get(key)`. The current caller gets
|
|
1383
|
+
* their unique release function once `previous` (the prior holder's
|
|
1384
|
+
* release signal) settles, so each holder is guaranteed a UNIQUE release
|
|
1385
|
+
* closure bound to their own `released` promise.
|
|
1386
|
+
*/
|
|
1387
|
+
acquireHashLock(entityId, sourceHash) {
|
|
1388
|
+
const key = this._hashLockKey(entityId, sourceHash);
|
|
1389
|
+
const previous = this.hashLocks.get(key) ?? Promise.resolve();
|
|
1390
|
+
let resolveReleased;
|
|
1391
|
+
const released = new Promise((resolve) => {
|
|
1392
|
+
resolveReleased = resolve;
|
|
1393
|
+
});
|
|
1394
|
+
let fired = false;
|
|
1395
|
+
const release = () => {
|
|
1396
|
+
if (fired) return;
|
|
1397
|
+
fired = true;
|
|
1398
|
+
if (this.hashLocks.get(key) === released) {
|
|
1399
|
+
this.hashLocks.delete(key);
|
|
1182
1400
|
}
|
|
1183
|
-
|
|
1401
|
+
resolveReleased();
|
|
1402
|
+
};
|
|
1403
|
+
this.hashLocks.set(key, released);
|
|
1404
|
+
return previous.then(() => release);
|
|
1184
1405
|
}
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
// src/utils/pure.ts
|
|
1188
|
-
var MAX_REPAIR_CANDIDATES = 5;
|
|
1189
|
-
function throwNoJsonFound(text, start) {
|
|
1190
|
-
throw new WikiParseError(
|
|
1191
|
-
"No JSON object/array found in LLM response",
|
|
1192
|
-
{ tier: "strict", position: start, slice: text }
|
|
1193
|
-
);
|
|
1194
|
-
}
|
|
1195
|
-
function parseJsonResponse(text) {
|
|
1196
|
-
const firstBrace = text.indexOf("{");
|
|
1197
|
-
const firstBracket = text.indexOf("[");
|
|
1198
|
-
let start;
|
|
1199
|
-
let openChar;
|
|
1200
|
-
const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
|
|
1201
|
-
if (useBrace) {
|
|
1202
|
-
start = firstBrace;
|
|
1203
|
-
openChar = "{";
|
|
1204
|
-
} else if (firstBracket !== -1) {
|
|
1205
|
-
start = firstBracket;
|
|
1206
|
-
openChar = "[";
|
|
1207
|
-
} else {
|
|
1208
|
-
throwNoJsonFound(text, null);
|
|
1406
|
+
_pruneKey(entityId) {
|
|
1407
|
+
return `${this.prefix}:${entityId}:prune`;
|
|
1209
1408
|
}
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
try {
|
|
1213
|
-
return JSON.parse(slice);
|
|
1214
|
-
} catch {
|
|
1215
|
-
}
|
|
1409
|
+
_reembedKey(entityId) {
|
|
1410
|
+
return `${this.prefix}:${entityId}:reembed`;
|
|
1216
1411
|
}
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
return repairOdd.success;
|
|
1412
|
+
_globalReembedKey() {
|
|
1413
|
+
return `${this.prefix}:reembed`;
|
|
1220
1414
|
}
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
|
|
1224
|
-
return pick.success;
|
|
1415
|
+
_importKey(entityId) {
|
|
1416
|
+
return `${this.prefix}:${entityId}:import`;
|
|
1225
1417
|
}
|
|
1226
|
-
|
|
1227
|
-
return
|
|
1418
|
+
_globalImportKey() {
|
|
1419
|
+
return `${this.prefix}:import`;
|
|
1228
1420
|
}
|
|
1229
|
-
|
|
1230
|
-
return
|
|
1421
|
+
_forgetKey(entityId) {
|
|
1422
|
+
return `${this.prefix}:${entityId}:forget`;
|
|
1231
1423
|
}
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
throw new WikiParseError(
|
|
1235
|
-
`Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
|
|
1236
|
-
{ tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
|
|
1237
|
-
);
|
|
1424
|
+
_librarianKey(entityId) {
|
|
1425
|
+
return `${this.prefix}:${entityId}:librarian`;
|
|
1238
1426
|
}
|
|
1239
|
-
|
|
1240
|
-
|
|
1427
|
+
_healKey(entityId) {
|
|
1428
|
+
return `${this.prefix}:${entityId}:heal`;
|
|
1241
1429
|
}
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
{ tier: "all", position: start, slice: text }
|
|
1245
|
-
);
|
|
1246
|
-
}
|
|
1247
|
-
function scanJsonSlice(text, start, openChar) {
|
|
1248
|
-
const closeChar = openChar === "{" ? "}" : "]";
|
|
1249
|
-
let depth = 0;
|
|
1250
|
-
let inString = false;
|
|
1251
|
-
let escape = false;
|
|
1252
|
-
let end = -1;
|
|
1253
|
-
for (let i = start; i < text.length; i++) {
|
|
1254
|
-
const ch = text[i];
|
|
1255
|
-
if (escape) {
|
|
1256
|
-
escape = false;
|
|
1257
|
-
continue;
|
|
1258
|
-
}
|
|
1259
|
-
if (ch === "\\" && inString) {
|
|
1260
|
-
escape = true;
|
|
1261
|
-
continue;
|
|
1262
|
-
}
|
|
1263
|
-
if (ch === '"') {
|
|
1264
|
-
inString = !inString;
|
|
1265
|
-
continue;
|
|
1266
|
-
}
|
|
1267
|
-
if (inString) continue;
|
|
1268
|
-
if (ch === openChar) {
|
|
1269
|
-
depth++;
|
|
1270
|
-
continue;
|
|
1271
|
-
}
|
|
1272
|
-
if (ch === closeChar) {
|
|
1273
|
-
depth--;
|
|
1274
|
-
if (depth === 0) {
|
|
1275
|
-
end = i;
|
|
1276
|
-
break;
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1430
|
+
_ontologyBackfillKey(entityId) {
|
|
1431
|
+
return `${this.prefix}:${entityId}:ontologyBackfill`;
|
|
1279
1432
|
}
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
}
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
try {
|
|
1299
|
-
bestSuccess = JSON.parse(candidate);
|
|
1300
|
-
return true;
|
|
1301
|
-
} catch (err) {
|
|
1302
|
-
if (bestFailed === null) {
|
|
1303
|
-
bestFailed = {
|
|
1304
|
-
candidate,
|
|
1305
|
-
position: extractParsePosition(err),
|
|
1306
|
-
message: safeErrorToString(err)
|
|
1307
|
-
};
|
|
1308
|
-
}
|
|
1309
|
-
return false;
|
|
1310
|
-
}
|
|
1433
|
+
_isReembedActive(entityId) {
|
|
1434
|
+
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey());
|
|
1435
|
+
}
|
|
1436
|
+
/**
|
|
1437
|
+
* True while any reembed sweep is in flight — per-entity or global.
|
|
1438
|
+
*
|
|
1439
|
+
* Read-only; acquires nothing. Callers that mutate embedding failure
|
|
1440
|
+
* markers across ALL entities (the dimension-promotion clear) must gate on
|
|
1441
|
+
* this and defer, rather than resurrect rows an in-flight sweep already
|
|
1442
|
+
* classified. See spec 2026-09-05-reembed-lock-scope-design.md §2.
|
|
1443
|
+
*
|
|
1444
|
+
* The global key `${prefix}:reembed` also ends in ':reembed', so the suffix
|
|
1445
|
+
* scan alone would already match it. The explicit check is kept because it
|
|
1446
|
+
* documents the intent and survives any future change to the global key's
|
|
1447
|
+
* format.
|
|
1448
|
+
*/
|
|
1449
|
+
isAnyReembedActive() {
|
|
1450
|
+
return this.activeMaintenanceJobs.has(this._globalReembedKey()) || this._isAnyMaintenanceActiveWithSuffix(":reembed");
|
|
1311
1451
|
}
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1452
|
+
_isImportActiveFor(entityId) {
|
|
1453
|
+
return this.activeMaintenanceJobs.has(this._importKey(entityId)) || this.activeMaintenanceJobs.has(this._globalImportKey());
|
|
1454
|
+
}
|
|
1455
|
+
_isForgetActiveFor(entityId) {
|
|
1456
|
+
return this.activeMaintenanceJobs.has(this._forgetKey(entityId));
|
|
1457
|
+
}
|
|
1458
|
+
_isAnyMaintenanceActiveWithSuffix(suffix) {
|
|
1459
|
+
const entityKeyPrefix = `${this.prefix}:`;
|
|
1460
|
+
for (const k of this.activeMaintenanceJobs) {
|
|
1461
|
+
if (k.startsWith(entityKeyPrefix) && k.endsWith(suffix)) return true;
|
|
1321
1462
|
}
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1463
|
+
return false;
|
|
1464
|
+
}
|
|
1465
|
+
_hasIngestJob(entityId, sourceRef) {
|
|
1466
|
+
return this.activeIngestJobs.get(entityId)?.has(sourceRef ?? "") ?? false;
|
|
1467
|
+
}
|
|
1468
|
+
_addIngestJob(entityId, sourceRef) {
|
|
1469
|
+
const sourceKey = sourceRef ?? "";
|
|
1470
|
+
let refs = this.activeIngestJobs.get(entityId);
|
|
1471
|
+
if (!refs) {
|
|
1472
|
+
refs = /* @__PURE__ */ new Set();
|
|
1473
|
+
this.activeIngestJobs.set(entityId, refs);
|
|
1327
1474
|
}
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1475
|
+
refs.add(sourceKey);
|
|
1476
|
+
}
|
|
1477
|
+
_removeIngestJob(entityId, sourceRef) {
|
|
1478
|
+
const sourceKey = sourceRef ?? "";
|
|
1479
|
+
const refs = this.activeIngestJobs.get(entityId);
|
|
1480
|
+
if (!refs) return;
|
|
1481
|
+
refs.delete(sourceKey);
|
|
1482
|
+
if (refs.size === 0) {
|
|
1483
|
+
this.activeIngestJobs.delete(entityId);
|
|
1333
1484
|
}
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1485
|
+
}
|
|
1486
|
+
_isIngestActiveFor(entityId) {
|
|
1487
|
+
return this.activeIngestJobs.has(entityId);
|
|
1488
|
+
}
|
|
1489
|
+
acquireLock(operation, entityId, sourceRef) {
|
|
1490
|
+
let blockingOperation = null;
|
|
1491
|
+
if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) {
|
|
1492
|
+
throw new WikiBusyError("import", "*");
|
|
1341
1493
|
}
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
if (
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
out += ch;
|
|
1364
|
-
inString = true;
|
|
1365
|
-
i++;
|
|
1366
|
-
continue;
|
|
1494
|
+
switch (operation) {
|
|
1495
|
+
case "prune":
|
|
1496
|
+
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1497
|
+
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
1498
|
+
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
1499
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
1500
|
+
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
1501
|
+
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
1502
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1503
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1504
|
+
break;
|
|
1505
|
+
case "librarian":
|
|
1506
|
+
case "heal":
|
|
1507
|
+
case "ontologyBackfill": {
|
|
1508
|
+
const opKey = this.lockKeyFns[operation](entityId);
|
|
1509
|
+
if (this.activeMaintenanceJobs.has(opKey)) blockingOperation = operation;
|
|
1510
|
+
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1511
|
+
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
1512
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1513
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1514
|
+
break;
|
|
1367
1515
|
}
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1516
|
+
case "reembed":
|
|
1517
|
+
if (this.activeMaintenanceJobs.has(this._reembedKey(entityId))) blockingOperation = "reembed";
|
|
1518
|
+
else if (this.activeMaintenanceJobs.has(this._globalReembedKey())) blockingOperation = "reembed";
|
|
1519
|
+
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1520
|
+
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
1521
|
+
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
1522
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
1523
|
+
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
1524
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1525
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1526
|
+
break;
|
|
1527
|
+
case "global_reembed":
|
|
1528
|
+
if (this.activeMaintenanceJobs.has(this._globalReembedKey())) blockingOperation = "reembed";
|
|
1529
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":reembed")) blockingOperation = "reembed";
|
|
1530
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":prune")) blockingOperation = "prune";
|
|
1531
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":librarian")) blockingOperation = "librarian";
|
|
1532
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":heal")) blockingOperation = "heal";
|
|
1533
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill")) blockingOperation = "ontologyBackfill";
|
|
1534
|
+
else if (this.activeIngestJobs.size > 0) blockingOperation = "ingest";
|
|
1535
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":import")) blockingOperation = "import";
|
|
1536
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":forget")) blockingOperation = "forget";
|
|
1537
|
+
break;
|
|
1538
|
+
case "import":
|
|
1539
|
+
case "forget": {
|
|
1540
|
+
const selfKey = operation === "import" ? this._importKey(entityId) : this._forgetKey(entityId);
|
|
1541
|
+
if (this.activeMaintenanceJobs.has(selfKey)) blockingOperation = operation;
|
|
1542
|
+
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
1543
|
+
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
1544
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
1545
|
+
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1546
|
+
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
1547
|
+
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
1548
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1549
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1550
|
+
break;
|
|
1385
1551
|
}
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
}
|
|
1398
|
-
if (!inString && ch === ",") {
|
|
1399
|
-
const topFrame = stack[stack.length - 1];
|
|
1400
|
-
if (topFrame && topFrame.container === "object") {
|
|
1401
|
-
topFrame.expectKey = true;
|
|
1552
|
+
case "global_import":
|
|
1553
|
+
if (this.activeMaintenanceJobs.has(this._globalImportKey())) blockingOperation = "import";
|
|
1554
|
+
break;
|
|
1555
|
+
case "ingest": {
|
|
1556
|
+
const sourceKey = sourceRef ?? "";
|
|
1557
|
+
if (this._hasIngestJob(entityId, sourceKey)) blockingOperation = "ingest";
|
|
1558
|
+
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1559
|
+
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
1560
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1561
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1562
|
+
break;
|
|
1402
1563
|
}
|
|
1403
|
-
out += ch;
|
|
1404
|
-
i++;
|
|
1405
|
-
continue;
|
|
1406
1564
|
}
|
|
1407
|
-
if (
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
1413
|
-
out = "";
|
|
1414
|
-
}
|
|
1415
|
-
i++;
|
|
1416
|
-
continue;
|
|
1565
|
+
if (blockingOperation) {
|
|
1566
|
+
throw new WikiBusyError(
|
|
1567
|
+
blockingOperation,
|
|
1568
|
+
operation === "global_reembed" || operation === "global_import" ? "*" : entityId
|
|
1569
|
+
);
|
|
1417
1570
|
}
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
}
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
return "\\n";
|
|
1427
|
-
case 9:
|
|
1428
|
-
return "\\t";
|
|
1429
|
-
case 13:
|
|
1430
|
-
return "\\r";
|
|
1431
|
-
case 8:
|
|
1432
|
-
return "\\b";
|
|
1433
|
-
case 12:
|
|
1434
|
-
return "\\f";
|
|
1435
|
-
default:
|
|
1436
|
-
return "\\u" + code.toString(16).padStart(4, "0");
|
|
1437
|
-
}
|
|
1438
|
-
}
|
|
1439
|
-
function extractParsePosition(err) {
|
|
1440
|
-
const match = /position\s+(\d+)/.exec(safeErrorToString(err));
|
|
1441
|
-
return match ? Number(match[1]) : null;
|
|
1442
|
-
}
|
|
1443
|
-
function safeErrorToString(e) {
|
|
1444
|
-
let isErrorLike = false;
|
|
1445
|
-
try {
|
|
1446
|
-
isErrorLike = e instanceof Error;
|
|
1447
|
-
} catch {
|
|
1448
|
-
}
|
|
1449
|
-
if (isErrorLike) {
|
|
1450
|
-
const msg = readErrorField(e, "message");
|
|
1451
|
-
if (typeof msg === "string" && msg.length > 0) return msg;
|
|
1452
|
-
const name = readErrorField(e, "name");
|
|
1453
|
-
if (typeof name === "string" && name.length > 0) return name;
|
|
1454
|
-
return "[Error]";
|
|
1455
|
-
}
|
|
1456
|
-
try {
|
|
1457
|
-
return String(e);
|
|
1458
|
-
} catch {
|
|
1459
|
-
try {
|
|
1460
|
-
return Object.prototype.toString.call(e);
|
|
1461
|
-
} catch {
|
|
1462
|
-
return "[unstringifiable error]";
|
|
1571
|
+
if (operation === "ingest") {
|
|
1572
|
+
this._addIngestJob(entityId, sourceRef);
|
|
1573
|
+
} else if (operation === "global_reembed") {
|
|
1574
|
+
this.activeMaintenanceJobs.add(this._globalReembedKey());
|
|
1575
|
+
} else if (operation === "global_import") {
|
|
1576
|
+
this.activeMaintenanceJobs.add(this._globalImportKey());
|
|
1577
|
+
} else {
|
|
1578
|
+
this.activeMaintenanceJobs.add(this.lockKeyFns[operation](entityId));
|
|
1463
1579
|
}
|
|
1580
|
+
this._notifyStatusSubscribers(entityId);
|
|
1464
1581
|
}
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
}
|
|
1473
|
-
function sanitizeRankerError(err, sanitizeRankerErrors) {
|
|
1474
|
-
let isErrorLike = false;
|
|
1475
|
-
try {
|
|
1476
|
-
isErrorLike = err instanceof Error;
|
|
1477
|
-
} catch {
|
|
1478
|
-
}
|
|
1479
|
-
if (sanitizeRankerErrors === false) {
|
|
1480
|
-
return isErrorLike ? err : new Error(safeErrorToString(err));
|
|
1481
|
-
}
|
|
1482
|
-
let errLike = null;
|
|
1483
|
-
if (isErrorLike) errLike = err;
|
|
1484
|
-
let typeName;
|
|
1485
|
-
try {
|
|
1486
|
-
if (errLike) {
|
|
1487
|
-
const rawName = errLike.constructor?.name;
|
|
1488
|
-
typeName = typeof rawName === "string" ? rawName : "Error";
|
|
1582
|
+
releaseLock(operation, entityId, sourceRef) {
|
|
1583
|
+
if (operation === "ingest") {
|
|
1584
|
+
this._removeIngestJob(entityId, sourceRef);
|
|
1585
|
+
} else if (operation === "global_reembed") {
|
|
1586
|
+
this.activeMaintenanceJobs.delete(this._globalReembedKey());
|
|
1587
|
+
} else if (operation === "global_import") {
|
|
1588
|
+
this.activeMaintenanceJobs.delete(this._globalImportKey());
|
|
1489
1589
|
} else {
|
|
1490
|
-
|
|
1590
|
+
this.activeMaintenanceJobs.delete(this.lockKeyFns[operation](entityId));
|
|
1491
1591
|
}
|
|
1492
|
-
|
|
1493
|
-
typeName = isErrorLike ? "Error" : typeof err;
|
|
1592
|
+
this._notifyStatusSubscribers(entityId);
|
|
1494
1593
|
}
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1594
|
+
/**
|
|
1595
|
+
* Returns true if acquireLock(operation, entityId) would throw WikiBusyError.
|
|
1596
|
+
* Use for non-throwing conflict checks (e.g. auto-trigger gating in write()).
|
|
1597
|
+
*/
|
|
1598
|
+
isBlocked(operation, entityId) {
|
|
1599
|
+
if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) return true;
|
|
1600
|
+
switch (operation) {
|
|
1601
|
+
case "librarian":
|
|
1602
|
+
case "heal":
|
|
1603
|
+
case "ontologyBackfill":
|
|
1604
|
+
return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1605
|
+
case "prune":
|
|
1606
|
+
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1607
|
+
case "reembed":
|
|
1608
|
+
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey()) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1609
|
+
case "global_reembed":
|
|
1610
|
+
return this.activeMaintenanceJobs.has(this._globalReembedKey()) || this._isAnyMaintenanceActiveWithSuffix(":reembed") || this._isAnyMaintenanceActiveWithSuffix(":prune") || this._isAnyMaintenanceActiveWithSuffix(":librarian") || this._isAnyMaintenanceActiveWithSuffix(":heal") || this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill") || this.activeIngestJobs.size > 0 || this._isAnyMaintenanceActiveWithSuffix(":import") || this._isAnyMaintenanceActiveWithSuffix(":forget");
|
|
1611
|
+
case "import":
|
|
1612
|
+
case "forget": {
|
|
1613
|
+
const selfKey = operation === "import" ? this._importKey(entityId) : this._forgetKey(entityId);
|
|
1614
|
+
return this.activeMaintenanceJobs.has(selfKey) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1510
1615
|
}
|
|
1511
|
-
|
|
1616
|
+
case "global_import":
|
|
1617
|
+
return this.activeMaintenanceJobs.has(this._globalImportKey());
|
|
1618
|
+
case "ingest":
|
|
1619
|
+
return this._hasIngestJob(entityId) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1620
|
+
default:
|
|
1621
|
+
return false;
|
|
1512
1622
|
}
|
|
1513
1623
|
}
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
let safeEnd = end === void 0 ? length : end < 0 ? Math.max(length + end, 0) : Math.min(end, length);
|
|
1528
|
-
if (safeStart > safeEnd) {
|
|
1529
|
-
[safeStart, safeEnd] = [safeEnd, safeStart];
|
|
1530
|
-
}
|
|
1531
|
-
if (safeStart > 0 && safeStart < length && value.charCodeAt(safeStart) >= 56320 && value.charCodeAt(safeStart) <= 57343 && value.charCodeAt(safeStart - 1) >= 55296 && value.charCodeAt(safeStart - 1) <= 56319) {
|
|
1532
|
-
safeStart--;
|
|
1533
|
-
}
|
|
1534
|
-
if (safeEnd > 0 && safeEnd < length && value.charCodeAt(safeEnd - 1) >= 55296 && value.charCodeAt(safeEnd - 1) <= 56319 && value.charCodeAt(safeEnd) >= 56320 && value.charCodeAt(safeEnd) <= 57343) {
|
|
1535
|
-
safeEnd--;
|
|
1536
|
-
}
|
|
1537
|
-
return value.slice(safeStart, safeEnd);
|
|
1538
|
-
}
|
|
1539
|
-
function chunkText(input, maxChunkLength, overlap) {
|
|
1540
|
-
const text = input.trim();
|
|
1541
|
-
if (text.length === 0) return { chunks: [], truncated: false };
|
|
1542
|
-
if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
|
|
1543
|
-
throw new Error("maxChunkLength must be an integer >= 2");
|
|
1624
|
+
/**
|
|
1625
|
+
* Auto-heal historically only gated on the heal self-key. Keep that
|
|
1626
|
+
* permissiveness for write() auto-trigger paths — stricter checks stay in
|
|
1627
|
+
* acquireLock() — with one exception: reembed sweeps, which are excluded
|
|
1628
|
+
* structurally rather than by accident (see below).
|
|
1629
|
+
*/
|
|
1630
|
+
tryAcquireAutoHealLock(entityId) {
|
|
1631
|
+
if (this.isAnyReembedActive()) return false;
|
|
1632
|
+
const healKey = this._healKey(entityId);
|
|
1633
|
+
if (this.activeMaintenanceJobs.has(healKey)) return false;
|
|
1634
|
+
this.activeMaintenanceJobs.add(healKey);
|
|
1635
|
+
this._notifyStatusSubscribers(entityId);
|
|
1636
|
+
return true;
|
|
1544
1637
|
}
|
|
1545
|
-
|
|
1546
|
-
|
|
1638
|
+
/**
|
|
1639
|
+
* Centralized ingest lock acquisition. Acquires the hash lock FIRST
|
|
1640
|
+
* (FIFO across callers racing the same hash), then the synchronous
|
|
1641
|
+
* sourceRef ingest lock (rejects cross-entity conflicts and busy
|
|
1642
|
+
* operations). Returns a single zero-argument release closure that
|
|
1643
|
+
* unwinds in the opposite order — sourceRef first, hash second — so
|
|
1644
|
+
* the sourceRef lock is freed the instant its work is done and only
|
|
1645
|
+
* the hash lock keeps serializing the duplicate-content race window.
|
|
1646
|
+
*
|
|
1647
|
+
* On a sourceRef conflict (WikiBusyError thrown synchronously by the
|
|
1648
|
+
* underlying acquireLock) the hash lock is released BEFORE the error
|
|
1649
|
+
* propagates, so the next caller in the hash chain is not held up by
|
|
1650
|
+
* a request that never reached the DB.
|
|
1651
|
+
*/
|
|
1652
|
+
async acquireIngestLocks(entityId, sourceRef, sourceHash) {
|
|
1653
|
+
const releaseHash = await this.acquireHashLock(entityId, sourceHash);
|
|
1654
|
+
let sourceLockAcquired = false;
|
|
1655
|
+
try {
|
|
1656
|
+
this.acquireLock("ingest", entityId, sourceRef);
|
|
1657
|
+
sourceLockAcquired = true;
|
|
1658
|
+
} catch (err) {
|
|
1659
|
+
releaseHash();
|
|
1660
|
+
throw err;
|
|
1661
|
+
}
|
|
1662
|
+
return () => {
|
|
1663
|
+
try {
|
|
1664
|
+
if (sourceLockAcquired) this.releaseLock("ingest", entityId, sourceRef);
|
|
1665
|
+
} finally {
|
|
1666
|
+
releaseHash();
|
|
1667
|
+
}
|
|
1668
|
+
};
|
|
1547
1669
|
}
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
const
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1670
|
+
/**
|
|
1671
|
+
* Validates then acquires global + per-entity import locks atomically.
|
|
1672
|
+
* Validates all entities before acquiring any lock (same as current importDump semantics).
|
|
1673
|
+
*/
|
|
1674
|
+
acquireImportLocks(entityIds) {
|
|
1675
|
+
for (const entityId of entityIds) {
|
|
1676
|
+
if (this.activeMaintenanceJobs.has(this._importKey(entityId))) throw new WikiBusyError("import", entityId);
|
|
1677
|
+
if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) throw new WikiBusyError("librarian", entityId);
|
|
1678
|
+
if (this.activeMaintenanceJobs.has(this._healKey(entityId))) throw new WikiBusyError("heal", entityId);
|
|
1679
|
+
if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) throw new WikiBusyError("ontologyBackfill", entityId);
|
|
1680
|
+
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) throw new WikiBusyError("prune", entityId);
|
|
1681
|
+
if (this._isReembedActive(entityId)) throw new WikiBusyError("reembed", entityId);
|
|
1682
|
+
if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);
|
|
1683
|
+
if (this._isForgetActiveFor(entityId)) throw new WikiBusyError("forget", entityId);
|
|
1557
1684
|
}
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
if (paraIdx >= minSplit && paraIdx + 2 <= windowEnd) {
|
|
1563
|
-
splitPoint = paraIdx + 2;
|
|
1685
|
+
if (this.activeMaintenanceJobs.has(this._globalImportKey())) throw new WikiBusyError("import", "*");
|
|
1686
|
+
this.activeMaintenanceJobs.add(this._globalImportKey());
|
|
1687
|
+
for (const entityId of entityIds) {
|
|
1688
|
+
this.activeMaintenanceJobs.add(this._importKey(entityId));
|
|
1564
1689
|
}
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
lastTerm = i + 2;
|
|
1571
|
-
}
|
|
1572
|
-
}
|
|
1573
|
-
if (lastTerm !== -1 && lastTerm <= windowEnd) splitPoint = lastTerm;
|
|
1690
|
+
}
|
|
1691
|
+
releaseImportLocks(entityIds) {
|
|
1692
|
+
this.activeMaintenanceJobs.delete(this._globalImportKey());
|
|
1693
|
+
for (const entityId of entityIds) {
|
|
1694
|
+
this.activeMaintenanceJobs.delete(this._importKey(entityId));
|
|
1574
1695
|
}
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1696
|
+
}
|
|
1697
|
+
getEntityStatus(entityId) {
|
|
1698
|
+
return {
|
|
1699
|
+
ingesting: this._isIngestActiveFor(entityId),
|
|
1700
|
+
librarian: this.activeMaintenanceJobs.has(this._librarianKey(entityId)),
|
|
1701
|
+
heal: this.activeMaintenanceJobs.has(this._healKey(entityId))
|
|
1702
|
+
};
|
|
1703
|
+
}
|
|
1704
|
+
subscribeEntityStatus(entityId, callback) {
|
|
1705
|
+
const initial = this.getEntityStatus(entityId);
|
|
1706
|
+
let set = this.statusSubscribers.get(entityId);
|
|
1707
|
+
if (!set) {
|
|
1708
|
+
set = /* @__PURE__ */ new Set();
|
|
1709
|
+
this.statusSubscribers.set(entityId, set);
|
|
1582
1710
|
}
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1711
|
+
const entry = { callback, last: this._copyEntityStatus(initial) };
|
|
1712
|
+
set.add(entry);
|
|
1713
|
+
try {
|
|
1714
|
+
callback(this._copyEntityStatus(initial));
|
|
1715
|
+
} catch (err) {
|
|
1716
|
+
console.error(`[JobManager] callback error for entityId="${entityId}" during initial emission`, err);
|
|
1586
1717
|
}
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1718
|
+
let active = true;
|
|
1719
|
+
return () => {
|
|
1720
|
+
if (!active) return;
|
|
1721
|
+
active = false;
|
|
1722
|
+
const s = this.statusSubscribers.get(entityId);
|
|
1723
|
+
if (!s) return;
|
|
1724
|
+
s.delete(entry);
|
|
1725
|
+
if (s.size === 0) this.statusSubscribers.delete(entityId);
|
|
1726
|
+
};
|
|
1727
|
+
}
|
|
1728
|
+
_copyEntityStatus(s) {
|
|
1729
|
+
return { ingesting: s.ingesting, librarian: s.librarian, heal: s.heal };
|
|
1590
1730
|
}
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
|
|
1731
|
+
_notifyStatusSubscribers(entityId) {
|
|
1732
|
+
if (entityId === "*") return;
|
|
1733
|
+
const set = this.statusSubscribers.get(entityId);
|
|
1734
|
+
if (!set || set.size === 0) return;
|
|
1735
|
+
for (const entry of Array.from(set)) {
|
|
1736
|
+
if (!set.has(entry)) continue;
|
|
1737
|
+
const next = this.getEntityStatus(entityId);
|
|
1738
|
+
if (entry.last.ingesting === next.ingesting && entry.last.librarian === next.librarian && entry.last.heal === next.heal) {
|
|
1739
|
+
continue;
|
|
1740
|
+
}
|
|
1741
|
+
entry.last = this._copyEntityStatus(next);
|
|
1601
1742
|
try {
|
|
1602
|
-
|
|
1603
|
-
} catch (
|
|
1604
|
-
|
|
1605
|
-
failed = true;
|
|
1606
|
-
firstError = e;
|
|
1607
|
-
}
|
|
1608
|
-
return;
|
|
1743
|
+
entry.callback(this._copyEntityStatus(next));
|
|
1744
|
+
} catch (err) {
|
|
1745
|
+
console.error(`[JobManager] callback error for entityId="${entityId}" during transition emission`, err);
|
|
1609
1746
|
}
|
|
1610
1747
|
}
|
|
1611
1748
|
}
|
|
1612
|
-
|
|
1613
|
-
await Promise.allSettled(Array.from({ length: workerCount }, worker));
|
|
1614
|
-
if (failed) throw firstError;
|
|
1615
|
-
return results;
|
|
1616
|
-
}
|
|
1617
|
-
function clip(value, max) {
|
|
1618
|
-
if (typeof value !== "string") return "";
|
|
1619
|
-
const s = value.trim();
|
|
1620
|
-
return s.length <= max ? s : safeSlice(s, 0, max).trimEnd();
|
|
1621
|
-
}
|
|
1622
|
-
function validateTags(tags) {
|
|
1623
|
-
if (!Array.isArray(tags)) return [];
|
|
1624
|
-
return tags.filter((t) => typeof t === "string").map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0 && t.length <= 40).slice(0, 6);
|
|
1625
|
-
}
|
|
1626
|
-
var MAX_EVIDENCE_QUOTES = 10;
|
|
1627
|
-
function normalizeEvidence(raw) {
|
|
1628
|
-
if (!Array.isArray(raw)) return void 0;
|
|
1629
|
-
const out = [];
|
|
1630
|
-
for (const entry of raw) {
|
|
1631
|
-
if (typeof entry !== "string") continue;
|
|
1632
|
-
const trimmed = entry.trim();
|
|
1633
|
-
if (!trimmed) continue;
|
|
1634
|
-
out.push(trimmed);
|
|
1635
|
-
if (out.length > MAX_EVIDENCE_QUOTES) break;
|
|
1636
|
-
}
|
|
1637
|
-
return out;
|
|
1638
|
-
}
|
|
1639
|
-
function validateFact(fact) {
|
|
1640
|
-
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
|
|
1641
|
-
const title = clip(fact.title, 80);
|
|
1642
|
-
const body = clip(fact.body, 800);
|
|
1643
|
-
if (!title || !body) return null;
|
|
1644
|
-
let confidence = fact.confidence;
|
|
1645
|
-
if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
|
|
1646
|
-
const valid = {
|
|
1647
|
-
...fact,
|
|
1648
|
-
title,
|
|
1649
|
-
body,
|
|
1650
|
-
confidence,
|
|
1651
|
-
tags: validateTags(fact.tags)
|
|
1652
|
-
};
|
|
1653
|
-
const evidence = normalizeEvidence(fact.evidence);
|
|
1654
|
-
if (evidence) valid.evidence = evidence;
|
|
1655
|
-
else delete valid.evidence;
|
|
1656
|
-
return valid;
|
|
1657
|
-
}
|
|
1658
|
-
function validateTask(task) {
|
|
1659
|
-
if (typeof task?.description !== "string") return null;
|
|
1660
|
-
const description = clip(task.description, 200);
|
|
1661
|
-
if (!description) return null;
|
|
1662
|
-
let priority = task.priority;
|
|
1663
|
-
if (typeof priority !== "number" || !isFinite(priority)) priority = 0;
|
|
1664
|
-
priority = Math.max(0, Math.min(10, Math.round(priority)));
|
|
1665
|
-
return {
|
|
1666
|
-
...task,
|
|
1667
|
-
description,
|
|
1668
|
-
priority
|
|
1669
|
-
};
|
|
1670
|
-
}
|
|
1671
|
-
function factRejectionReason(raw) {
|
|
1672
|
-
const fact = raw;
|
|
1673
|
-
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return "invalid_shape";
|
|
1674
|
-
if (!clip(fact.title, 80)) return "missing_title";
|
|
1675
|
-
return "missing_body";
|
|
1676
|
-
}
|
|
1677
|
-
function taskRejectionReason(raw) {
|
|
1678
|
-
const task = raw;
|
|
1679
|
-
if (typeof task?.description !== "string") return "invalid_shape";
|
|
1680
|
-
return "missing_description";
|
|
1681
|
-
}
|
|
1682
|
-
function normalizeSourceRef(value) {
|
|
1683
|
-
if (typeof value !== "string") return null;
|
|
1684
|
-
const cleaned = value.replace(/[^A-Za-z0-9._\- ]/g, "").trim().slice(0, 255);
|
|
1685
|
-
return cleaned.length > 0 ? cleaned : null;
|
|
1686
|
-
}
|
|
1687
|
-
function normalizeSourceHash(value) {
|
|
1688
|
-
if (typeof value !== "string") return null;
|
|
1689
|
-
return /^[0-9a-f]{64}$/i.test(value) ? value.toLowerCase() : null;
|
|
1690
|
-
}
|
|
1691
|
-
function titleTokens(title) {
|
|
1692
|
-
return new Set(title.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3));
|
|
1693
|
-
}
|
|
1694
|
-
function jaccardScore(a, b) {
|
|
1695
|
-
if (a.size === 0 && b.size === 0) return 0;
|
|
1696
|
-
const intersection = new Set([...a].filter((x) => b.has(x)));
|
|
1697
|
-
const union = /* @__PURE__ */ new Set([...a, ...b]);
|
|
1698
|
-
return intersection.size / union.size;
|
|
1699
|
-
}
|
|
1749
|
+
};
|
|
1700
1750
|
|
|
1701
1751
|
// src/prompts.ts
|
|
1702
1752
|
var LIBRARIAN_SYSTEM_PROMPT = `You are a knowledge extraction agent. Your job is to analyze recent episodic events and extract stable facts and actionable tasks about the user or entity.
|
|
@@ -1843,12 +1893,24 @@ ${groundingEvidenceBlock(writer, cfg)}` : systemPrompt;
|
|
|
1843
1893
|
hasOntologyPlaceholders(template) {
|
|
1844
1894
|
return /\{\{\s*ontology(?:Manifest|ModeInstructions)\s*\}\}/.test(template);
|
|
1845
1895
|
}
|
|
1896
|
+
/** Removes ontology placeholders until none remain, so a nested form cannot reassemble one (closes CodeQL #11). */
|
|
1897
|
+
stripOntologyPlaceholders(template) {
|
|
1898
|
+
let out = template;
|
|
1899
|
+
let prev;
|
|
1900
|
+
do {
|
|
1901
|
+
prev = out;
|
|
1902
|
+
out = out.replace(/\{\{\s*ontology(?:Manifest|ModeInstructions)\s*\}\}/g, "");
|
|
1903
|
+
} while (out !== prev);
|
|
1904
|
+
return out;
|
|
1905
|
+
}
|
|
1846
1906
|
buildSystemPrompt(template, variables, ontologyContext) {
|
|
1907
|
+
const hasOntologyPlaceholders = this.hasOntologyPlaceholders(template);
|
|
1908
|
+
const effective = hasOntologyPlaceholders && ontologyContext == null ? this.stripOntologyPlaceholders(template) : template;
|
|
1847
1909
|
const shouldHydrate = Object.keys(variables).some(
|
|
1848
|
-
(key) => new RegExp(`\\{\\{\\s*${key}\\s*\\}\\}`).test(
|
|
1849
|
-
) || ontologyContext != null &&
|
|
1850
|
-
const hydrated = shouldHydrate ? this.hydrate(
|
|
1851
|
-
return
|
|
1910
|
+
(key) => new RegExp(`\\{\\{\\s*${key}\\s*\\}\\}`).test(effective)
|
|
1911
|
+
) || ontologyContext != null && hasOntologyPlaceholders;
|
|
1912
|
+
const hydrated = shouldHydrate ? this.hydrate(effective, { ...variables, ...ontologyContext ?? {} }) : effective;
|
|
1913
|
+
return hasOntologyPlaceholders ? hydrated : this.appendOntology(hydrated, ontologyContext);
|
|
1852
1914
|
}
|
|
1853
1915
|
appendOntology(systemPrompt, ctx) {
|
|
1854
1916
|
if (!ctx) return systemPrompt;
|
|
@@ -2104,7 +2166,7 @@ var MESSAGE = {
|
|
|
2104
2166
|
embedding_failed: "A fact could not be embedded.",
|
|
2105
2167
|
hook_failed: "A host hook threw or rejected.",
|
|
2106
2168
|
background_job_failed: "A background maintenance job failed.",
|
|
2107
|
-
heal_skipped: "Heal skipped a candidate fact.",
|
|
2169
|
+
heal_skipped: "Heal skipped a candidate fact, or the whole LLM pass when keyword search is unavailable.",
|
|
2108
2170
|
grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
|
|
2109
2171
|
grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
|
|
2110
2172
|
classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
|
|
@@ -2611,6 +2673,9 @@ var IngestionService = class {
|
|
|
2611
2673
|
* ordering and counts so success counts match the pre-change contract.
|
|
2612
2674
|
*/
|
|
2613
2675
|
async upsertGraphCore(entityId, params, tx, opts) {
|
|
2676
|
+
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
2677
|
+
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
2678
|
+
params = { ...params, sourceRef };
|
|
2614
2679
|
await this.assertGraphNodeOwnership(entityId, params.nodes, tx);
|
|
2615
2680
|
const now = Date.now();
|
|
2616
2681
|
const ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
@@ -3310,6 +3375,598 @@ async function runBatched(args) {
|
|
|
3310
3375
|
return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
|
|
3311
3376
|
}
|
|
3312
3377
|
|
|
3378
|
+
// src/repositories/rowMappers.ts
|
|
3379
|
+
function parseJsonArray(s, fallback) {
|
|
3380
|
+
if (Array.isArray(s)) return s;
|
|
3381
|
+
if (typeof s === "string" && s.length > 0) {
|
|
3382
|
+
try {
|
|
3383
|
+
const p = JSON.parse(s);
|
|
3384
|
+
if (Array.isArray(p)) return p;
|
|
3385
|
+
} catch {
|
|
3386
|
+
}
|
|
3387
|
+
}
|
|
3388
|
+
return fallback;
|
|
3389
|
+
}
|
|
3390
|
+
function parseJsonObject(s, fallback = null) {
|
|
3391
|
+
if (s && typeof s === "object" && !Array.isArray(s)) return s;
|
|
3392
|
+
if (typeof s === "string" && s.length > 0) {
|
|
3393
|
+
try {
|
|
3394
|
+
const p = JSON.parse(s);
|
|
3395
|
+
if (p && typeof p === "object" && !Array.isArray(p)) return p;
|
|
3396
|
+
} catch {
|
|
3397
|
+
}
|
|
3398
|
+
}
|
|
3399
|
+
return fallback;
|
|
3400
|
+
}
|
|
3401
|
+
|
|
3402
|
+
// src/db/schema.ts
|
|
3403
|
+
async function setupDatabase(db, prefix) {
|
|
3404
|
+
await db.execAsync(`
|
|
3405
|
+
CREATE TABLE IF NOT EXISTS ${prefix}entries (
|
|
3406
|
+
id TEXT PRIMARY KEY,
|
|
3407
|
+
entity_id TEXT NOT NULL,
|
|
3408
|
+
title TEXT NOT NULL,
|
|
3409
|
+
body TEXT NOT NULL,
|
|
3410
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
3411
|
+
confidence TEXT NOT NULL DEFAULT 'inferred',
|
|
3412
|
+
source_type TEXT NOT NULL DEFAULT 'librarian_inferred',
|
|
3413
|
+
source_hash TEXT,
|
|
3414
|
+
source_ref TEXT,
|
|
3415
|
+
created_at INTEGER NOT NULL,
|
|
3416
|
+
updated_at INTEGER NOT NULL,
|
|
3417
|
+
last_accessed_at INTEGER,
|
|
3418
|
+
access_count INTEGER NOT NULL DEFAULT 0,
|
|
3419
|
+
deleted_at INTEGER,
|
|
3420
|
+
embedding TEXT,
|
|
3421
|
+
embedding_blob BLOB,
|
|
3422
|
+
okf_type TEXT,
|
|
3423
|
+
ontology_checked_at INTEGER,
|
|
3424
|
+
heal_checked_at INTEGER,
|
|
3425
|
+
lifecycle_status TEXT NOT NULL DEFAULT 'stable',
|
|
3426
|
+
stale_after INTEGER,
|
|
3427
|
+
generated_by TEXT,
|
|
3428
|
+
last_verified_at INTEGER,
|
|
3429
|
+
last_verified_by TEXT,
|
|
3430
|
+
okf_sources TEXT,
|
|
3431
|
+
okf_verified TEXT,
|
|
3432
|
+
okf_usage_window TEXT,
|
|
3433
|
+
embedding_failed_at INTEGER,
|
|
3434
|
+
embedding_failure_kind TEXT,
|
|
3435
|
+
embedding_attempts INTEGER NOT NULL DEFAULT 0,
|
|
3436
|
+
valid_from INTEGER,
|
|
3437
|
+
valid_to INTEGER,
|
|
3438
|
+
superseded_by TEXT,
|
|
3439
|
+
superseded_at INTEGER
|
|
3440
|
+
);
|
|
3441
|
+
|
|
3442
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
|
|
3443
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_source_ref_idx ON ${prefix}entries(entity_id, source_ref);
|
|
3444
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_source_hash_idx ON ${prefix}entries(entity_id, source_hash) WHERE source_hash IS NOT NULL;
|
|
3445
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_updated_idx ON ${prefix}entries(updated_at DESC);
|
|
3446
|
+
|
|
3447
|
+
-- source_ref_index: per-(entity, source_hash) record of the canonical sourceRef
|
|
3448
|
+
-- currently holding that hash. The partial UNIQUE index on (entity_id, source_hash)
|
|
3449
|
+
-- WHERE deleted_at IS NULL enforces the sourceRef-level TOCTOU-race invariant;
|
|
3450
|
+
-- entries-level uniqueness cannot express it because a single ingestDocument call
|
|
3451
|
+
-- writes N facts that all share (entity_id, source_ref, source_hash). See
|
|
3452
|
+
-- docs/superpowers/specs/2026-08-07-dependabot-concurrency-release-hygiene-design.md \xA7B1.
|
|
3453
|
+
CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
3454
|
+
id TEXT PRIMARY KEY,
|
|
3455
|
+
entity_id TEXT NOT NULL,
|
|
3456
|
+
source_hash TEXT NOT NULL,
|
|
3457
|
+
source_ref TEXT NOT NULL,
|
|
3458
|
+
created_at INTEGER NOT NULL,
|
|
3459
|
+
deleted_at INTEGER
|
|
3460
|
+
);
|
|
3461
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
3462
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
3463
|
+
WHERE deleted_at IS NULL;
|
|
3464
|
+
|
|
3465
|
+
CREATE TABLE IF NOT EXISTS ${prefix}tasks (
|
|
3466
|
+
id TEXT PRIMARY KEY,
|
|
3467
|
+
entity_id TEXT NOT NULL,
|
|
3468
|
+
description TEXT NOT NULL,
|
|
3469
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
3470
|
+
priority INTEGER NOT NULL DEFAULT 0,
|
|
3471
|
+
created_at INTEGER NOT NULL,
|
|
3472
|
+
updated_at INTEGER NOT NULL,
|
|
3473
|
+
resolved_at INTEGER,
|
|
3474
|
+
deleted_at INTEGER,
|
|
3475
|
+
okf_type TEXT,
|
|
3476
|
+
lifecycle_status TEXT NOT NULL DEFAULT 'stable',
|
|
3477
|
+
stale_after INTEGER,
|
|
3478
|
+
generated_by TEXT,
|
|
3479
|
+
last_verified_at INTEGER,
|
|
3480
|
+
last_verified_by TEXT,
|
|
3481
|
+
okf_sources TEXT,
|
|
3482
|
+
okf_verified TEXT,
|
|
3483
|
+
okf_usage_window TEXT
|
|
3484
|
+
);
|
|
3485
|
+
|
|
3486
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_entity_idx ON ${prefix}tasks(entity_id, status);
|
|
3487
|
+
|
|
3488
|
+
CREATE TABLE IF NOT EXISTS ${prefix}edges (
|
|
3489
|
+
id TEXT PRIMARY KEY,
|
|
3490
|
+
entity_id TEXT NOT NULL,
|
|
3491
|
+
source_id TEXT NOT NULL,
|
|
3492
|
+
target_id TEXT NOT NULL,
|
|
3493
|
+
edge_type TEXT NOT NULL,
|
|
3494
|
+
created_at INTEGER NOT NULL,
|
|
3495
|
+
UNIQUE(entity_id, source_id, target_id, edge_type)
|
|
3496
|
+
);
|
|
3497
|
+
|
|
3498
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
|
|
3499
|
+
|
|
3500
|
+
CREATE TABLE IF NOT EXISTS ${prefix}events (
|
|
3501
|
+
id TEXT PRIMARY KEY,
|
|
3502
|
+
entity_id TEXT NOT NULL,
|
|
3503
|
+
event_type TEXT NOT NULL,
|
|
3504
|
+
summary TEXT NOT NULL,
|
|
3505
|
+
related_entry_id TEXT,
|
|
3506
|
+
created_at INTEGER NOT NULL,
|
|
3507
|
+
occurred_at INTEGER
|
|
3508
|
+
);
|
|
3509
|
+
|
|
3510
|
+
CREATE INDEX IF NOT EXISTS ${prefix}events_entity_idx ON ${prefix}events(entity_id, created_at DESC);
|
|
3511
|
+
|
|
3512
|
+
CREATE TABLE IF NOT EXISTS ${prefix}checkpoints (
|
|
3513
|
+
entity_id TEXT PRIMARY KEY,
|
|
3514
|
+
heal_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
3515
|
+
memory_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
3516
|
+
librarian_watermark_at INTEGER,
|
|
3517
|
+
librarian_watermark_id TEXT
|
|
3518
|
+
);
|
|
3519
|
+
|
|
3520
|
+
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
3521
|
+
entity_id TEXT PRIMARY KEY,
|
|
3522
|
+
mode TEXT NOT NULL DEFAULT 'off',
|
|
3523
|
+
manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
|
|
3524
|
+
updated_at INTEGER NOT NULL
|
|
3525
|
+
);
|
|
3526
|
+
|
|
3527
|
+
CREATE TABLE IF NOT EXISTS ${prefix}meta (
|
|
3528
|
+
key TEXT PRIMARY KEY,
|
|
3529
|
+
value TEXT NOT NULL
|
|
3530
|
+
);
|
|
3531
|
+
|
|
3532
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
3533
|
+
id TEXT PRIMARY KEY,
|
|
3534
|
+
entity_id TEXT NOT NULL,
|
|
3535
|
+
table_name TEXT NOT NULL,
|
|
3536
|
+
record_id TEXT NOT NULL,
|
|
3537
|
+
operation TEXT NOT NULL,
|
|
3538
|
+
payload TEXT NOT NULL,
|
|
3539
|
+
created_at INTEGER NOT NULL
|
|
3540
|
+
);
|
|
3541
|
+
|
|
3542
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
3543
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
3544
|
+
|
|
3545
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_created_at
|
|
3546
|
+
ON ${prefix}outbox (created_at);
|
|
3547
|
+
`);
|
|
3548
|
+
await createOkfV02IndexesIfColumnsExist(db, prefix);
|
|
3549
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
3550
|
+
}
|
|
3551
|
+
async function createOkfV02IndexesIfColumnsExist(db, prefix) {
|
|
3552
|
+
for (const table of ["entries", "tasks"]) {
|
|
3553
|
+
const cols = await db.getAllAsync(
|
|
3554
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
3555
|
+
);
|
|
3556
|
+
const colNames = new Set(cols.map((c) => c.name));
|
|
3557
|
+
if (colNames.has("lifecycle_status")) {
|
|
3558
|
+
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_lifecycle_status_idx ON ${prefix}${table}(lifecycle_status);`);
|
|
3559
|
+
}
|
|
3560
|
+
if (colNames.has("stale_after")) {
|
|
3561
|
+
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_stale_after_idx ON ${prefix}${table}(stale_after);`);
|
|
3562
|
+
}
|
|
3563
|
+
if (colNames.has("last_verified_at")) {
|
|
3564
|
+
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_last_verified_at_idx ON ${prefix}${table}(last_verified_at);`);
|
|
3565
|
+
}
|
|
3566
|
+
}
|
|
3567
|
+
}
|
|
3568
|
+
async function createTemporalIndexesIfColumnsExist(db, prefix) {
|
|
3569
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}entries)`);
|
|
3570
|
+
const names = new Set(cols.map((c) => c.name));
|
|
3571
|
+
if (names.has("superseded_by")) {
|
|
3572
|
+
await db.execAsync(
|
|
3573
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_superseded_idx ON ${prefix}entries(entity_id, superseded_by) WHERE superseded_by IS NOT NULL;`
|
|
3574
|
+
);
|
|
3575
|
+
}
|
|
3576
|
+
if (names.has("valid_from") && names.has("valid_to")) {
|
|
3577
|
+
await db.execAsync(
|
|
3578
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_temporal_idx ON ${prefix}entries(entity_id) WHERE valid_from IS NOT NULL OR valid_to IS NOT NULL;`
|
|
3579
|
+
);
|
|
3580
|
+
}
|
|
3581
|
+
}
|
|
3582
|
+
|
|
3583
|
+
// src/db/migrations.ts
|
|
3584
|
+
var MIGRATIONS = [
|
|
3585
|
+
{
|
|
3586
|
+
version: 1,
|
|
3587
|
+
description: "Rebuild FTS5 with porter unicode61 tokenizer (superseded by v2)",
|
|
3588
|
+
run: async (_db, _prefix) => {
|
|
3589
|
+
}
|
|
3590
|
+
},
|
|
3591
|
+
{
|
|
3592
|
+
version: 2,
|
|
3593
|
+
description: "Remove FTS5; add embedding column for semantic retrieval",
|
|
3594
|
+
run: async (db, prefix) => {
|
|
3595
|
+
await db.withTransactionAsync(async (tx) => {
|
|
3596
|
+
await tx.execAsync(`
|
|
3597
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_ai;
|
|
3598
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_ad;
|
|
3599
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_au;
|
|
3600
|
+
DROP TABLE IF EXISTS ${prefix}entries_fts;
|
|
3601
|
+
`);
|
|
3602
|
+
});
|
|
3603
|
+
const cols = await db.getAllAsync(
|
|
3604
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3605
|
+
);
|
|
3606
|
+
if (!cols.some((c) => c.name === "embedding")) {
|
|
3607
|
+
await db.execAsync(`ALTER TABLE ${prefix}entries ADD COLUMN embedding TEXT`);
|
|
3608
|
+
}
|
|
3609
|
+
}
|
|
3610
|
+
},
|
|
3611
|
+
{
|
|
3612
|
+
version: 3,
|
|
3613
|
+
description: "Add embedding_blob BLOB column for Float32Array vector storage",
|
|
3614
|
+
run: async (db, prefix) => {
|
|
3615
|
+
const cols = await db.getAllAsync(
|
|
3616
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3617
|
+
);
|
|
3618
|
+
if (!cols.some((c) => c.name === "embedding_blob")) {
|
|
3619
|
+
await db.execAsync(
|
|
3620
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN embedding_blob BLOB`
|
|
3621
|
+
);
|
|
3622
|
+
}
|
|
3623
|
+
}
|
|
3624
|
+
},
|
|
3625
|
+
{
|
|
3626
|
+
version: 4,
|
|
3627
|
+
description: "Create outbox table for change data capture",
|
|
3628
|
+
run: async (db, prefix) => {
|
|
3629
|
+
await db.execAsync(`
|
|
3630
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
3631
|
+
id TEXT PRIMARY KEY,
|
|
3632
|
+
entity_id TEXT NOT NULL,
|
|
3633
|
+
table_name TEXT NOT NULL,
|
|
3634
|
+
record_id TEXT NOT NULL,
|
|
3635
|
+
operation TEXT NOT NULL,
|
|
3636
|
+
payload TEXT NOT NULL,
|
|
3637
|
+
created_at INTEGER NOT NULL
|
|
3638
|
+
);
|
|
3639
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
3640
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
3641
|
+
`);
|
|
3642
|
+
}
|
|
3643
|
+
},
|
|
3644
|
+
{
|
|
3645
|
+
version: 5,
|
|
3646
|
+
description: "Add okf_type to entries/tasks for OKF type fidelity; create edges table for OKF graph import",
|
|
3647
|
+
run: async (db, prefix) => {
|
|
3648
|
+
for (const table of ["entries", "tasks"]) {
|
|
3649
|
+
const cols = await db.getAllAsync(
|
|
3650
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
3651
|
+
);
|
|
3652
|
+
if (!cols.some((c) => c.name === "okf_type")) {
|
|
3653
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN okf_type TEXT`);
|
|
3654
|
+
}
|
|
3655
|
+
}
|
|
3656
|
+
await db.execAsync(`
|
|
3657
|
+
CREATE TABLE IF NOT EXISTS ${prefix}edges (
|
|
3658
|
+
id TEXT PRIMARY KEY,
|
|
3659
|
+
entity_id TEXT NOT NULL,
|
|
3660
|
+
source_id TEXT NOT NULL,
|
|
3661
|
+
target_id TEXT NOT NULL,
|
|
3662
|
+
edge_type TEXT NOT NULL,
|
|
3663
|
+
created_at INTEGER NOT NULL,
|
|
3664
|
+
UNIQUE(entity_id, source_id, target_id, edge_type)
|
|
3665
|
+
);
|
|
3666
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_idx ON ${prefix}edges (entity_id);
|
|
3667
|
+
`);
|
|
3668
|
+
}
|
|
3669
|
+
},
|
|
3670
|
+
{
|
|
3671
|
+
version: 6,
|
|
3672
|
+
description: "Add entity_manifests table for per-entity ontology state",
|
|
3673
|
+
run: async (db, prefix) => {
|
|
3674
|
+
await db.execAsync(`
|
|
3675
|
+
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
3676
|
+
entity_id TEXT PRIMARY KEY,
|
|
3677
|
+
mode TEXT NOT NULL DEFAULT 'off',
|
|
3678
|
+
manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
|
|
3679
|
+
updated_at INTEGER NOT NULL
|
|
3680
|
+
);
|
|
3681
|
+
`);
|
|
3682
|
+
}
|
|
3683
|
+
},
|
|
3684
|
+
{
|
|
3685
|
+
version: 7,
|
|
3686
|
+
description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
|
|
3687
|
+
run: async (db, prefix) => {
|
|
3688
|
+
const cols = await db.getAllAsync(
|
|
3689
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3690
|
+
);
|
|
3691
|
+
if (!cols.some((c) => c.name === "ontology_checked_at")) {
|
|
3692
|
+
await db.execAsync(
|
|
3693
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
|
|
3694
|
+
);
|
|
3695
|
+
}
|
|
3696
|
+
}
|
|
3697
|
+
},
|
|
3698
|
+
{
|
|
3699
|
+
version: 8,
|
|
3700
|
+
description: "Add heal_checked_at to entries for heal recheck cooldown",
|
|
3701
|
+
run: async (db, prefix) => {
|
|
3702
|
+
const cols = await db.getAllAsync(
|
|
3703
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3704
|
+
);
|
|
3705
|
+
if (!cols.some((c) => c.name === "heal_checked_at")) {
|
|
3706
|
+
await db.execAsync(
|
|
3707
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN heal_checked_at INTEGER`
|
|
3708
|
+
);
|
|
3709
|
+
}
|
|
3710
|
+
}
|
|
3711
|
+
},
|
|
3712
|
+
{
|
|
3713
|
+
version: 9,
|
|
3714
|
+
description: "add_source_ref_index",
|
|
3715
|
+
run: async (db, prefix) => {
|
|
3716
|
+
const duplicates = await db.getAllAsync(
|
|
3717
|
+
`SELECT entity_id, source_hash, COUNT(DISTINCT source_ref) AS n_refs
|
|
3718
|
+
FROM ${prefix}entries
|
|
3719
|
+
WHERE deleted_at IS NULL AND source_hash IS NOT NULL
|
|
3720
|
+
GROUP BY entity_id, source_hash
|
|
3721
|
+
HAVING COUNT(DISTINCT source_ref) > 1`
|
|
3722
|
+
);
|
|
3723
|
+
if (duplicates.length > 0) {
|
|
3724
|
+
const sample = duplicates.slice(0, 5).map((d) => `(entity_id=${d.entity_id}, source_hash=${d.source_hash.slice(0, 12)}\u2026, n_refs=${d.n_refs})`).join(", ");
|
|
3725
|
+
throw new Error(
|
|
3726
|
+
`Migration v9 (add_source_ref_index) failed: existing live rows have multiple sourceRefs sharing a hash. Found ${duplicates.length} duplicate (entity_id, source_hash) groups. First ${Math.min(5, duplicates.length)}: ${sample}. Resolve each by calling forget({ sourceRef: <loser> }) for the offending sourceRef, then re-run setup.`
|
|
3727
|
+
);
|
|
3728
|
+
}
|
|
3729
|
+
await db.execAsync(
|
|
3730
|
+
`CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
3731
|
+
id TEXT PRIMARY KEY,
|
|
3732
|
+
entity_id TEXT NOT NULL,
|
|
3733
|
+
source_hash TEXT NOT NULL,
|
|
3734
|
+
source_ref TEXT NOT NULL,
|
|
3735
|
+
created_at INTEGER NOT NULL,
|
|
3736
|
+
deleted_at INTEGER
|
|
3737
|
+
);
|
|
3738
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
3739
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
3740
|
+
WHERE deleted_at IS NULL;`
|
|
3741
|
+
);
|
|
3742
|
+
await db.execAsync(
|
|
3743
|
+
`INSERT OR IGNORE INTO ${prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
|
|
3744
|
+
SELECT
|
|
3745
|
+
'sri:' || entity_id || ':' || source_hash,
|
|
3746
|
+
entity_id,
|
|
3747
|
+
source_hash,
|
|
3748
|
+
source_ref,
|
|
3749
|
+
updated_at,
|
|
3750
|
+
NULL
|
|
3751
|
+
FROM (
|
|
3752
|
+
SELECT
|
|
3753
|
+
entity_id, source_hash, source_ref, updated_at,
|
|
3754
|
+
ROW_NUMBER() OVER (
|
|
3755
|
+
PARTITION BY entity_id, source_hash
|
|
3756
|
+
ORDER BY updated_at ASC, id ASC
|
|
3757
|
+
) AS rn
|
|
3758
|
+
FROM ${prefix}entries
|
|
3759
|
+
WHERE deleted_at IS NULL
|
|
3760
|
+
AND source_hash IS NOT NULL
|
|
3761
|
+
AND source_ref IS NOT NULL
|
|
3762
|
+
) ranked
|
|
3763
|
+
WHERE rn = 1;`
|
|
3764
|
+
);
|
|
3765
|
+
}
|
|
3766
|
+
},
|
|
3767
|
+
{
|
|
3768
|
+
version: 10,
|
|
3769
|
+
description: "OKF v0.2: add lifecycle_status, stale_after, generated_by, last_verified_*, okf_sources, okf_verified, okf_usage_window to entries and tasks",
|
|
3770
|
+
run: async (db, prefix) => {
|
|
3771
|
+
for (const table of ["entries", "tasks"]) {
|
|
3772
|
+
const cols = await db.getAllAsync(
|
|
3773
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
3774
|
+
);
|
|
3775
|
+
const existing = new Set(cols.map((c) => c.name));
|
|
3776
|
+
const adds = [
|
|
3777
|
+
["lifecycle_status", `ALTER TABLE ${prefix}${table} ADD COLUMN lifecycle_status TEXT NOT NULL DEFAULT 'stable'`],
|
|
3778
|
+
["stale_after", `ALTER TABLE ${prefix}${table} ADD COLUMN stale_after INTEGER`],
|
|
3779
|
+
["generated_by", `ALTER TABLE ${prefix}${table} ADD COLUMN generated_by TEXT`],
|
|
3780
|
+
["last_verified_at", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_at INTEGER`],
|
|
3781
|
+
["last_verified_by", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_by TEXT`],
|
|
3782
|
+
["okf_sources", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_sources TEXT`],
|
|
3783
|
+
["okf_verified", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_verified TEXT`],
|
|
3784
|
+
["okf_usage_window", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_usage_window TEXT`]
|
|
3785
|
+
];
|
|
3786
|
+
for (const [name, sql] of adds) {
|
|
3787
|
+
if (!existing.has(name)) await db.execAsync(sql);
|
|
3788
|
+
}
|
|
3789
|
+
}
|
|
3790
|
+
await db.execAsync(`
|
|
3791
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_lifecycle_status_idx ON ${prefix}entries(lifecycle_status);
|
|
3792
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_stale_after_idx ON ${prefix}entries(stale_after);
|
|
3793
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_last_verified_at_idx ON ${prefix}entries(last_verified_at);
|
|
3794
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_lifecycle_status_idx ON ${prefix}tasks(lifecycle_status);
|
|
3795
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_stale_after_idx ON ${prefix}tasks(stale_after);
|
|
3796
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_last_verified_at_idx ON ${prefix}tasks(last_verified_at);
|
|
3797
|
+
`);
|
|
3798
|
+
}
|
|
3799
|
+
},
|
|
3800
|
+
{
|
|
3801
|
+
version: 11,
|
|
3802
|
+
description: "Add embedding_failed_at, embedding_failure_kind, embedding_attempts to entries for embed retry state",
|
|
3803
|
+
run: async (db, prefix) => {
|
|
3804
|
+
const cols = await db.getAllAsync(
|
|
3805
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3806
|
+
);
|
|
3807
|
+
const existing = new Set(cols.map((c) => c.name));
|
|
3808
|
+
const adds = [
|
|
3809
|
+
["embedding_failed_at", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failed_at INTEGER`],
|
|
3810
|
+
["embedding_failure_kind", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failure_kind TEXT`],
|
|
3811
|
+
["embedding_attempts", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_attempts INTEGER NOT NULL DEFAULT 0`]
|
|
3812
|
+
];
|
|
3813
|
+
for (const [name, sql] of adds) {
|
|
3814
|
+
if (!existing.has(name)) await db.execAsync(sql);
|
|
3815
|
+
}
|
|
3816
|
+
}
|
|
3817
|
+
},
|
|
3818
|
+
{
|
|
3819
|
+
version: 12,
|
|
3820
|
+
description: "Replace edges_entity_idx with composite edges(entity_id, id) for keyset paging",
|
|
3821
|
+
run: async (db, prefix) => {
|
|
3822
|
+
await db.execAsync(`
|
|
3823
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
|
|
3824
|
+
DROP INDEX IF EXISTS ${prefix}edges_entity_idx;
|
|
3825
|
+
`);
|
|
3826
|
+
}
|
|
3827
|
+
},
|
|
3828
|
+
{
|
|
3829
|
+
version: 13,
|
|
3830
|
+
description: "Temporal facts: valid_from/valid_to/superseded_by/superseded_at on entries, occurred_at on events, librarian watermark on checkpoints",
|
|
3831
|
+
run: async (db, prefix) => {
|
|
3832
|
+
const adds = [
|
|
3833
|
+
["entries", "valid_from", "INTEGER"],
|
|
3834
|
+
["entries", "valid_to", "INTEGER"],
|
|
3835
|
+
["entries", "superseded_by", "TEXT"],
|
|
3836
|
+
["entries", "superseded_at", "INTEGER"],
|
|
3837
|
+
["events", "occurred_at", "INTEGER"],
|
|
3838
|
+
["checkpoints", "librarian_watermark_at", "INTEGER"],
|
|
3839
|
+
["checkpoints", "librarian_watermark_id", "TEXT"]
|
|
3840
|
+
];
|
|
3841
|
+
const existingByTable = /* @__PURE__ */ new Map();
|
|
3842
|
+
for (const [table, column, type] of adds) {
|
|
3843
|
+
let existing = existingByTable.get(table);
|
|
3844
|
+
if (!existing) {
|
|
3845
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}${table})`);
|
|
3846
|
+
existing = new Set(cols.map((c) => c.name));
|
|
3847
|
+
existingByTable.set(table, existing);
|
|
3848
|
+
}
|
|
3849
|
+
if (!existing.has(column)) {
|
|
3850
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN ${column} ${type}`);
|
|
3851
|
+
existing.add(column);
|
|
3852
|
+
}
|
|
3853
|
+
}
|
|
3854
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
3855
|
+
}
|
|
3856
|
+
},
|
|
3857
|
+
{
|
|
3858
|
+
version: 14,
|
|
3859
|
+
description: "Partial index for the every-open legacy source_type probe",
|
|
3860
|
+
run: async (db, prefix) => {
|
|
3861
|
+
await createLegacySourceTypeIndex(db, prefix);
|
|
3862
|
+
}
|
|
3863
|
+
}
|
|
3864
|
+
];
|
|
3865
|
+
async function createLegacySourceTypeIndex(db, prefix) {
|
|
3866
|
+
await db.execAsync(
|
|
3867
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_legacy_source_type_idx ON ${prefix}entries(source_type) WHERE source_type IN ('user_document', 'agent_inferred')`
|
|
3868
|
+
);
|
|
3869
|
+
}
|
|
3870
|
+
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
3871
|
+
if (MIGRATIONS[i].version <= MIGRATIONS[i - 1].version) {
|
|
3872
|
+
throw new Error(
|
|
3873
|
+
`migrations.ts: MIGRATIONS must be in strictly ascending version order. Found version ${MIGRATIONS[i].version} after ${MIGRATIONS[i - 1].version} at index ${i}.`
|
|
3874
|
+
);
|
|
3875
|
+
}
|
|
3876
|
+
}
|
|
3877
|
+
var CURRENT_SCHEMA_VERSION = MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
|
|
3878
|
+
|
|
3879
|
+
// src/db/sourceRefNormalize.ts
|
|
3880
|
+
var SOURCE_REF_NORMALIZE_ALG = 1;
|
|
3881
|
+
var NORMALIZE_STAMP_SCHEMA_VERSION = 14;
|
|
3882
|
+
var NORMALIZE_STAMP_KEY = "source_ref_normalize_stamp";
|
|
3883
|
+
var ENTRIES_WM_KEY = "normalize_v14_entries_wm";
|
|
3884
|
+
var SRI_WM_KEY = "normalize_v14_sri_wm";
|
|
3885
|
+
var CHUNK = 500;
|
|
3886
|
+
function stampMatches(raw) {
|
|
3887
|
+
return parseJsonObject(raw)?.normalize_alg === SOURCE_REF_NORMALIZE_ALG;
|
|
3888
|
+
}
|
|
3889
|
+
async function writeNormalizeStamp(meta, tx) {
|
|
3890
|
+
await meta.setMeta(NORMALIZE_STAMP_KEY, JSON.stringify({ schema_version: NORMALIZE_STAMP_SCHEMA_VERSION, normalize_alg: SOURCE_REF_NORMALIZE_ALG }), tx);
|
|
3891
|
+
}
|
|
3892
|
+
async function resumeRowid(db, prefix, meta, table, key) {
|
|
3893
|
+
const wm = parseJsonObject(await meta.getMeta(key));
|
|
3894
|
+
if (typeof wm?.rowid !== "number" || typeof wm.id !== "string") return 0;
|
|
3895
|
+
const row = await db.getFirstAsync(`SELECT id FROM ${prefix}${table} WHERE rowid = ?`, [wm.rowid]);
|
|
3896
|
+
return row?.id === wm.id ? wm.rowid : 0;
|
|
3897
|
+
}
|
|
3898
|
+
async function resetNormalizeWatermarks(meta) {
|
|
3899
|
+
await meta.deleteMeta(ENTRIES_WM_KEY);
|
|
3900
|
+
await meta.deleteMeta(SRI_WM_KEY);
|
|
3901
|
+
}
|
|
3902
|
+
async function detectSchemaVersion(meta, prefix) {
|
|
3903
|
+
const stored = await meta.getMeta("schema_version");
|
|
3904
|
+
if (stored) {
|
|
3905
|
+
const v = parseInt(stored, 10);
|
|
3906
|
+
return Number.isFinite(v) ? v : 0;
|
|
3907
|
+
}
|
|
3908
|
+
const ftsDdl = await meta.getTableDdl(`${prefix}entries_fts`);
|
|
3909
|
+
return /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl ?? "") ? 1 : 0;
|
|
3910
|
+
}
|
|
3911
|
+
async function runPass(db, prefix, meta, table, key, stampOnCompletion) {
|
|
3912
|
+
let cursor = await resumeRowid(db, prefix, meta, table, key);
|
|
3913
|
+
while (cursor !== null) {
|
|
3914
|
+
const from = cursor;
|
|
3915
|
+
let next = from;
|
|
3916
|
+
await db.withTransactionAsync(async (tx) => {
|
|
3917
|
+
const rows = await tx.getAllAsync(
|
|
3918
|
+
`SELECT rowid, id, source_ref FROM ${prefix}${table} WHERE rowid > ? ORDER BY rowid LIMIT ${CHUNK}`,
|
|
3919
|
+
[from]
|
|
3920
|
+
);
|
|
3921
|
+
for (const r of rows) {
|
|
3922
|
+
if (r.source_ref == null) continue;
|
|
3923
|
+
const n = normalizeSourceRef(r.source_ref);
|
|
3924
|
+
if (n === r.source_ref) continue;
|
|
3925
|
+
if (table === "entries") {
|
|
3926
|
+
await tx.runAsync(`UPDATE ${prefix}entries SET source_ref = ? WHERE rowid = ?`, [n, r.rowid]);
|
|
3927
|
+
} else if (n === null) {
|
|
3928
|
+
await tx.runAsync(`UPDATE ${prefix}source_ref_index SET deleted_at = COALESCE(deleted_at, ?) WHERE rowid = ?`, [Date.now(), r.rowid]);
|
|
3929
|
+
} else {
|
|
3930
|
+
await tx.runAsync(`UPDATE ${prefix}source_ref_index SET source_ref = ? WHERE rowid = ?`, [n, r.rowid]);
|
|
3931
|
+
}
|
|
3932
|
+
}
|
|
3933
|
+
if (rows.length < CHUNK) {
|
|
3934
|
+
await meta.deleteMeta(key, tx);
|
|
3935
|
+
if (stampOnCompletion) await writeNormalizeStamp(meta, tx);
|
|
3936
|
+
next = null;
|
|
3937
|
+
return;
|
|
3938
|
+
}
|
|
3939
|
+
const last = rows[rows.length - 1];
|
|
3940
|
+
await meta.setMeta(key, JSON.stringify({ rowid: last.rowid, id: last.id }), tx);
|
|
3941
|
+
next = last.rowid;
|
|
3942
|
+
});
|
|
3943
|
+
cursor = next;
|
|
3944
|
+
}
|
|
3945
|
+
}
|
|
3946
|
+
async function probePendingWork(db, meta, prefix) {
|
|
3947
|
+
if (!await meta.tableExists(`${prefix}entries`)) return null;
|
|
3948
|
+
if (!await meta.tableExists(`${prefix}meta`)) {
|
|
3949
|
+
return { type: "schema_migration", fromVersion: 0, toVersion: CURRENT_SCHEMA_VERSION };
|
|
3950
|
+
}
|
|
3951
|
+
const version = await detectSchemaVersion(meta, prefix);
|
|
3952
|
+
if (version < CURRENT_SCHEMA_VERSION) {
|
|
3953
|
+
return { type: "schema_migration", fromVersion: version, toVersion: CURRENT_SCHEMA_VERSION };
|
|
3954
|
+
}
|
|
3955
|
+
if (!stampMatches(await meta.getMeta(NORMALIZE_STAMP_KEY))) {
|
|
3956
|
+
return {
|
|
3957
|
+
type: "source_ref_normalize",
|
|
3958
|
+
entriesWatermark: await resumeRowid(db, prefix, meta, "entries", ENTRIES_WM_KEY),
|
|
3959
|
+
sourceRefIndexWatermark: await resumeRowid(db, prefix, meta, "source_ref_index", SRI_WM_KEY)
|
|
3960
|
+
};
|
|
3961
|
+
}
|
|
3962
|
+
return null;
|
|
3963
|
+
}
|
|
3964
|
+
async function runSourceRefNormalize(db, prefix, meta) {
|
|
3965
|
+
if (stampMatches(await meta.getMeta(NORMALIZE_STAMP_KEY))) return;
|
|
3966
|
+
await runPass(db, prefix, meta, "entries", ENTRIES_WM_KEY, false);
|
|
3967
|
+
await runPass(db, prefix, meta, "source_ref_index", SRI_WM_KEY, true);
|
|
3968
|
+
}
|
|
3969
|
+
|
|
3313
3970
|
// src/services/librarian/constants.ts
|
|
3314
3971
|
var FUZZY_THRESHOLD = 0.5;
|
|
3315
3972
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
@@ -3594,6 +4251,22 @@ function classifyCandidate(candidate, neighbours, cfg, mode) {
|
|
|
3594
4251
|
if (score < cfg.novelThreshold) return { kind: "add" };
|
|
3595
4252
|
return { kind: "ambiguous" };
|
|
3596
4253
|
}
|
|
4254
|
+
async function loadStoredVectors(deps, entityId, nonLive) {
|
|
4255
|
+
const stored = [];
|
|
4256
|
+
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
4257
|
+
if (nonLive.has(row.id)) continue;
|
|
4258
|
+
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
4259
|
+
if (vec) stored.push({ id: row.id, vec });
|
|
4260
|
+
}
|
|
4261
|
+
return stored;
|
|
4262
|
+
}
|
|
4263
|
+
async function gateNeedsKeyword(deps, entityId) {
|
|
4264
|
+
if (!await deps.entryRepo.hasLiveEntry(entityId)) return false;
|
|
4265
|
+
if (typeof deps.options.llmProvider.embed !== "function") return true;
|
|
4266
|
+
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
4267
|
+
const stored = await loadStoredVectors(deps, entityId, nonLive);
|
|
4268
|
+
return stored.length === 0 || stored.some((s) => s.vec.length !== stored[0].vec.length);
|
|
4269
|
+
}
|
|
3597
4270
|
async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
3598
4271
|
const { entityId, trigger } = ctx;
|
|
3599
4272
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -3609,14 +4282,8 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
|
3609
4282
|
}
|
|
3610
4283
|
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
3611
4284
|
const canEmbed = typeof deps.options.llmProvider.embed === "function";
|
|
3612
|
-
const stored = [];
|
|
3613
|
-
|
|
3614
|
-
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
3615
|
-
if (nonLive.has(row.id)) continue;
|
|
3616
|
-
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
3617
|
-
if (vec) stored.push({ id: row.id, vec });
|
|
3618
|
-
}
|
|
3619
|
-
}
|
|
4285
|
+
const stored = canEmbed && unique.length > 0 ? await loadStoredVectors(deps, entityId, nonLive) : [];
|
|
4286
|
+
const hasFacts = unique.length > 0 && await deps.entryRepo.hasLiveEntry(entityId);
|
|
3620
4287
|
const gated = [];
|
|
3621
4288
|
for (const c of unique) {
|
|
3622
4289
|
let ranked = null;
|
|
@@ -3637,7 +4304,7 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
|
3637
4304
|
}
|
|
3638
4305
|
}
|
|
3639
4306
|
if (!ranked) {
|
|
3640
|
-
ranked = (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
|
|
4307
|
+
ranked = hasFacts ? (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null })) : [];
|
|
3641
4308
|
}
|
|
3642
4309
|
const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
|
|
3643
4310
|
const byId = new Map(facts.map((f) => [f.id, f]));
|
|
@@ -3998,6 +4665,9 @@ var runOpsLibrarianPass = async (deps, ctx) => {
|
|
|
3998
4665
|
const sel = await selectBatch(deps, entityId, wm, ctx.meter);
|
|
3999
4666
|
if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
|
|
4000
4667
|
if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
|
|
4668
|
+
if (!deps.searchService.isKeywordAvailable() && await gateNeedsKeyword(deps, entityId)) {
|
|
4669
|
+
throw new WikiSearchUnavailableError("librarian: keyword search unavailable and this pass may need it; retry the pass");
|
|
4670
|
+
}
|
|
4001
4671
|
let ex;
|
|
4002
4672
|
try {
|
|
4003
4673
|
ex = await runExtract(deps, ctx, sel.batch);
|
|
@@ -4227,6 +4897,7 @@ var MaintenanceService = class {
|
|
|
4227
4897
|
}
|
|
4228
4898
|
}
|
|
4229
4899
|
if (vacuum) {
|
|
4900
|
+
await resetNormalizeWatermarks(this.metadataRepo);
|
|
4230
4901
|
await this.metadataRepo.vacuum();
|
|
4231
4902
|
}
|
|
4232
4903
|
await this.searchService.syncEntries(entityId, syncedIds);
|
|
@@ -4526,7 +5197,8 @@ var MaintenanceService = class {
|
|
|
4526
5197
|
*
|
|
4527
5198
|
* Bounded: at most `batchSize` candidates per pass (#67). Loop on
|
|
4528
5199
|
* `result.remaining > 0` for convergence — see {@link HealResult.remaining}
|
|
4529
|
-
* for what convergence means here
|
|
5200
|
+
* for what convergence means here, and for the no-progress flags
|
|
5201
|
+
* (`searchUnavailable`, `budgetStop`) a loop must also stop on.
|
|
4530
5202
|
*/
|
|
4531
5203
|
async doRunHeal(entityId, options) {
|
|
4532
5204
|
const promptOverride = options?.promptOverride;
|
|
@@ -4572,7 +5244,11 @@ var MaintenanceService = class {
|
|
|
4572
5244
|
}
|
|
4573
5245
|
}
|
|
4574
5246
|
const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId, batchSize, recheckCutoff);
|
|
4575
|
-
|
|
5247
|
+
const keywordUnavailable = healCandidates.length > 0 && !this.searchService.isKeywordAvailable();
|
|
5248
|
+
if (healCandidates.length === 0 || keywordUnavailable) {
|
|
5249
|
+
if (keywordUnavailable) {
|
|
5250
|
+
emitDiagnostic(this.options, { ...diagBase, code: "heal_skipped", detail: { reason: "keyword_search_unavailable", count: healCandidates.length } });
|
|
5251
|
+
}
|
|
4576
5252
|
await this.searchService.syncEntries(entityId, Array.from(/* @__PURE__ */ new Set([...orphanedIds, ...staleDowngradedIds])));
|
|
4577
5253
|
this.searchService.evictCache(entityId);
|
|
4578
5254
|
const counts2 = await this.entryRepo.countHealCandidatesByEntityId(entityId, recheckCutoff);
|
|
@@ -4584,7 +5260,10 @@ var MaintenanceService = class {
|
|
|
4584
5260
|
skipped: [],
|
|
4585
5261
|
degraded: [],
|
|
4586
5262
|
remaining: counts2.eligible,
|
|
4587
|
-
deferred: counts2.deferred
|
|
5263
|
+
deferred: counts2.deferred,
|
|
5264
|
+
// remaining stays > 0 so maybeRunHeal holds the checkpoint back and
|
|
5265
|
+
// retries after the outage; the flag lets a host loop stop instead.
|
|
5266
|
+
...keywordUnavailable ? { searchUnavailable: true } : {}
|
|
4588
5267
|
};
|
|
4589
5268
|
}
|
|
4590
5269
|
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
@@ -6117,7 +6796,12 @@ var RetrievalService = class {
|
|
|
6117
6796
|
candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
|
|
6118
6797
|
}
|
|
6119
6798
|
if (weight !== void 0 && weight < 1) {
|
|
6120
|
-
|
|
6799
|
+
try {
|
|
6800
|
+
miniSearchScores = await this.searchService.getKeywordScores(trimmedQuery, scoredEntityIds);
|
|
6801
|
+
} catch (err) {
|
|
6802
|
+
if (!isWikiSearchUnavailableError(err)) throw err;
|
|
6803
|
+
miniSearchScores = /* @__PURE__ */ new Map();
|
|
6804
|
+
}
|
|
6121
6805
|
}
|
|
6122
6806
|
}
|
|
6123
6807
|
if (candidateRows === null) {
|
|
@@ -6612,6 +7296,7 @@ var WriteService = class {
|
|
|
6612
7296
|
this.jobManager = jobManager;
|
|
6613
7297
|
this.maintenanceService = maintenanceService;
|
|
6614
7298
|
this.inflight = /* @__PURE__ */ new Set();
|
|
7299
|
+
this.warnedSearchUnavailable = false;
|
|
6615
7300
|
}
|
|
6616
7301
|
track(p) {
|
|
6617
7302
|
this.inflight.add(p);
|
|
@@ -6708,8 +7393,7 @@ var WriteService = class {
|
|
|
6708
7393
|
this.jobManager.acquireLock("librarian", entityId);
|
|
6709
7394
|
const stage = { job: "librarian" };
|
|
6710
7395
|
const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
6711
|
-
|
|
6712
|
-
this.reportBackgroundFailure(entityId, stage.job);
|
|
7396
|
+
this.reportBackgroundFailure(entityId, stage.job, err);
|
|
6713
7397
|
}).finally(() => {
|
|
6714
7398
|
this.jobManager.releaseLock("librarian", entityId);
|
|
6715
7399
|
});
|
|
@@ -6720,8 +7404,7 @@ var WriteService = class {
|
|
|
6720
7404
|
}
|
|
6721
7405
|
} else if (!this.jobManager.isBlocked("librarian", entityId)) {
|
|
6722
7406
|
const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
6723
|
-
|
|
6724
|
-
this.reportBackgroundFailure(entityId, "heal");
|
|
7407
|
+
this.reportBackgroundFailure(entityId, "heal", err);
|
|
6725
7408
|
});
|
|
6726
7409
|
this.track(healJob);
|
|
6727
7410
|
}
|
|
@@ -6759,17 +7442,23 @@ var WriteService = class {
|
|
|
6759
7442
|
}
|
|
6760
7443
|
}
|
|
6761
7444
|
}
|
|
6762
|
-
reportBackgroundFailure(entityId, job) {
|
|
7445
|
+
reportBackgroundFailure(entityId, job, err) {
|
|
7446
|
+
const searchUnavailable = isWikiSearchUnavailableError(err);
|
|
7447
|
+
if (!searchUnavailable) console.error(err);
|
|
7448
|
+
else if (!this.warnedSearchUnavailable) {
|
|
7449
|
+
this.warnedSearchUnavailable = true;
|
|
7450
|
+
console.warn("[WikiMemory] Auto librarian/heal pass skipped: keyword search is unavailable. Work is held, not lost, until it recovers (a no-arg syncSearchIndex() retries the init); later skips report only via onDiagnostic.");
|
|
7451
|
+
}
|
|
6763
7452
|
emitDiagnostic(this.options, {
|
|
6764
7453
|
code: "background_job_failed",
|
|
6765
7454
|
operation: job,
|
|
6766
7455
|
trigger: "auto",
|
|
6767
7456
|
entityId,
|
|
6768
|
-
detail: { reason: "unhandled_rejection" }
|
|
7457
|
+
detail: { reason: searchUnavailable ? "keyword_search_unavailable" : "unhandled_rejection" }
|
|
6769
7458
|
});
|
|
6770
7459
|
}
|
|
6771
7460
|
};
|
|
6772
7461
|
|
|
6773
|
-
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MAX_EMBED_ATTEMPTS, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, TemporalService, UsageMeter, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, edgeTripleAllowed, emptyManifest, entitySummaryMetaKey, extractSqliteCode, factTokenCost, generateId, isLiveAt, liveAtSql, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, packFactsByBudget, parseEmbedding, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
|
|
6774
|
-
//# sourceMappingURL=chunk-
|
|
6775
|
-
//# sourceMappingURL=chunk-
|
|
7462
|
+
export { BaseRepository, CURRENT_SCHEMA_VERSION, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MAX_EMBED_ATTEMPTS, MIGRATIONS, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, TemporalService, UnavailableIndexStrategy, UsageMeter, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiMigrationRequiredError, WikiParseError, WikiSearchUnavailableError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, createLegacySourceTypeIndex, detectSchemaVersion, edgeTripleAllowed, emptyManifest, entitySummaryMetaKey, extractSqliteCode, factTokenCost, generateId, isLiveAt, isWikiSearchUnavailableError, liveAtSql, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, packFactsByBudget, parseEmbedding, parseJsonArray, parseJsonObject, probePendingWork, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, runSourceRefNormalize, safeSlice, setupDatabase, typeSatisfies, validateInlineEdges, validateManifest, writeNormalizeStamp };
|
|
7463
|
+
//# sourceMappingURL=chunk-KN4XSLLB.mjs.map
|
|
7464
|
+
//# sourceMappingURL=chunk-KN4XSLLB.mjs.map
|