@equationalapplications/core-llm-wiki 7.11.2 → 7.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/{chunk-522SCN2E.mjs → chunk-PCRYIUV5.mjs} +2299 -1408
- package/dist/chunk-PCRYIUV5.mjs.map +1 -0
- package/dist/index.d.mts +10 -3
- package/dist/index.d.ts +10 -3
- package/dist/index.js +5336 -4465
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +645 -669
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-B0UtFCq1.d.mts → testing-CqsXv2_l.d.mts} +195 -23
- package/dist/{testing-B0UtFCq1.d.ts → testing-CqsXv2_l.d.ts} +195 -23
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +494 -35
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +3 -3
- package/dist/chunk-522SCN2E.mjs.map +0 -1
|
@@ -44,6 +44,8 @@ var SQLITE_RESULT_CODE_NAMES = {
|
|
|
44
44
|
// prefix and we normalize it back to the symbolic name so callers can branch on
|
|
45
45
|
// the same string the rest of the ecosystem uses.
|
|
46
46
|
2067: "SQLITE_CONSTRAINT_UNIQUE",
|
|
47
|
+
// 267 = SQLITE_CORRUPT_VTAB — a corrupt FTS5 shadow table (ftsCorruptionClass).
|
|
48
|
+
267: "SQLITE_CORRUPT_VTAB",
|
|
47
49
|
100: "SQLITE_ROW",
|
|
48
50
|
101: "SQLITE_DONE"
|
|
49
51
|
};
|
|
@@ -184,1519 +186,1592 @@ var WikiSupersedeError = class _WikiSupersedeError extends Error {
|
|
|
184
186
|
Object.setPrototypeOf(this, _WikiSupersedeError.prototype);
|
|
185
187
|
}
|
|
186
188
|
};
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
189
|
+
var WikiMigrationRequiredError = class _WikiMigrationRequiredError extends Error {
|
|
190
|
+
constructor(message, pendingWork) {
|
|
191
|
+
super(message);
|
|
192
|
+
this.code = "WIKI_MIGRATION_REQUIRED";
|
|
193
|
+
this.name = "WikiMigrationRequiredError";
|
|
194
|
+
this.pendingWork = pendingWork;
|
|
195
|
+
if (typeof Error.captureStackTrace === "function") {
|
|
196
|
+
Error.captureStackTrace(this, _WikiMigrationRequiredError);
|
|
197
|
+
}
|
|
196
198
|
}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
199
|
+
};
|
|
200
|
+
var WikiSearchUnavailableError = class extends Error {
|
|
201
|
+
constructor(message = "Keyword search is temporarily unavailable", cause) {
|
|
202
|
+
super(message, cause !== void 0 ? { cause } : void 0);
|
|
203
|
+
this.name = "WikiSearchUnavailableError";
|
|
201
204
|
}
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
return
|
|
205
|
+
};
|
|
206
|
+
function isWikiSearchUnavailableError(err) {
|
|
207
|
+
try {
|
|
208
|
+
return err instanceof WikiSearchUnavailableError;
|
|
209
|
+
} catch {
|
|
210
|
+
return false;
|
|
206
211
|
}
|
|
207
|
-
throw new Error(
|
|
208
|
-
"generateId: no cryptographically secure random source available (crypto.randomUUID and crypto.getRandomValues are both missing, and no configureRandomSource() injection was provided)."
|
|
209
|
-
);
|
|
210
212
|
}
|
|
211
213
|
|
|
212
|
-
// src/utils/
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
}
|
|
219
|
-
function resolveNodeType(raw, manifest) {
|
|
220
|
-
const slug = raw.trim();
|
|
221
|
-
if (!slug) return null;
|
|
222
|
-
const hit = manifest.node_types.find((n) => n.type.toLowerCase() === slug.toLowerCase());
|
|
223
|
-
return hit?.type ?? null;
|
|
224
|
-
}
|
|
225
|
-
function resolveEdgeDefinitions(rawEdgeType, manifest) {
|
|
226
|
-
const slug = rawEdgeType.trim();
|
|
227
|
-
if (!slug) return [];
|
|
228
|
-
return manifest.edge_types.filter((e) => e.type.toLowerCase() === slug.toLowerCase());
|
|
229
|
-
}
|
|
230
|
-
function edgeTripleKey(type, sourceType, targetType) {
|
|
231
|
-
return `${type.trim().toLowerCase()}|${sourceType.trim().toLowerCase()}|${targetType.trim().toLowerCase()}`;
|
|
232
|
-
}
|
|
233
|
-
function typeSatisfies(declaredType, concreteType, manifest) {
|
|
234
|
-
const concrete = concreteType.trim().toLowerCase();
|
|
235
|
-
const declared = declaredType.trim().toLowerCase();
|
|
236
|
-
if (!concrete || !declared) return false;
|
|
237
|
-
if (declared === concrete) return true;
|
|
238
|
-
const def = (manifest.node_types ?? []).find(
|
|
239
|
-
(n) => typeof n?.type === "string" && n.type.trim().toLowerCase() === concrete
|
|
214
|
+
// src/utils/pure.ts
|
|
215
|
+
var MAX_REPAIR_CANDIDATES = 5;
|
|
216
|
+
function throwNoJsonFound(text, start) {
|
|
217
|
+
throw new WikiParseError(
|
|
218
|
+
"No JSON object/array found in LLM response",
|
|
219
|
+
{ tier: "strict", position: start, slice: text }
|
|
240
220
|
);
|
|
241
|
-
const parent = typeof def?.parent_type === "string" ? def.parent_type.trim().toLowerCase() : "";
|
|
242
|
-
return parent !== "" && parent === declared;
|
|
243
|
-
}
|
|
244
|
-
function edgeTripleAllowed(manifest, edgeType, sourceType, targetType) {
|
|
245
|
-
const wanted = edgeType.trim().toLowerCase();
|
|
246
|
-
return (manifest.edge_types ?? []).some((d) => typeof d?.type === "string" && d.type.trim().toLowerCase() === wanted && typeof d.source_type === "string" && typeSatisfies(d.source_type, sourceType, manifest) && typeof d.target_type === "string" && typeSatisfies(d.target_type, targetType, manifest));
|
|
247
221
|
}
|
|
248
|
-
function
|
|
249
|
-
const
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
222
|
+
function parseJsonResponse(text) {
|
|
223
|
+
const firstBrace = text.indexOf("{");
|
|
224
|
+
const firstBracket = text.indexOf("[");
|
|
225
|
+
let start;
|
|
226
|
+
let openChar;
|
|
227
|
+
const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
|
|
228
|
+
if (useBrace) {
|
|
229
|
+
start = firstBrace;
|
|
230
|
+
openChar = "{";
|
|
231
|
+
} else if (firstBracket !== -1) {
|
|
232
|
+
start = firstBracket;
|
|
233
|
+
openChar = "[";
|
|
234
|
+
} else {
|
|
235
|
+
throwNoJsonFound(text, null);
|
|
261
236
|
}
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
}
|
|
267
|
-
const parentSlug = node.parent_type.trim().toLowerCase();
|
|
268
|
-
if (parentSlug === node.type.trim().toLowerCase()) {
|
|
269
|
-
throw new Error(`Self-parent: ${node.type}`);
|
|
270
|
-
}
|
|
271
|
-
if (!nodeSlugs.has(parentSlug)) {
|
|
272
|
-
throw new Error(`Parent type not found: ${node.parent_type}`);
|
|
273
|
-
}
|
|
274
|
-
const grandparent = parentOf.get(parentSlug);
|
|
275
|
-
if (grandparent) {
|
|
276
|
-
throw new Error(`Parent chain too deep: ${node.type} \u2192 ${node.parent_type} \u2192 ${grandparent}`);
|
|
237
|
+
const slice = scanJsonSlice(text, start, openChar);
|
|
238
|
+
if (slice !== null) {
|
|
239
|
+
try {
|
|
240
|
+
return JSON.parse(slice);
|
|
241
|
+
} catch {
|
|
277
242
|
}
|
|
278
243
|
}
|
|
279
|
-
const
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
const edgeType = edge.type?.trim();
|
|
283
|
-
const sourceType = edge.source_type?.trim();
|
|
284
|
-
const targetType = edge.target_type?.trim();
|
|
285
|
-
if (!edgeType) throw new Error("Ontology edge type slug must be non-empty");
|
|
286
|
-
if (!sourceType || !targetType || !nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) {
|
|
287
|
-
throw new Error(`Edge type ${edgeType} references unknown node type`);
|
|
288
|
-
}
|
|
289
|
-
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
290
|
-
if (edgeKeys.has(edgeKey)) {
|
|
291
|
-
throw new Error(`Duplicate edge definition: ${edgeType} (${sourceType} \u2192 ${targetType})`);
|
|
292
|
-
}
|
|
293
|
-
edgeKeys.add(edgeKey);
|
|
294
|
-
const canonical = edgeNames.get(edgeType.toLowerCase());
|
|
295
|
-
if (canonical === void 0) {
|
|
296
|
-
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
297
|
-
} else if (canonical !== edgeType) {
|
|
298
|
-
throw new Error(`Inconsistent casing for edge type: ${edgeType} conflicts with ${canonical}`);
|
|
299
|
-
}
|
|
244
|
+
const repairOdd = containerAwareRepair(text, start, false);
|
|
245
|
+
if (repairOdd.success !== null && !repairOdd.ambiguous) {
|
|
246
|
+
return repairOdd.success;
|
|
300
247
|
}
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
const slug = typeof n?.type === "string" ? n.type.trim().toLowerCase() : "";
|
|
306
|
-
if (!slug || declaresParent.has(slug)) continue;
|
|
307
|
-
declaresParent.set(slug, n?.parent_type !== void 0);
|
|
248
|
+
const repairEven = containerAwareRepair(text, start, true);
|
|
249
|
+
if (repairOdd.success !== null && repairEven.success !== null) {
|
|
250
|
+
const pick = repairEven.escapes < repairOdd.escapes ? repairEven : repairOdd;
|
|
251
|
+
return pick.success;
|
|
308
252
|
}
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
function mergeOntologyUpdates(current, updates) {
|
|
312
|
-
const node_types = [...current.node_types];
|
|
313
|
-
const edge_types = [...current.edge_types];
|
|
314
|
-
const nodeSlugs = new Set(node_types.map((n) => n.type.trim().toLowerCase()));
|
|
315
|
-
const edgeKeys = new Set(edge_types.map((e) => edgeTripleKey(e.type, e.source_type, e.target_type)));
|
|
316
|
-
const edgeNames = new Map(edge_types.map((e) => [e.type.trim().toLowerCase(), e.type.trim()]));
|
|
317
|
-
const declaresParent = buildDeclaresParentIndex([
|
|
318
|
-
...current.node_types,
|
|
319
|
-
...updates.node_types ?? []
|
|
320
|
-
]);
|
|
321
|
-
for (const node of updates.node_types ?? []) {
|
|
322
|
-
const type = node?.type?.trim();
|
|
323
|
-
if (!type) continue;
|
|
324
|
-
const key = type.toLowerCase();
|
|
325
|
-
if (nodeSlugs.has(key)) continue;
|
|
326
|
-
const rawParent = typeof node?.parent_type === "string" ? node.parent_type.trim() : "";
|
|
327
|
-
const parentSlug = rawParent.toLowerCase();
|
|
328
|
-
const keepParent = parentSlug !== "" && parentSlug !== key && declaresParent.get(parentSlug) === false;
|
|
329
|
-
node_types.push({
|
|
330
|
-
type,
|
|
331
|
-
description: String(node.description ?? ""),
|
|
332
|
-
...keepParent ? { parent_type: rawParent } : {}
|
|
333
|
-
});
|
|
334
|
-
nodeSlugs.add(key);
|
|
253
|
+
if (repairOdd.success !== null) {
|
|
254
|
+
return repairOdd.success;
|
|
335
255
|
}
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
const sourceType = edge?.source_type?.trim();
|
|
339
|
-
const targetType = edge?.target_type?.trim();
|
|
340
|
-
if (!rawEdgeType || !sourceType || !targetType) continue;
|
|
341
|
-
const edgeType = edgeNames.get(rawEdgeType.toLowerCase()) ?? rawEdgeType;
|
|
342
|
-
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
343
|
-
if (edgeKeys.has(edgeKey)) continue;
|
|
344
|
-
if (!nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) continue;
|
|
345
|
-
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
346
|
-
edge_types.push({
|
|
347
|
-
type: edgeType,
|
|
348
|
-
source_type: sourceType,
|
|
349
|
-
target_type: targetType,
|
|
350
|
-
description: String(edge.description ?? "")
|
|
351
|
-
});
|
|
352
|
-
edgeKeys.add(edgeKey);
|
|
256
|
+
if (repairEven.success !== null) {
|
|
257
|
+
return repairEven.success;
|
|
353
258
|
}
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
const drop = (reason, edgeType) => {
|
|
361
|
-
drops?.push({ reason, sourceId: null, edgeType, sourceNodeType: sourceType, targetNodeType: null });
|
|
362
|
-
};
|
|
363
|
-
if (!Array.isArray(edges)) {
|
|
364
|
-
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", "");
|
|
365
|
-
drop("invalid_shape", null);
|
|
366
|
-
return [];
|
|
259
|
+
const repair = repairEven.failed !== null ? repairEven : repairOdd;
|
|
260
|
+
if (repair.failed !== null) {
|
|
261
|
+
throw new WikiParseError(
|
|
262
|
+
`Repair produced a candidate but JSON.parse rejected it: ${repair.failed.message}`,
|
|
263
|
+
{ tier: "repair", position: repair.failed.position, slice: repair.failed.candidate }
|
|
264
|
+
);
|
|
367
265
|
}
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
266
|
+
if (slice === null) {
|
|
267
|
+
throwNoJsonFound(text, start);
|
|
268
|
+
}
|
|
269
|
+
throw new WikiParseError(
|
|
270
|
+
"No parsable JSON candidate found",
|
|
271
|
+
{ tier: "all", position: start, slice: text }
|
|
272
|
+
);
|
|
273
|
+
}
|
|
274
|
+
function scanJsonSlice(text, start, openChar) {
|
|
275
|
+
const closeChar = openChar === "{" ? "}" : "]";
|
|
276
|
+
let depth = 0;
|
|
277
|
+
let inString = false;
|
|
278
|
+
let escape = false;
|
|
279
|
+
let end = -1;
|
|
280
|
+
for (let i = start; i < text.length; i++) {
|
|
281
|
+
const ch = text[i];
|
|
282
|
+
if (escape) {
|
|
283
|
+
escape = false;
|
|
373
284
|
continue;
|
|
374
285
|
}
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
if (!match) {
|
|
378
|
-
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", edge.edge_type);
|
|
379
|
-
drop("type_not_in_manifest", edge.edge_type);
|
|
286
|
+
if (ch === "\\" && inString) {
|
|
287
|
+
escape = true;
|
|
380
288
|
continue;
|
|
381
289
|
}
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
}
|
|
386
|
-
|
|
387
|
-
// src/utils/embedding.ts
|
|
388
|
-
function parseEmbedding(blob, text) {
|
|
389
|
-
if (blob && blob.byteLength > 0) {
|
|
390
|
-
if (blob.byteLength % 4 !== 0) return null;
|
|
391
|
-
const copy = new ArrayBuffer(blob.byteLength);
|
|
392
|
-
new Uint8Array(copy).set(blob);
|
|
393
|
-
const vector = new Float32Array(copy);
|
|
394
|
-
for (const value of vector) {
|
|
395
|
-
if (!Number.isFinite(value)) return null;
|
|
290
|
+
if (ch === '"') {
|
|
291
|
+
inString = !inString;
|
|
292
|
+
continue;
|
|
396
293
|
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
294
|
+
if (inString) continue;
|
|
295
|
+
if (ch === openChar) {
|
|
296
|
+
depth++;
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
if (ch === closeChar) {
|
|
300
|
+
depth--;
|
|
301
|
+
if (depth === 0) {
|
|
302
|
+
end = i;
|
|
303
|
+
break;
|
|
406
304
|
}
|
|
407
|
-
return vector;
|
|
408
|
-
} catch {
|
|
409
|
-
return null;
|
|
410
305
|
}
|
|
411
306
|
}
|
|
412
|
-
return null;
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
// src/utils/cosine.ts
|
|
416
|
-
function cosineSimilarity(a, b) {
|
|
417
|
-
let dot = 0, normA = 0, normB = 0;
|
|
418
|
-
const len = Math.min(a.length, b.length);
|
|
419
|
-
for (let i = 0; i < len; i++) {
|
|
420
|
-
dot += a[i] * b[i];
|
|
421
|
-
normA += a[i] * a[i];
|
|
422
|
-
normB += b[i] * b[i];
|
|
423
|
-
}
|
|
424
|
-
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
425
|
-
return denom === 0 ? 0 : dot / denom;
|
|
307
|
+
if (end === -1) return null;
|
|
308
|
+
return safeSlice(text, start, end + 1);
|
|
426
309
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
310
|
+
function containerAwareRepair(text, start, closeOnEvenParity) {
|
|
311
|
+
const stack = [];
|
|
312
|
+
let inString = false;
|
|
313
|
+
let stringRole = null;
|
|
314
|
+
let escape = false;
|
|
315
|
+
let i = start;
|
|
316
|
+
let out = "";
|
|
317
|
+
let bareQuoteCount = 0;
|
|
318
|
+
let bestFailed = null;
|
|
319
|
+
let escapes = 0;
|
|
320
|
+
let ambiguous = false;
|
|
321
|
+
let bestSuccess = null;
|
|
322
|
+
let attempts = 0;
|
|
323
|
+
function tryEmitCandidate(candidate) {
|
|
324
|
+
attempts++;
|
|
325
|
+
try {
|
|
326
|
+
bestSuccess = JSON.parse(candidate);
|
|
327
|
+
return true;
|
|
328
|
+
} catch (err) {
|
|
329
|
+
if (bestFailed === null) {
|
|
330
|
+
bestFailed = {
|
|
331
|
+
candidate,
|
|
332
|
+
position: extractParsePosition(err),
|
|
333
|
+
message: safeErrorToString(err)
|
|
334
|
+
};
|
|
441
335
|
}
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
// src/services/SearchService.ts
|
|
447
|
-
var _SearchService = class _SearchService {
|
|
448
|
-
constructor(entryRepo, indexStrategy) {
|
|
449
|
-
this.entryRepo = entryRepo;
|
|
450
|
-
this.vectorCache = /* @__PURE__ */ new Map();
|
|
451
|
-
/**
|
|
452
|
-
* Serializes rebuilds. `rebuildIndex` awaits a repository read between
|
|
453
|
-
* snapshotting the previous id set and discarding it, so two concurrent
|
|
454
|
-
* sync() calls for one entity can interleave: a slow, stale read lands last
|
|
455
|
-
* and discards documents the fresh read just added. Chaining also keeps
|
|
456
|
-
* discard()/addAll() out of each other's way, which is what accrued the
|
|
457
|
-
* auto-vacuum debt behind the TypeError in #64.
|
|
458
|
-
*/
|
|
459
|
-
this.syncChain = Promise.resolve();
|
|
460
|
-
/**
|
|
461
|
-
* Entities whose index may have drifted from SQLite: an incremental update
|
|
462
|
-
* failed part-way, or core wrote rows it could not index (upsertGraph runs in
|
|
463
|
-
* the host's transaction). Their next syncEntries rebuilds the entity in full.
|
|
464
|
-
* See spec 2026-09-28 §5.
|
|
465
|
-
*/
|
|
466
|
-
this.staleEntities = /* @__PURE__ */ new Set();
|
|
467
|
-
/**
|
|
468
|
-
* Per-entity count of markStale() calls. A rebuild clears an entity's stale
|
|
469
|
-
* flag only when the count it snapshotted before its read still holds
|
|
470
|
-
* afterwards: markStale() runs inside the host's still-open transaction, so
|
|
471
|
-
* a call landing mid-read belongs to rows the read cannot have seen, and
|
|
472
|
-
* its flag must outlive the turn (#233 review).
|
|
473
|
-
*/
|
|
474
|
-
this.staleEpochs = /* @__PURE__ */ new Map();
|
|
475
|
-
this.indexStrategy = indexStrategy;
|
|
476
|
-
}
|
|
477
|
-
/**
|
|
478
|
-
* Install the strategy resolved by `WikiMemory.setup()`. Before setup, the
|
|
479
|
-
* service runs on an empty MiniSearch placeholder, as it always has.
|
|
480
|
-
*/
|
|
481
|
-
setIndexStrategy(strategy) {
|
|
482
|
-
this.indexStrategy = strategy;
|
|
336
|
+
return false;
|
|
337
|
+
}
|
|
483
338
|
}
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
const work = this.syncChain.then(async () => {
|
|
491
|
-
try {
|
|
492
|
-
try {
|
|
493
|
-
await this.indexStrategy.drain();
|
|
494
|
-
} finally {
|
|
495
|
-
this.evictCache(entityId);
|
|
496
|
-
}
|
|
497
|
-
} catch (err) {
|
|
498
|
-
console.warn(`[WikiMemory] search index drain failed for ${entityId ?? "*"}:`, err);
|
|
339
|
+
while (i < text.length) {
|
|
340
|
+
const ch = text[i];
|
|
341
|
+
if (stack.length === 0 && !inString) {
|
|
342
|
+
if (ch === "{" || ch === "[") {
|
|
343
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
344
|
+
out += ch;
|
|
499
345
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
for (const id of [...this.staleEntities]) {
|
|
522
|
-
if ((this.staleEpochs.get(id) ?? 0) === (epochsBefore.get(id) ?? 0)) {
|
|
523
|
-
this.staleEntities.delete(id);
|
|
524
|
-
}
|
|
525
|
-
}
|
|
526
|
-
} else {
|
|
527
|
-
await this.rebuildIndex(entityId);
|
|
528
|
-
if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
|
|
529
|
-
this.staleEntities.delete(entityId);
|
|
530
|
-
}
|
|
531
|
-
}
|
|
532
|
-
await this.indexStrategy.vacuum?.();
|
|
533
|
-
} finally {
|
|
534
|
-
this.evictCache(entityId);
|
|
535
|
-
}
|
|
536
|
-
} catch (err) {
|
|
537
|
-
console.warn(`[WikiMemory] search index rebuild failed for ${entityId ?? "*"}:`, err);
|
|
346
|
+
i++;
|
|
347
|
+
continue;
|
|
348
|
+
}
|
|
349
|
+
if (escape) {
|
|
350
|
+
out += ch;
|
|
351
|
+
escape = false;
|
|
352
|
+
i++;
|
|
353
|
+
continue;
|
|
354
|
+
}
|
|
355
|
+
if (inString && ch === "\\") {
|
|
356
|
+
out += ch;
|
|
357
|
+
escape = true;
|
|
358
|
+
i++;
|
|
359
|
+
continue;
|
|
360
|
+
}
|
|
361
|
+
if (inString) {
|
|
362
|
+
const code = ch.charCodeAt(0);
|
|
363
|
+
if (code < 32) {
|
|
364
|
+
out += controlCharEscape(code);
|
|
365
|
+
i++;
|
|
366
|
+
continue;
|
|
538
367
|
}
|
|
539
|
-
});
|
|
540
|
-
this.syncChain = work;
|
|
541
|
-
return work;
|
|
542
|
-
}
|
|
543
|
-
/**
|
|
544
|
-
* Re-indexes only `ids` for `entityId`: drops each from the index, then
|
|
545
|
-
* re-adds the ones still live in SQLite, so soft-deleted or missing ids end
|
|
546
|
-
* up absent. Costs O(ids), where sync(entityId) costs O(entity) — callers that
|
|
547
|
-
* know what a write touched use this so chunked imports stay linear (#232).
|
|
548
|
-
*
|
|
549
|
-
* Serialized with sync() on the same chain and, like it, never rejects. An
|
|
550
|
-
* entity that is stale or has never been indexed gets a full rebuild instead
|
|
551
|
-
* — including on an empty `ids` list, which is a no-op only for an entity
|
|
552
|
-
* the index already tracks, and even then one that waits for rebuilds
|
|
553
|
-
* already queued on the chain.
|
|
554
|
-
*/
|
|
555
|
-
async syncEntries(entityId, ids) {
|
|
556
|
-
if (this.indexStrategy.drain) return this.drainTurn(entityId);
|
|
557
|
-
const uniqueIds = [...new Set(ids)];
|
|
558
|
-
const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
|
|
559
|
-
if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
|
|
560
|
-
return this.syncChain;
|
|
561
368
|
}
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
const needsRebuild = () => this.staleEntities.has(entityId) || this.indexStrategy.hasIndexedEntity?.(entityId) !== true;
|
|
567
|
-
if (!needsRebuild()) {
|
|
568
|
-
const rows = await this.entryRepo.findMiniSearchRowsByIds(entityId, uniqueIds);
|
|
569
|
-
if (!this.staleEntities.has(entityId)) {
|
|
570
|
-
await this.indexStrategy.replace(entityId, uniqueIds, rows.map(toIndexDoc));
|
|
571
|
-
await this.indexStrategy.vacuum?.();
|
|
572
|
-
return;
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
await this.rebuildIndex(entityId);
|
|
576
|
-
if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
|
|
577
|
-
this.staleEntities.delete(entityId);
|
|
578
|
-
}
|
|
579
|
-
await this.indexStrategy.vacuum?.();
|
|
580
|
-
} finally {
|
|
581
|
-
this.evictCache(entityId);
|
|
582
|
-
}
|
|
583
|
-
} catch (err) {
|
|
584
|
-
this.staleEntities.add(entityId);
|
|
585
|
-
console.warn(`[WikiMemory] search index incremental sync failed for ${entityId}:`, err);
|
|
369
|
+
if (ch === '"') {
|
|
370
|
+
let j = i + 1;
|
|
371
|
+
while (j < text.length && (text[j] === " " || text[j] === " " || text[j] === "\n" || text[j] === "\r")) {
|
|
372
|
+
j++;
|
|
586
373
|
}
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
374
|
+
const next = j < text.length ? text[j] : "";
|
|
375
|
+
if (!inString) {
|
|
376
|
+
if (next === "}" || next === "]") {
|
|
377
|
+
if (j > i + 1) out += text.slice(i + 1, j);
|
|
378
|
+
out += next;
|
|
379
|
+
i = j + 1;
|
|
380
|
+
stack.pop();
|
|
381
|
+
if (stack.length === 0) {
|
|
382
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
383
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
384
|
+
out = "";
|
|
385
|
+
}
|
|
386
|
+
continue;
|
|
387
|
+
}
|
|
388
|
+
const top = stack[stack.length - 1];
|
|
389
|
+
stringRole = top?.container === "object" && top.expectKey ? "key" : "value";
|
|
390
|
+
out += ch;
|
|
391
|
+
inString = true;
|
|
392
|
+
i++;
|
|
393
|
+
continue;
|
|
394
|
+
}
|
|
395
|
+
const commaOnly = next === ",";
|
|
396
|
+
const isKeyClose = stringRole === "key" && (next === ":" || next === '"' && j + 1 < text.length && text[j + 1] === ":");
|
|
397
|
+
const isValueClose = !commaOnly && (next === "," || next === "}" || next === "]");
|
|
398
|
+
if (commaOnly && bareQuoteCount > 0) ambiguous = true;
|
|
399
|
+
const shouldCommaClose = commaOnly && (bareQuoteCount === 0 || (closeOnEvenParity ? bareQuoteCount % 2 === 0 : bareQuoteCount % 2 === 1));
|
|
400
|
+
const isClosing = isKeyClose || isValueClose || shouldCommaClose;
|
|
401
|
+
if (isClosing) {
|
|
402
|
+
out += ch;
|
|
403
|
+
inString = false;
|
|
404
|
+
const topFrame = stack[stack.length - 1];
|
|
405
|
+
if (topFrame && topFrame.container === "object") {
|
|
406
|
+
topFrame.expectKey = stringRole === "value";
|
|
407
|
+
}
|
|
408
|
+
stringRole = null;
|
|
409
|
+
bareQuoteCount = 0;
|
|
410
|
+
i++;
|
|
411
|
+
continue;
|
|
412
|
+
}
|
|
413
|
+
out += "\\" + ch;
|
|
414
|
+
bareQuoteCount++;
|
|
415
|
+
escapes++;
|
|
416
|
+
i++;
|
|
417
|
+
continue;
|
|
418
|
+
}
|
|
419
|
+
if (!inString && (ch === "{" || ch === "[")) {
|
|
420
|
+
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
421
|
+
out += ch;
|
|
422
|
+
i++;
|
|
423
|
+
continue;
|
|
424
|
+
}
|
|
425
|
+
if (!inString && ch === ",") {
|
|
426
|
+
const topFrame = stack[stack.length - 1];
|
|
427
|
+
if (topFrame && topFrame.container === "object") {
|
|
428
|
+
topFrame.expectKey = true;
|
|
429
|
+
}
|
|
430
|
+
out += ch;
|
|
431
|
+
i++;
|
|
432
|
+
continue;
|
|
433
|
+
}
|
|
434
|
+
if (!inString && (ch === "}" || ch === "]")) {
|
|
435
|
+
out += ch;
|
|
436
|
+
stack.pop();
|
|
437
|
+
if (stack.length === 0) {
|
|
438
|
+
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
439
|
+
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
440
|
+
out = "";
|
|
441
|
+
}
|
|
442
|
+
i++;
|
|
443
|
+
continue;
|
|
444
|
+
}
|
|
445
|
+
out += ch;
|
|
446
|
+
i++;
|
|
599
447
|
}
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
448
|
+
return { success: null, failed: bestFailed, escapes, ambiguous };
|
|
449
|
+
}
|
|
450
|
+
function controlCharEscape(code) {
|
|
451
|
+
switch (code) {
|
|
452
|
+
case 10:
|
|
453
|
+
return "\\n";
|
|
454
|
+
case 9:
|
|
455
|
+
return "\\t";
|
|
456
|
+
case 13:
|
|
457
|
+
return "\\r";
|
|
458
|
+
case 8:
|
|
459
|
+
return "\\b";
|
|
460
|
+
case 12:
|
|
461
|
+
return "\\f";
|
|
462
|
+
default:
|
|
463
|
+
return "\\u" + code.toString(16).padStart(4, "0");
|
|
610
464
|
}
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
465
|
+
}
|
|
466
|
+
function extractParsePosition(err) {
|
|
467
|
+
const match = /position\s+(\d+)/.exec(safeErrorToString(err));
|
|
468
|
+
return match ? Number(match[1]) : null;
|
|
469
|
+
}
|
|
470
|
+
function safeErrorToString(e) {
|
|
471
|
+
let isErrorLike = false;
|
|
472
|
+
try {
|
|
473
|
+
isErrorLike = e instanceof Error;
|
|
474
|
+
} catch {
|
|
475
|
+
}
|
|
476
|
+
if (isErrorLike) {
|
|
477
|
+
const msg = readErrorField(e, "message");
|
|
478
|
+
if (typeof msg === "string" && msg.length > 0) return msg;
|
|
479
|
+
const name = readErrorField(e, "name");
|
|
480
|
+
if (typeof name === "string" && name.length > 0) return name;
|
|
481
|
+
return "[Error]";
|
|
482
|
+
}
|
|
483
|
+
try {
|
|
484
|
+
return String(e);
|
|
485
|
+
} catch {
|
|
486
|
+
try {
|
|
487
|
+
return Object.prototype.toString.call(e);
|
|
488
|
+
} catch {
|
|
489
|
+
return "[unstringifiable error]";
|
|
620
490
|
}
|
|
621
491
|
}
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
this.staleEntities.clear();
|
|
629
|
-
this.staleEpochs.clear();
|
|
492
|
+
}
|
|
493
|
+
function readErrorField(e, key) {
|
|
494
|
+
try {
|
|
495
|
+
return e[key];
|
|
496
|
+
} catch {
|
|
497
|
+
return void 0;
|
|
630
498
|
}
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
499
|
+
}
|
|
500
|
+
function sanitizeRankerError(err, sanitizeRankerErrors) {
|
|
501
|
+
let isErrorLike = false;
|
|
502
|
+
try {
|
|
503
|
+
isErrorLike = err instanceof Error;
|
|
504
|
+
} catch {
|
|
637
505
|
}
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
*/
|
|
641
|
-
async getKeywordScores(query, entityIds, preFilterLimit) {
|
|
642
|
-
const opts = preFilterLimit !== void 0 ? { entityIds, limit: preFilterLimit, preFilterLimit } : { entityIds, limit: Number.MAX_SAFE_INTEGER };
|
|
643
|
-
const results = finiteResults(await this.indexStrategy.search(query, opts));
|
|
644
|
-
if (results.length === 0) return /* @__PURE__ */ new Map();
|
|
645
|
-
const maxScore = Math.max(1, results[0]?.score ?? 1);
|
|
646
|
-
return new Map(results.map((r) => [r.id, r.score / maxScore]));
|
|
506
|
+
if (sanitizeRankerErrors === false) {
|
|
507
|
+
return isErrorLike ? err : new Error(safeErrorToString(err));
|
|
647
508
|
}
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
if (tooLarge && entityCache) {
|
|
658
|
-
this.vectorCache.delete(entityId);
|
|
659
|
-
entityCache = void 0;
|
|
509
|
+
let errLike = null;
|
|
510
|
+
if (isErrorLike) errLike = err;
|
|
511
|
+
let typeName;
|
|
512
|
+
try {
|
|
513
|
+
if (errLike) {
|
|
514
|
+
const rawName = errLike.constructor?.name;
|
|
515
|
+
typeName = typeof rawName === "string" ? rawName : "Error";
|
|
516
|
+
} else {
|
|
517
|
+
typeName = typeof err;
|
|
660
518
|
}
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
519
|
+
} catch {
|
|
520
|
+
typeName = isErrorLike ? "Error" : typeof err;
|
|
521
|
+
}
|
|
522
|
+
let innerCause;
|
|
523
|
+
if (errLike) {
|
|
524
|
+
let cause;
|
|
525
|
+
try {
|
|
526
|
+
cause = errLike.cause;
|
|
527
|
+
} catch {
|
|
528
|
+
cause = void 0;
|
|
664
529
|
}
|
|
665
|
-
|
|
666
|
-
let
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
const cosSim = cosineSimilarity(queryVec, vector);
|
|
673
|
-
if (weight !== void 0) {
|
|
674
|
-
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
675
|
-
score = weight * Math.max(0, cosSim) + (1 - weight) * kwScore;
|
|
676
|
-
} else {
|
|
677
|
-
score = cosSim;
|
|
678
|
-
}
|
|
679
|
-
} else if (weight !== void 0 && weight < 1) {
|
|
680
|
-
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
681
|
-
score = (1 - weight) * kwScore;
|
|
682
|
-
} else {
|
|
683
|
-
score = -2;
|
|
684
|
-
}
|
|
685
|
-
return {
|
|
686
|
-
id: row.id,
|
|
687
|
-
entity_id: row.entity_id,
|
|
688
|
-
score,
|
|
689
|
-
updated_at: row.updated_at,
|
|
690
|
-
access_count: row.access_count
|
|
691
|
-
};
|
|
692
|
-
});
|
|
693
|
-
if (canCache && entityCache && entityCache.size > 0) {
|
|
694
|
-
if (!this.vectorCache.has(entityId)) {
|
|
695
|
-
if (this.vectorCache.size >= _SearchService.MAX_VECTOR_CACHE_ENTITIES) {
|
|
696
|
-
const oldestKey = this.vectorCache.keys().next().value;
|
|
697
|
-
if (oldestKey !== void 0) this.vectorCache.delete(oldestKey);
|
|
698
|
-
}
|
|
699
|
-
this.vectorCache.set(entityId, entityCache);
|
|
530
|
+
if (cause !== void 0) {
|
|
531
|
+
let causeName;
|
|
532
|
+
try {
|
|
533
|
+
const rawCauseName = cause?.constructor?.name;
|
|
534
|
+
causeName = typeof rawCauseName === "string" ? rawCauseName : typeof cause;
|
|
535
|
+
} catch {
|
|
536
|
+
causeName = typeof cause;
|
|
700
537
|
}
|
|
538
|
+
innerCause = new Error(`Caused by: ${causeName}`);
|
|
701
539
|
}
|
|
702
|
-
if (!skipSort) {
|
|
703
|
-
this._tieBreakSort(scored);
|
|
704
|
-
}
|
|
705
|
-
return scored.slice(0, limit);
|
|
706
540
|
}
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
*/
|
|
715
|
-
async rebuildIndex(entityId) {
|
|
716
|
-
const rows = await this.entryRepo.findMiniSearchRows(entityId);
|
|
717
|
-
await this.indexStrategy.replaceEntity(entityId, rows.map(toIndexDoc));
|
|
541
|
+
const sanitized = new Error(
|
|
542
|
+
`VectorRanker ${typeName} (message scrubbed for security)`,
|
|
543
|
+
innerCause ? { cause: innerCause } : void 0
|
|
544
|
+
);
|
|
545
|
+
try {
|
|
546
|
+
sanitized.name = typeName;
|
|
547
|
+
} catch {
|
|
718
548
|
}
|
|
719
|
-
|
|
720
|
-
|
|
549
|
+
return sanitized;
|
|
550
|
+
}
|
|
551
|
+
function safeSlice(value, start, end) {
|
|
552
|
+
const length = value.length;
|
|
553
|
+
let safeStart = start < 0 ? Math.max(length + start, 0) : Math.min(start, length);
|
|
554
|
+
let safeEnd = end === void 0 ? length : end < 0 ? Math.max(length + end, 0) : Math.min(end, length);
|
|
555
|
+
if (safeStart > safeEnd) {
|
|
556
|
+
[safeStart, safeEnd] = [safeEnd, safeStart];
|
|
721
557
|
}
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
if (!Number.isNaN(scoreDiff) && scoreDiff !== 0) return scoreDiff;
|
|
725
|
-
const accessCountDiff = (b.access_count ?? 0) - (a.access_count ?? 0);
|
|
726
|
-
if (accessCountDiff !== 0) return accessCountDiff;
|
|
727
|
-
const updatedAtDiff = (b.updated_at ?? 0) - (a.updated_at ?? 0);
|
|
728
|
-
if (updatedAtDiff !== 0) return updatedAtDiff;
|
|
729
|
-
return a.id.localeCompare(b.id);
|
|
558
|
+
if (safeStart > 0 && safeStart < length && value.charCodeAt(safeStart) >= 56320 && value.charCodeAt(safeStart) <= 57343 && value.charCodeAt(safeStart - 1) >= 55296 && value.charCodeAt(safeStart - 1) <= 56319) {
|
|
559
|
+
safeStart--;
|
|
730
560
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
* which syncEntries changes: a discarded-and-re-added id moves to the end.
|
|
734
|
-
* Every caller truncates these results to a limit, so which ids survive a
|
|
735
|
-
* tie at the boundary must not depend on write history. Re-sort exact
|
|
736
|
-
* score ties by id — the same final tie-break _compareScoredRows applies.
|
|
737
|
-
*/
|
|
738
|
-
_compareSearchResults(a, b) {
|
|
739
|
-
const scoreDiff = b.score - a.score;
|
|
740
|
-
if (!Number.isNaN(scoreDiff) && scoreDiff !== 0) return scoreDiff;
|
|
741
|
-
return a.id.localeCompare(b.id);
|
|
561
|
+
if (safeEnd > 0 && safeEnd < length && value.charCodeAt(safeEnd - 1) >= 55296 && value.charCodeAt(safeEnd - 1) <= 56319 && value.charCodeAt(safeEnd) >= 56320 && value.charCodeAt(safeEnd) <= 57343) {
|
|
562
|
+
safeEnd--;
|
|
742
563
|
}
|
|
743
|
-
|
|
744
|
-
/**
|
|
745
|
-
* Maximum number of entities whose parsed embedding vectors are held in
|
|
746
|
-
* memory. This cap is intentionally conservative so the cache remains safe
|
|
747
|
-
* on memory-constrained runtimes (e.g., mobile/Expo).
|
|
748
|
-
*/
|
|
749
|
-
_SearchService.MAX_VECTOR_CACHE_ENTITIES = 16;
|
|
750
|
-
/**
|
|
751
|
-
* Maximum number of fact vectors cached per entity. Keep this high enough to
|
|
752
|
-
* preserve the parsed-embedding reuse optimization for common mid-sized
|
|
753
|
-
* entities while still maintaining a bounded memory footprint.
|
|
754
|
-
*/
|
|
755
|
-
_SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY = 500;
|
|
756
|
-
var SearchService = _SearchService;
|
|
757
|
-
function finiteResults(results) {
|
|
758
|
-
return results.every((r) => Number.isFinite(r.score)) ? results : results.filter((r) => Number.isFinite(r.score));
|
|
564
|
+
return value.slice(safeStart, safeEnd);
|
|
759
565
|
}
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
this.activeMaintenanceJobs = /* @__PURE__ */ new Set();
|
|
766
|
-
this.activeIngestJobs = /* @__PURE__ */ new Map();
|
|
767
|
-
/**
|
|
768
|
-
* Per-(entityId, sourceHash) promise-chain lock. Serializes ingest races
|
|
769
|
-
* at the application level (the v9 partial UNIQUE index enforces it at
|
|
770
|
-
* the DB level as defense-in-depth). Two callers holding the same hash
|
|
771
|
-
* see FIFO order; different hashes never block one another.
|
|
772
|
-
*
|
|
773
|
-
* Keyed `${entityId}\0${sourceHash}` so distinct (entityId, sourceHash)
|
|
774
|
-
* pairs can never share a lock. A raw concatenation would let
|
|
775
|
-
* ("ab", "c…") collide with ("a", "bc…"); the NUL separator makes that
|
|
776
|
-
* impossible because neither component can contain NUL (entityIds are
|
|
777
|
-
* user-supplied identifiers; sourceHashes are hex digests).
|
|
778
|
-
*/
|
|
779
|
-
this.hashLocks = /* @__PURE__ */ new Map();
|
|
780
|
-
this.statusSubscribers = /* @__PURE__ */ new Map();
|
|
781
|
-
/**
|
|
782
|
-
* Lookup table for acquireLock/releaseLock's dynamic-dispatch branch.
|
|
783
|
-
* Excludes 'ingest' | 'global_reembed' | 'global_import', which those
|
|
784
|
-
* methods already handle via explicit if/else branches before reaching
|
|
785
|
-
* this table.
|
|
786
|
-
*/
|
|
787
|
-
this.lockKeyFns = {
|
|
788
|
-
prune: (id) => this._pruneKey(id),
|
|
789
|
-
librarian: (id) => this._librarianKey(id),
|
|
790
|
-
heal: (id) => this._healKey(id),
|
|
791
|
-
reembed: (id) => this._reembedKey(id),
|
|
792
|
-
import: (id) => this._importKey(id),
|
|
793
|
-
forget: (id) => this._forgetKey(id),
|
|
794
|
-
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
795
|
-
};
|
|
566
|
+
function chunkText(input, maxChunkLength, overlap) {
|
|
567
|
+
const text = input.trim();
|
|
568
|
+
if (text.length === 0) return { chunks: [], truncated: false };
|
|
569
|
+
if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
|
|
570
|
+
throw new Error("maxChunkLength must be an integer >= 2");
|
|
796
571
|
}
|
|
797
|
-
|
|
798
|
-
|
|
572
|
+
if (!Number.isInteger(overlap) || overlap < 0 || overlap >= maxChunkLength) {
|
|
573
|
+
throw new Error("overlap must be a non-negative integer < maxChunkLength");
|
|
799
574
|
}
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
const key = this._hashLockKey(entityId, sourceHash);
|
|
825
|
-
const previous = this.hashLocks.get(key) ?? Promise.resolve();
|
|
826
|
-
let resolveReleased;
|
|
827
|
-
const released = new Promise((resolve) => {
|
|
828
|
-
resolveReleased = resolve;
|
|
829
|
-
});
|
|
830
|
-
let fired = false;
|
|
831
|
-
const release = () => {
|
|
832
|
-
if (fired) return;
|
|
833
|
-
fired = true;
|
|
834
|
-
if (this.hashLocks.get(key) === released) {
|
|
835
|
-
this.hashLocks.delete(key);
|
|
575
|
+
const chunks = [];
|
|
576
|
+
let truncated = false;
|
|
577
|
+
let cursor = 0;
|
|
578
|
+
const halfMax = Math.floor(maxChunkLength / 2);
|
|
579
|
+
while (cursor < text.length) {
|
|
580
|
+
const remaining = text.length - cursor;
|
|
581
|
+
if (remaining <= maxChunkLength) {
|
|
582
|
+
chunks.push(safeSlice(text, cursor, text.length));
|
|
583
|
+
break;
|
|
584
|
+
}
|
|
585
|
+
const windowEnd = cursor + maxChunkLength;
|
|
586
|
+
const minSplit = cursor + halfMax;
|
|
587
|
+
let splitPoint = -1;
|
|
588
|
+
const paraIdx = text.lastIndexOf("\n\n", windowEnd);
|
|
589
|
+
if (paraIdx >= minSplit && paraIdx + 2 <= windowEnd) {
|
|
590
|
+
splitPoint = paraIdx + 2;
|
|
591
|
+
}
|
|
592
|
+
if (splitPoint === -1) {
|
|
593
|
+
let lastTerm = -1;
|
|
594
|
+
for (let i = minSplit; i < windowEnd - 1; i++) {
|
|
595
|
+
const ch = text[i];
|
|
596
|
+
if ((ch === "." || ch === "!" || ch === "?") && /\s/.test(text[i + 1])) {
|
|
597
|
+
lastTerm = i + 2;
|
|
598
|
+
}
|
|
836
599
|
}
|
|
837
|
-
|
|
838
|
-
}
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
_globalImportKey() {
|
|
855
|
-
return `${this.prefix}:import`;
|
|
600
|
+
if (lastTerm !== -1 && lastTerm <= windowEnd) splitPoint = lastTerm;
|
|
601
|
+
}
|
|
602
|
+
if (splitPoint === -1) {
|
|
603
|
+
for (let i = windowEnd - 1; i >= minSplit; i--) {
|
|
604
|
+
if (/\s/.test(text[i])) {
|
|
605
|
+
splitPoint = i + 1;
|
|
606
|
+
break;
|
|
607
|
+
}
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
if (splitPoint === -1) {
|
|
611
|
+
truncated = true;
|
|
612
|
+
splitPoint = windowEnd;
|
|
613
|
+
}
|
|
614
|
+
chunks.push(safeSlice(text, cursor, splitPoint));
|
|
615
|
+
const next = Math.max(splitPoint - overlap, cursor + 1);
|
|
616
|
+
cursor = next;
|
|
856
617
|
}
|
|
857
|
-
|
|
858
|
-
|
|
618
|
+
return { chunks, truncated };
|
|
619
|
+
}
|
|
620
|
+
async function withConcurrency(tasks, limit) {
|
|
621
|
+
const results = new Array(tasks.length);
|
|
622
|
+
let index = 0;
|
|
623
|
+
let failed = false;
|
|
624
|
+
let firstError;
|
|
625
|
+
async function worker() {
|
|
626
|
+
while (index < tasks.length && !failed) {
|
|
627
|
+
const i = index++;
|
|
628
|
+
try {
|
|
629
|
+
results[i] = await tasks[i]();
|
|
630
|
+
} catch (e) {
|
|
631
|
+
if (!failed) {
|
|
632
|
+
failed = true;
|
|
633
|
+
firstError = e;
|
|
634
|
+
}
|
|
635
|
+
return;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
859
638
|
}
|
|
860
|
-
|
|
861
|
-
|
|
639
|
+
const workerCount = tasks.length === 0 ? 0 : Math.min(Math.max(limit, 1), tasks.length);
|
|
640
|
+
await Promise.allSettled(Array.from({ length: workerCount }, worker));
|
|
641
|
+
if (failed) throw firstError;
|
|
642
|
+
return results;
|
|
643
|
+
}
|
|
644
|
+
function clip(value, max) {
|
|
645
|
+
if (typeof value !== "string") return "";
|
|
646
|
+
const s = value.trim();
|
|
647
|
+
return s.length <= max ? s : safeSlice(s, 0, max).trimEnd();
|
|
648
|
+
}
|
|
649
|
+
function validateTags(tags) {
|
|
650
|
+
if (!Array.isArray(tags)) return [];
|
|
651
|
+
return tags.filter((t) => typeof t === "string").map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0 && t.length <= 40).slice(0, 6);
|
|
652
|
+
}
|
|
653
|
+
var MAX_EVIDENCE_QUOTES = 10;
|
|
654
|
+
function normalizeEvidence(raw) {
|
|
655
|
+
if (!Array.isArray(raw)) return void 0;
|
|
656
|
+
const out = [];
|
|
657
|
+
for (const entry of raw) {
|
|
658
|
+
if (typeof entry !== "string") continue;
|
|
659
|
+
const trimmed = entry.trim();
|
|
660
|
+
if (!trimmed) continue;
|
|
661
|
+
out.push(trimmed);
|
|
662
|
+
if (out.length > MAX_EVIDENCE_QUOTES) break;
|
|
862
663
|
}
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
664
|
+
return out;
|
|
665
|
+
}
|
|
666
|
+
function validateFact(fact) {
|
|
667
|
+
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
|
|
668
|
+
const title = clip(fact.title, 80);
|
|
669
|
+
const body = clip(fact.body, 800);
|
|
670
|
+
if (!title || !body) return null;
|
|
671
|
+
let confidence = fact.confidence;
|
|
672
|
+
if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
|
|
673
|
+
const valid = {
|
|
674
|
+
...fact,
|
|
675
|
+
title,
|
|
676
|
+
body,
|
|
677
|
+
confidence,
|
|
678
|
+
tags: validateTags(fact.tags)
|
|
679
|
+
};
|
|
680
|
+
const evidence = normalizeEvidence(fact.evidence);
|
|
681
|
+
if (evidence) valid.evidence = evidence;
|
|
682
|
+
else delete valid.evidence;
|
|
683
|
+
return valid;
|
|
684
|
+
}
|
|
685
|
+
function validateTask(task) {
|
|
686
|
+
if (typeof task?.description !== "string") return null;
|
|
687
|
+
const description = clip(task.description, 200);
|
|
688
|
+
if (!description) return null;
|
|
689
|
+
let priority = task.priority;
|
|
690
|
+
if (typeof priority !== "number" || !isFinite(priority)) priority = 0;
|
|
691
|
+
priority = Math.max(0, Math.min(10, Math.round(priority)));
|
|
692
|
+
return {
|
|
693
|
+
...task,
|
|
694
|
+
description,
|
|
695
|
+
priority
|
|
696
|
+
};
|
|
697
|
+
}
|
|
698
|
+
function factRejectionReason(raw) {
|
|
699
|
+
const fact = raw;
|
|
700
|
+
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return "invalid_shape";
|
|
701
|
+
if (!clip(fact.title, 80)) return "missing_title";
|
|
702
|
+
return "missing_body";
|
|
703
|
+
}
|
|
704
|
+
function taskRejectionReason(raw) {
|
|
705
|
+
const task = raw;
|
|
706
|
+
if (typeof task?.description !== "string") return "invalid_shape";
|
|
707
|
+
return "missing_description";
|
|
708
|
+
}
|
|
709
|
+
function normalizeSourceRef(value) {
|
|
710
|
+
if (typeof value !== "string") return null;
|
|
711
|
+
const cleaned = value.replace(/[^A-Za-z0-9._\- ]/g, "").trim().slice(0, 255).trimEnd();
|
|
712
|
+
return cleaned.length > 0 ? cleaned : null;
|
|
713
|
+
}
|
|
714
|
+
function normalizeSourceHash(value) {
|
|
715
|
+
if (typeof value !== "string") return null;
|
|
716
|
+
return /^[0-9a-f]{64}$/i.test(value) ? value.toLowerCase() : null;
|
|
717
|
+
}
|
|
718
|
+
function titleTokens(title) {
|
|
719
|
+
return new Set(title.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3));
|
|
720
|
+
}
|
|
721
|
+
function jaccardScore(a, b) {
|
|
722
|
+
if (a.size === 0 && b.size === 0) return 0;
|
|
723
|
+
const intersection = new Set([...a].filter((x) => b.has(x)));
|
|
724
|
+
const union = /* @__PURE__ */ new Set([...a, ...b]);
|
|
725
|
+
return intersection.size / union.size;
|
|
726
|
+
}
|
|
727
|
+
|
|
728
|
+
// src/utils/ids.ts
|
|
729
|
+
var _injectedGetRandomValues = null;
|
|
730
|
+
function configureRandomSource(fn) {
|
|
731
|
+
_injectedGetRandomValues = fn;
|
|
732
|
+
}
|
|
733
|
+
function generateId(prefix = "") {
|
|
734
|
+
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
|
|
735
|
+
return prefix + crypto.randomUUID().replace(/-/g, "").substring(0, 24);
|
|
868
736
|
}
|
|
869
|
-
|
|
870
|
-
|
|
737
|
+
if (typeof crypto !== "undefined" && typeof crypto.getRandomValues === "function") {
|
|
738
|
+
const bytes = new Uint8Array(16);
|
|
739
|
+
crypto.getRandomValues(bytes);
|
|
740
|
+
return prefix + Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("").substring(0, 24);
|
|
871
741
|
}
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
* markers across ALL entities (the dimension-promotion clear) must gate on
|
|
877
|
-
* this and defer, rather than resurrect rows an in-flight sweep already
|
|
878
|
-
* classified. See spec 2026-09-05-reembed-lock-scope-design.md §2.
|
|
879
|
-
*
|
|
880
|
-
* The global key `${prefix}:reembed` also ends in ':reembed', so the suffix
|
|
881
|
-
* scan alone would already match it. The explicit check is kept because it
|
|
882
|
-
* documents the intent and survives any future change to the global key's
|
|
883
|
-
* format.
|
|
884
|
-
*/
|
|
885
|
-
isAnyReembedActive() {
|
|
886
|
-
return this.activeMaintenanceJobs.has(this._globalReembedKey()) || this._isAnyMaintenanceActiveWithSuffix(":reembed");
|
|
742
|
+
if (_injectedGetRandomValues) {
|
|
743
|
+
const bytes = new Uint8Array(16);
|
|
744
|
+
_injectedGetRandomValues(bytes);
|
|
745
|
+
return prefix + Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("").substring(0, 24);
|
|
887
746
|
}
|
|
888
|
-
|
|
889
|
-
|
|
747
|
+
throw new Error(
|
|
748
|
+
"generateId: no cryptographically secure random source available (crypto.randomUUID and crypto.getRandomValues are both missing, and no configureRandomSource() injection was provided)."
|
|
749
|
+
);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
// src/utils/ontology.ts
|
|
753
|
+
function emptyManifest() {
|
|
754
|
+
return { node_types: [], edge_types: [] };
|
|
755
|
+
}
|
|
756
|
+
function normalizeTitleKey(title) {
|
|
757
|
+
return title.trim().toLowerCase().replace(/\s+/g, " ");
|
|
758
|
+
}
|
|
759
|
+
function resolveNodeType(raw, manifest) {
|
|
760
|
+
const slug = raw.trim();
|
|
761
|
+
if (!slug) return null;
|
|
762
|
+
const hit = manifest.node_types.find((n) => n.type.toLowerCase() === slug.toLowerCase());
|
|
763
|
+
return hit?.type ?? null;
|
|
764
|
+
}
|
|
765
|
+
function resolveEdgeDefinitions(rawEdgeType, manifest) {
|
|
766
|
+
const slug = rawEdgeType.trim();
|
|
767
|
+
if (!slug) return [];
|
|
768
|
+
return manifest.edge_types.filter((e) => e.type.toLowerCase() === slug.toLowerCase());
|
|
769
|
+
}
|
|
770
|
+
function edgeTripleKey(type, sourceType, targetType) {
|
|
771
|
+
return `${type.trim().toLowerCase()}|${sourceType.trim().toLowerCase()}|${targetType.trim().toLowerCase()}`;
|
|
772
|
+
}
|
|
773
|
+
function typeSatisfies(declaredType, concreteType, manifest) {
|
|
774
|
+
const concrete = concreteType.trim().toLowerCase();
|
|
775
|
+
const declared = declaredType.trim().toLowerCase();
|
|
776
|
+
if (!concrete || !declared) return false;
|
|
777
|
+
if (declared === concrete) return true;
|
|
778
|
+
const def = (manifest.node_types ?? []).find(
|
|
779
|
+
(n) => typeof n?.type === "string" && n.type.trim().toLowerCase() === concrete
|
|
780
|
+
);
|
|
781
|
+
const parent = typeof def?.parent_type === "string" ? def.parent_type.trim().toLowerCase() : "";
|
|
782
|
+
return parent !== "" && parent === declared;
|
|
783
|
+
}
|
|
784
|
+
function edgeTripleAllowed(manifest, edgeType, sourceType, targetType) {
|
|
785
|
+
const wanted = edgeType.trim().toLowerCase();
|
|
786
|
+
return (manifest.edge_types ?? []).some((d) => typeof d?.type === "string" && d.type.trim().toLowerCase() === wanted && typeof d.source_type === "string" && typeSatisfies(d.source_type, sourceType, manifest) && typeof d.target_type === "string" && typeSatisfies(d.target_type, targetType, manifest));
|
|
787
|
+
}
|
|
788
|
+
function validateManifest(manifest) {
|
|
789
|
+
const nodeSlugs = /* @__PURE__ */ new Set();
|
|
790
|
+
for (const node of manifest.node_types ?? []) {
|
|
791
|
+
const type = node.type?.trim();
|
|
792
|
+
if (!type) throw new Error("Ontology node type slug must be non-empty");
|
|
793
|
+
const key = type.toLowerCase();
|
|
794
|
+
if (nodeSlugs.has(key)) throw new Error(`Duplicate node type: ${type}`);
|
|
795
|
+
nodeSlugs.add(key);
|
|
890
796
|
}
|
|
891
|
-
|
|
892
|
-
|
|
797
|
+
const parentOf = /* @__PURE__ */ new Map();
|
|
798
|
+
for (const node of manifest.node_types ?? []) {
|
|
799
|
+
const parent = typeof node.parent_type === "string" ? node.parent_type.trim().toLowerCase() : void 0;
|
|
800
|
+
parentOf.set(node.type.trim().toLowerCase(), parent);
|
|
893
801
|
}
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
802
|
+
for (const node of manifest.node_types ?? []) {
|
|
803
|
+
if (node.parent_type === void 0) continue;
|
|
804
|
+
if (typeof node.parent_type !== "string" || !node.parent_type.trim()) {
|
|
805
|
+
throw new Error(`Ontology parent_type must be a non-empty string when present: ${node.type}`);
|
|
898
806
|
}
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
if (
|
|
908
|
-
|
|
909
|
-
this.activeIngestJobs.set(entityId, refs);
|
|
807
|
+
const parentSlug = node.parent_type.trim().toLowerCase();
|
|
808
|
+
if (parentSlug === node.type.trim().toLowerCase()) {
|
|
809
|
+
throw new Error(`Self-parent: ${node.type}`);
|
|
810
|
+
}
|
|
811
|
+
if (!nodeSlugs.has(parentSlug)) {
|
|
812
|
+
throw new Error(`Parent type not found: ${node.parent_type}`);
|
|
813
|
+
}
|
|
814
|
+
const grandparent = parentOf.get(parentSlug);
|
|
815
|
+
if (grandparent) {
|
|
816
|
+
throw new Error(`Parent chain too deep: ${node.type} \u2192 ${node.parent_type} \u2192 ${grandparent}`);
|
|
910
817
|
}
|
|
911
|
-
refs.add(sourceKey);
|
|
912
818
|
}
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
819
|
+
const edgeKeys = /* @__PURE__ */ new Set();
|
|
820
|
+
const edgeNames = /* @__PURE__ */ new Map();
|
|
821
|
+
for (const edge of manifest.edge_types ?? []) {
|
|
822
|
+
const edgeType = edge.type?.trim();
|
|
823
|
+
const sourceType = edge.source_type?.trim();
|
|
824
|
+
const targetType = edge.target_type?.trim();
|
|
825
|
+
if (!edgeType) throw new Error("Ontology edge type slug must be non-empty");
|
|
826
|
+
if (!sourceType || !targetType || !nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) {
|
|
827
|
+
throw new Error(`Edge type ${edgeType} references unknown node type`);
|
|
828
|
+
}
|
|
829
|
+
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
830
|
+
if (edgeKeys.has(edgeKey)) {
|
|
831
|
+
throw new Error(`Duplicate edge definition: ${edgeType} (${sourceType} \u2192 ${targetType})`);
|
|
832
|
+
}
|
|
833
|
+
edgeKeys.add(edgeKey);
|
|
834
|
+
const canonical = edgeNames.get(edgeType.toLowerCase());
|
|
835
|
+
if (canonical === void 0) {
|
|
836
|
+
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
837
|
+
} else if (canonical !== edgeType) {
|
|
838
|
+
throw new Error(`Inconsistent casing for edge type: ${edgeType} conflicts with ${canonical}`);
|
|
920
839
|
}
|
|
921
840
|
}
|
|
922
|
-
|
|
923
|
-
|
|
841
|
+
}
|
|
842
|
+
function buildDeclaresParentIndex(nodes) {
|
|
843
|
+
const declaresParent = /* @__PURE__ */ new Map();
|
|
844
|
+
for (const n of nodes) {
|
|
845
|
+
const slug = typeof n?.type === "string" ? n.type.trim().toLowerCase() : "";
|
|
846
|
+
if (!slug || declaresParent.has(slug)) continue;
|
|
847
|
+
declaresParent.set(slug, n?.parent_type !== void 0);
|
|
924
848
|
}
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
break;
|
|
991
|
-
case "ingest": {
|
|
992
|
-
const sourceKey = sourceRef ?? "";
|
|
993
|
-
if (this._hasIngestJob(entityId, sourceKey)) blockingOperation = "ingest";
|
|
994
|
-
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
995
|
-
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
996
|
-
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
997
|
-
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
998
|
-
break;
|
|
999
|
-
}
|
|
1000
|
-
}
|
|
1001
|
-
if (blockingOperation) {
|
|
1002
|
-
throw new WikiBusyError(
|
|
1003
|
-
blockingOperation,
|
|
1004
|
-
operation === "global_reembed" || operation === "global_import" ? "*" : entityId
|
|
1005
|
-
);
|
|
849
|
+
return declaresParent;
|
|
850
|
+
}
|
|
851
|
+
function mergeOntologyUpdates(current, updates) {
|
|
852
|
+
const node_types = [...current.node_types];
|
|
853
|
+
const edge_types = [...current.edge_types];
|
|
854
|
+
const nodeSlugs = new Set(node_types.map((n) => n.type.trim().toLowerCase()));
|
|
855
|
+
const edgeKeys = new Set(edge_types.map((e) => edgeTripleKey(e.type, e.source_type, e.target_type)));
|
|
856
|
+
const edgeNames = new Map(edge_types.map((e) => [e.type.trim().toLowerCase(), e.type.trim()]));
|
|
857
|
+
const declaresParent = buildDeclaresParentIndex([
|
|
858
|
+
...current.node_types,
|
|
859
|
+
...updates.node_types ?? []
|
|
860
|
+
]);
|
|
861
|
+
for (const node of updates.node_types ?? []) {
|
|
862
|
+
const type = node?.type?.trim();
|
|
863
|
+
if (!type) continue;
|
|
864
|
+
const key = type.toLowerCase();
|
|
865
|
+
if (nodeSlugs.has(key)) continue;
|
|
866
|
+
const rawParent = typeof node?.parent_type === "string" ? node.parent_type.trim() : "";
|
|
867
|
+
const parentSlug = rawParent.toLowerCase();
|
|
868
|
+
const keepParent = parentSlug !== "" && parentSlug !== key && declaresParent.get(parentSlug) === false;
|
|
869
|
+
node_types.push({
|
|
870
|
+
type,
|
|
871
|
+
description: String(node.description ?? ""),
|
|
872
|
+
...keepParent ? { parent_type: rawParent } : {}
|
|
873
|
+
});
|
|
874
|
+
nodeSlugs.add(key);
|
|
875
|
+
}
|
|
876
|
+
for (const edge of updates.edge_types ?? []) {
|
|
877
|
+
const rawEdgeType = edge?.type?.trim();
|
|
878
|
+
const sourceType = edge?.source_type?.trim();
|
|
879
|
+
const targetType = edge?.target_type?.trim();
|
|
880
|
+
if (!rawEdgeType || !sourceType || !targetType) continue;
|
|
881
|
+
const edgeType = edgeNames.get(rawEdgeType.toLowerCase()) ?? rawEdgeType;
|
|
882
|
+
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
883
|
+
if (edgeKeys.has(edgeKey)) continue;
|
|
884
|
+
if (!nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) continue;
|
|
885
|
+
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
886
|
+
edge_types.push({
|
|
887
|
+
type: edgeType,
|
|
888
|
+
source_type: sourceType,
|
|
889
|
+
target_type: targetType,
|
|
890
|
+
description: String(edge.description ?? "")
|
|
891
|
+
});
|
|
892
|
+
edgeKeys.add(edgeKey);
|
|
893
|
+
}
|
|
894
|
+
return { node_types, edge_types };
|
|
895
|
+
}
|
|
896
|
+
function validateInlineEdges(sourceType, _targetType, edges, manifest, opts) {
|
|
897
|
+
const strict = opts?.strict === true;
|
|
898
|
+
const entityId = opts?.entityId ?? "";
|
|
899
|
+
const drops = opts?.drops;
|
|
900
|
+
const drop = (reason, edgeType) => {
|
|
901
|
+
drops?.push({ reason, sourceId: null, edgeType, sourceNodeType: sourceType, targetNodeType: null });
|
|
902
|
+
};
|
|
903
|
+
if (!Array.isArray(edges)) {
|
|
904
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", "");
|
|
905
|
+
drop("invalid_shape", null);
|
|
906
|
+
return [];
|
|
907
|
+
}
|
|
908
|
+
const valid = [];
|
|
909
|
+
for (const edge of edges) {
|
|
910
|
+
if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string") {
|
|
911
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", String(edge?.edge_type ?? ""));
|
|
912
|
+
drop("invalid_shape", typeof edge?.edge_type === "string" ? edge.edge_type : null);
|
|
913
|
+
continue;
|
|
1006
914
|
}
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
} else {
|
|
1014
|
-
this.activeMaintenanceJobs.add(this.lockKeyFns[operation](entityId));
|
|
915
|
+
const defs = resolveEdgeDefinitions(edge.edge_type, manifest);
|
|
916
|
+
const match = defs.find((d) => typeSatisfies(d.source_type, sourceType, manifest));
|
|
917
|
+
if (!match) {
|
|
918
|
+
if (strict) throw new WikiStrictOntologyViolation(entityId, "edge", edge.edge_type);
|
|
919
|
+
drop("type_not_in_manifest", edge.edge_type);
|
|
920
|
+
continue;
|
|
1015
921
|
}
|
|
1016
|
-
|
|
922
|
+
valid.push({ edge_type: match.type, target_title: edge.target_title });
|
|
1017
923
|
}
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
924
|
+
return valid;
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
// src/utils/embedding.ts
|
|
928
|
+
function parseEmbedding(blob, text) {
|
|
929
|
+
if (blob && blob.byteLength > 0) {
|
|
930
|
+
if (blob.byteLength % 4 !== 0) return null;
|
|
931
|
+
const copy = new ArrayBuffer(blob.byteLength);
|
|
932
|
+
new Uint8Array(copy).set(blob);
|
|
933
|
+
const vector = new Float32Array(copy);
|
|
934
|
+
for (const value of vector) {
|
|
935
|
+
if (!Number.isFinite(value)) return null;
|
|
1027
936
|
}
|
|
1028
|
-
|
|
937
|
+
return vector;
|
|
1029
938
|
}
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
case "librarian":
|
|
1038
|
-
case "heal":
|
|
1039
|
-
case "ontologyBackfill":
|
|
1040
|
-
return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1041
|
-
case "prune":
|
|
1042
|
-
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1043
|
-
case "reembed":
|
|
1044
|
-
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey()) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1045
|
-
case "global_reembed":
|
|
1046
|
-
return this.activeMaintenanceJobs.has(this._globalReembedKey()) || this._isAnyMaintenanceActiveWithSuffix(":reembed") || this._isAnyMaintenanceActiveWithSuffix(":prune") || this._isAnyMaintenanceActiveWithSuffix(":librarian") || this._isAnyMaintenanceActiveWithSuffix(":heal") || this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill") || this.activeIngestJobs.size > 0 || this._isAnyMaintenanceActiveWithSuffix(":import") || this._isAnyMaintenanceActiveWithSuffix(":forget");
|
|
1047
|
-
case "import":
|
|
1048
|
-
case "forget": {
|
|
1049
|
-
const selfKey = operation === "import" ? this._importKey(entityId) : this._forgetKey(entityId);
|
|
1050
|
-
return this.activeMaintenanceJobs.has(selfKey) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
939
|
+
if (text) {
|
|
940
|
+
try {
|
|
941
|
+
const arr = JSON.parse(text);
|
|
942
|
+
if (!Array.isArray(arr) || !arr.every((v) => typeof v === "number" && isFinite(v))) return null;
|
|
943
|
+
const vector = new Float32Array(arr);
|
|
944
|
+
for (const value of vector) {
|
|
945
|
+
if (!Number.isFinite(value)) return null;
|
|
1051
946
|
}
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
return this._hasIngestJob(entityId) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1056
|
-
default:
|
|
1057
|
-
return false;
|
|
947
|
+
return vector;
|
|
948
|
+
} catch {
|
|
949
|
+
return null;
|
|
1058
950
|
}
|
|
1059
951
|
}
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
this._notifyStatusSubscribers(entityId);
|
|
1072
|
-
return true;
|
|
952
|
+
return null;
|
|
953
|
+
}
|
|
954
|
+
|
|
955
|
+
// src/utils/cosine.ts
|
|
956
|
+
function cosineSimilarity(a, b) {
|
|
957
|
+
let dot = 0, normA = 0, normB = 0;
|
|
958
|
+
const len = Math.min(a.length, b.length);
|
|
959
|
+
for (let i = 0; i < len; i++) {
|
|
960
|
+
dot += a[i] * b[i];
|
|
961
|
+
normA += a[i] * a[i];
|
|
962
|
+
normB += b[i] * b[i];
|
|
1073
963
|
}
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
* a request that never reached the DB.
|
|
1087
|
-
*/
|
|
1088
|
-
async acquireIngestLocks(entityId, sourceRef, sourceHash) {
|
|
1089
|
-
const releaseHash = await this.acquireHashLock(entityId, sourceHash);
|
|
1090
|
-
let sourceLockAcquired = false;
|
|
1091
|
-
try {
|
|
1092
|
-
this.acquireLock("ingest", entityId, sourceRef);
|
|
1093
|
-
sourceLockAcquired = true;
|
|
1094
|
-
} catch (err) {
|
|
1095
|
-
releaseHash();
|
|
1096
|
-
throw err;
|
|
1097
|
-
}
|
|
1098
|
-
return () => {
|
|
964
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
965
|
+
return denom === 0 ? 0 : dot / denom;
|
|
966
|
+
}
|
|
967
|
+
|
|
968
|
+
// src/utils/indexDoc.ts
|
|
969
|
+
function toIndexDoc(row) {
|
|
970
|
+
return {
|
|
971
|
+
id: row.id,
|
|
972
|
+
entity_id: row.entity_id,
|
|
973
|
+
title: row.title,
|
|
974
|
+
body: row.body,
|
|
975
|
+
tags: (() => {
|
|
1099
976
|
try {
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
977
|
+
const parsed = JSON.parse(row.tags);
|
|
978
|
+
return Array.isArray(parsed) ? parsed.join(" ") : row.tags;
|
|
979
|
+
} catch {
|
|
980
|
+
return row.tags;
|
|
1103
981
|
}
|
|
1104
|
-
}
|
|
982
|
+
})()
|
|
983
|
+
};
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
// src/services/search/UnavailableIndexStrategy.ts
|
|
987
|
+
var UnavailableIndexStrategy = class {
|
|
988
|
+
constructor(cause) {
|
|
989
|
+
this.cause = cause;
|
|
1105
990
|
}
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
*/
|
|
1110
|
-
acquireImportLocks(entityIds) {
|
|
1111
|
-
for (const entityId of entityIds) {
|
|
1112
|
-
if (this.activeMaintenanceJobs.has(this._importKey(entityId))) throw new WikiBusyError("import", entityId);
|
|
1113
|
-
if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) throw new WikiBusyError("librarian", entityId);
|
|
1114
|
-
if (this.activeMaintenanceJobs.has(this._healKey(entityId))) throw new WikiBusyError("heal", entityId);
|
|
1115
|
-
if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) throw new WikiBusyError("ontologyBackfill", entityId);
|
|
1116
|
-
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) throw new WikiBusyError("prune", entityId);
|
|
1117
|
-
if (this._isReembedActive(entityId)) throw new WikiBusyError("reembed", entityId);
|
|
1118
|
-
if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);
|
|
1119
|
-
if (this._isForgetActiveFor(entityId)) throw new WikiBusyError("forget", entityId);
|
|
1120
|
-
}
|
|
1121
|
-
if (this.activeMaintenanceJobs.has(this._globalImportKey())) throw new WikiBusyError("import", "*");
|
|
1122
|
-
this.activeMaintenanceJobs.add(this._globalImportKey());
|
|
1123
|
-
for (const entityId of entityIds) {
|
|
1124
|
-
this.activeMaintenanceJobs.add(this._importKey(entityId));
|
|
1125
|
-
}
|
|
991
|
+
async search(_query, options) {
|
|
992
|
+
if (options.entityIds.length === 0) return [];
|
|
993
|
+
throw new WikiSearchUnavailableError(void 0, this.cause);
|
|
1126
994
|
}
|
|
1127
|
-
|
|
1128
|
-
this.activeMaintenanceJobs.delete(this._globalImportKey());
|
|
1129
|
-
for (const entityId of entityIds) {
|
|
1130
|
-
this.activeMaintenanceJobs.delete(this._importKey(entityId));
|
|
1131
|
-
}
|
|
995
|
+
async replace(_entityId, _ids, _documents) {
|
|
1132
996
|
}
|
|
1133
|
-
|
|
1134
|
-
return {
|
|
1135
|
-
ingesting: this._isIngestActiveFor(entityId),
|
|
1136
|
-
librarian: this.activeMaintenanceJobs.has(this._librarianKey(entityId)),
|
|
1137
|
-
heal: this.activeMaintenanceJobs.has(this._healKey(entityId))
|
|
1138
|
-
};
|
|
997
|
+
async replaceEntity(_entityId, _documents) {
|
|
1139
998
|
}
|
|
1140
|
-
|
|
1141
|
-
const initial = this.getEntityStatus(entityId);
|
|
1142
|
-
let set = this.statusSubscribers.get(entityId);
|
|
1143
|
-
if (!set) {
|
|
1144
|
-
set = /* @__PURE__ */ new Set();
|
|
1145
|
-
this.statusSubscribers.set(entityId, set);
|
|
1146
|
-
}
|
|
1147
|
-
const entry = { callback, last: this._copyEntityStatus(initial) };
|
|
1148
|
-
set.add(entry);
|
|
1149
|
-
try {
|
|
1150
|
-
callback(this._copyEntityStatus(initial));
|
|
1151
|
-
} catch (err) {
|
|
1152
|
-
console.error(`[JobManager] callback error for entityId="${entityId}" during initial emission`, err);
|
|
1153
|
-
}
|
|
1154
|
-
let active = true;
|
|
1155
|
-
return () => {
|
|
1156
|
-
if (!active) return;
|
|
1157
|
-
active = false;
|
|
1158
|
-
const s = this.statusSubscribers.get(entityId);
|
|
1159
|
-
if (!s) return;
|
|
1160
|
-
s.delete(entry);
|
|
1161
|
-
if (s.size === 0) this.statusSubscribers.delete(entityId);
|
|
1162
|
-
};
|
|
999
|
+
async replaceAll(_documents) {
|
|
1163
1000
|
}
|
|
1164
|
-
|
|
1165
|
-
return
|
|
1001
|
+
hasIndexedEntity(_entityId) {
|
|
1002
|
+
return false;
|
|
1166
1003
|
}
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1004
|
+
async drain(_opts) {
|
|
1005
|
+
}
|
|
1006
|
+
};
|
|
1007
|
+
|
|
1008
|
+
// src/services/SearchService.ts
|
|
1009
|
+
var _SearchService = class _SearchService {
|
|
1010
|
+
constructor(entryRepo, indexStrategy) {
|
|
1011
|
+
this.entryRepo = entryRepo;
|
|
1012
|
+
this.vectorCache = /* @__PURE__ */ new Map();
|
|
1013
|
+
/**
|
|
1014
|
+
* Serializes rebuilds. `rebuildIndex` awaits a repository read between
|
|
1015
|
+
* snapshotting the previous id set and discarding it, so two concurrent
|
|
1016
|
+
* sync() calls for one entity can interleave: a slow, stale read lands last
|
|
1017
|
+
* and discards documents the fresh read just added. Chaining also keeps
|
|
1018
|
+
* discard()/addAll() out of each other's way, which is what accrued the
|
|
1019
|
+
* auto-vacuum debt behind the TypeError in #64.
|
|
1020
|
+
*/
|
|
1021
|
+
this.syncChain = Promise.resolve();
|
|
1022
|
+
/**
|
|
1023
|
+
* Entities whose index may have drifted from SQLite: an incremental update
|
|
1024
|
+
* failed part-way, or core wrote rows it could not index (upsertGraph runs in
|
|
1025
|
+
* the host's transaction). Their next syncEntries rebuilds the entity in full.
|
|
1026
|
+
* See spec 2026-09-28 §5.
|
|
1027
|
+
*/
|
|
1028
|
+
this.staleEntities = /* @__PURE__ */ new Set();
|
|
1029
|
+
/**
|
|
1030
|
+
* Per-entity count of markStale() calls. A rebuild clears an entity's stale
|
|
1031
|
+
* flag only when the count it snapshotted before its read still holds
|
|
1032
|
+
* afterwards: markStale() runs inside the host's still-open transaction, so
|
|
1033
|
+
* a call landing mid-read belongs to rows the read cannot have seen, and
|
|
1034
|
+
* its flag must outlive the turn (#233 review).
|
|
1035
|
+
*/
|
|
1036
|
+
this.staleEpochs = /* @__PURE__ */ new Map();
|
|
1037
|
+
this.indexStrategy = indexStrategy;
|
|
1038
|
+
}
|
|
1039
|
+
/**
|
|
1040
|
+
* Install the strategy resolved by `WikiMemory.setup()`. Before setup, the
|
|
1041
|
+
* service runs on an empty MiniSearch placeholder, as it always has.
|
|
1042
|
+
*/
|
|
1043
|
+
setIndexStrategy(strategy) {
|
|
1044
|
+
this.indexStrategy = strategy;
|
|
1045
|
+
}
|
|
1046
|
+
/** False while the installed strategy is the degraded UnavailableIndexStrategy (spec #280 Change 4). */
|
|
1047
|
+
isKeywordAvailable() {
|
|
1048
|
+
return !(this.indexStrategy instanceof UnavailableIndexStrategy);
|
|
1049
|
+
}
|
|
1050
|
+
/** True when the installed strategy drains a durable ledger (FTS5, or the degraded stand-in). */
|
|
1051
|
+
hasLedger() {
|
|
1052
|
+
return this.indexStrategy.drain !== void 0;
|
|
1053
|
+
}
|
|
1054
|
+
/**
|
|
1055
|
+
* One serialized drain turn for strategies with a durable ledger. Like
|
|
1056
|
+
* sync(), it never rejects: the keyword index is rebuildable, and a failed
|
|
1057
|
+
* drain leaves the ledger rows in place for the next turn. Resolves false
|
|
1058
|
+
* when the drain failed.
|
|
1059
|
+
*/
|
|
1060
|
+
drainTurn(entityId, opts) {
|
|
1061
|
+
const work = this.syncChain.then(async () => {
|
|
1178
1062
|
try {
|
|
1179
|
-
|
|
1063
|
+
try {
|
|
1064
|
+
await this.indexStrategy.drain(opts);
|
|
1065
|
+
} finally {
|
|
1066
|
+
this.evictCache(entityId);
|
|
1067
|
+
}
|
|
1068
|
+
return true;
|
|
1180
1069
|
} catch (err) {
|
|
1181
|
-
console.
|
|
1070
|
+
console.warn(`[WikiMemory] search index drain failed for ${entityId ?? "*"}:`, err);
|
|
1071
|
+
return false;
|
|
1182
1072
|
}
|
|
1183
|
-
}
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
// src/utils/pure.ts
|
|
1188
|
-
var MAX_REPAIR_CANDIDATES = 5;
|
|
1189
|
-
function throwNoJsonFound(text, start) {
|
|
1190
|
-
throw new WikiParseError(
|
|
1191
|
-
"No JSON object/array found in LLM response",
|
|
1192
|
-
{ tier: "strict", position: start, slice: text }
|
|
1193
|
-
);
|
|
1194
|
-
}
|
|
1195
|
-
function parseJsonResponse(text) {
|
|
1196
|
-
const firstBrace = text.indexOf("{");
|
|
1197
|
-
const firstBracket = text.indexOf("[");
|
|
1198
|
-
let start;
|
|
1199
|
-
let openChar;
|
|
1200
|
-
const useBrace = firstBrace !== -1 && (firstBracket === -1 || firstBrace < firstBracket);
|
|
1201
|
-
if (useBrace) {
|
|
1202
|
-
start = firstBrace;
|
|
1203
|
-
openChar = "{";
|
|
1204
|
-
} else if (firstBracket !== -1) {
|
|
1205
|
-
start = firstBracket;
|
|
1206
|
-
openChar = "[";
|
|
1207
|
-
} else {
|
|
1208
|
-
throwNoJsonFound(text, null);
|
|
1073
|
+
});
|
|
1074
|
+
this.syncChain = work.then(() => void 0);
|
|
1075
|
+
return work;
|
|
1209
1076
|
}
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1077
|
+
/**
|
|
1078
|
+
* Rebuilds the search index and clears the vector cache for a given entity.
|
|
1079
|
+
* A direct replacement for manually syncing state after a DB transaction.
|
|
1080
|
+
*
|
|
1081
|
+
* Rebuilds are serialized per instance and never reject: the keyword index
|
|
1082
|
+
* is a rebuildable cache over SQLite, so degraded keyword search is the
|
|
1083
|
+
* correct failure mode and killing the host process is not.
|
|
1084
|
+
*
|
|
1085
|
+
* With a ledger strategy, `sync()` runs one full quantum bounded by
|
|
1086
|
+
* `opts.deadline`; `sync(entityId)` is a write turn (ledger only).
|
|
1087
|
+
* `opts.ids` are the caller's touched ids: a ledger strategy drains them
|
|
1088
|
+
* first, to completion; MiniSearch ignores them and rebuilds the entity in
|
|
1089
|
+
* one scan.
|
|
1090
|
+
*/
|
|
1091
|
+
async sync(entityId, opts) {
|
|
1092
|
+
if (this.indexStrategy.drain) {
|
|
1093
|
+
await this.drainTurn(entityId, entityId === void 0 ? { mode: "full", deadline: opts?.deadline, ids: opts?.ids } : { mode: "ledger", entityId, ids: opts?.ids });
|
|
1094
|
+
return;
|
|
1215
1095
|
}
|
|
1096
|
+
const work = this.syncChain.then(async () => {
|
|
1097
|
+
try {
|
|
1098
|
+
try {
|
|
1099
|
+
const epochsBefore = new Map(this.staleEpochs);
|
|
1100
|
+
if (entityId === void 0) {
|
|
1101
|
+
const rows = await this.entryRepo.findMiniSearchRows();
|
|
1102
|
+
await this.indexStrategy.replaceAll(rows.map(toIndexDoc));
|
|
1103
|
+
for (const id of [...this.staleEntities]) {
|
|
1104
|
+
if ((this.staleEpochs.get(id) ?? 0) === (epochsBefore.get(id) ?? 0)) {
|
|
1105
|
+
this.staleEntities.delete(id);
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
} else {
|
|
1109
|
+
await this.rebuildIndex(entityId);
|
|
1110
|
+
if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
|
|
1111
|
+
this.staleEntities.delete(entityId);
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
await this.indexStrategy.vacuum?.();
|
|
1115
|
+
} finally {
|
|
1116
|
+
this.evictCache(entityId);
|
|
1117
|
+
}
|
|
1118
|
+
} catch (err) {
|
|
1119
|
+
console.warn(`[WikiMemory] search index rebuild failed for ${entityId ?? "*"}:`, err);
|
|
1120
|
+
}
|
|
1121
|
+
});
|
|
1122
|
+
this.syncChain = work;
|
|
1123
|
+
return work;
|
|
1216
1124
|
}
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1125
|
+
/**
|
|
1126
|
+
* Re-indexes only `ids` for `entityId`: drops each from the index, then
|
|
1127
|
+
* re-adds the ones still live in SQLite, so soft-deleted or missing ids end
|
|
1128
|
+
* up absent. Costs O(ids), where sync(entityId) costs O(entity) — callers that
|
|
1129
|
+
* know what a write touched use this so chunked imports stay linear (#232).
|
|
1130
|
+
*
|
|
1131
|
+
* Serialized with sync() on the same chain and, like it, never rejects. An
|
|
1132
|
+
* entity that is stale or has never been indexed gets a full rebuild instead
|
|
1133
|
+
* — including on an empty `ids` list, which is a no-op only for an entity
|
|
1134
|
+
* the index already tracks, and even then one that waits for rebuilds
|
|
1135
|
+
* already queued on the chain.
|
|
1136
|
+
*
|
|
1137
|
+
* Resolves false only when a ledger drain failed: the caller's ids were not
|
|
1138
|
+
* applied (their ledger rows remain), so a caller holding them retries.
|
|
1139
|
+
*/
|
|
1140
|
+
async syncEntries(entityId, ids) {
|
|
1141
|
+
if (this.indexStrategy.drain) return this.drainTurn(entityId, { mode: "ledger", entityId, ids: [...new Set(ids)] });
|
|
1142
|
+
const uniqueIds = [...new Set(ids)];
|
|
1143
|
+
const indexed = this.indexStrategy.hasIndexedEntity?.(entityId);
|
|
1144
|
+
if (uniqueIds.length === 0 && !this.staleEntities.has(entityId) && indexed === true) {
|
|
1145
|
+
await this.syncChain;
|
|
1146
|
+
return true;
|
|
1147
|
+
}
|
|
1148
|
+
const work = this.syncChain.then(async () => {
|
|
1149
|
+
try {
|
|
1150
|
+
try {
|
|
1151
|
+
const epochsBefore = new Map(this.staleEpochs);
|
|
1152
|
+
const needsRebuild = () => this.staleEntities.has(entityId) || this.indexStrategy.hasIndexedEntity?.(entityId) !== true;
|
|
1153
|
+
if (!needsRebuild()) {
|
|
1154
|
+
const rows = await this.entryRepo.findMiniSearchRowsByIds(entityId, uniqueIds);
|
|
1155
|
+
if (!this.staleEntities.has(entityId)) {
|
|
1156
|
+
await this.indexStrategy.replace(entityId, uniqueIds, rows.map(toIndexDoc));
|
|
1157
|
+
await this.indexStrategy.vacuum?.();
|
|
1158
|
+
return;
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
await this.rebuildIndex(entityId);
|
|
1162
|
+
if ((this.staleEpochs.get(entityId) ?? 0) === (epochsBefore.get(entityId) ?? 0)) {
|
|
1163
|
+
this.staleEntities.delete(entityId);
|
|
1164
|
+
}
|
|
1165
|
+
await this.indexStrategy.vacuum?.();
|
|
1166
|
+
} finally {
|
|
1167
|
+
this.evictCache(entityId);
|
|
1168
|
+
}
|
|
1169
|
+
} catch (err) {
|
|
1170
|
+
this.staleEntities.add(entityId);
|
|
1171
|
+
console.warn(`[WikiMemory] search index incremental sync failed for ${entityId}:`, err);
|
|
1172
|
+
}
|
|
1173
|
+
});
|
|
1174
|
+
this.syncChain = work;
|
|
1175
|
+
await work;
|
|
1176
|
+
return true;
|
|
1225
1177
|
}
|
|
1226
|
-
|
|
1227
|
-
|
|
1178
|
+
/**
|
|
1179
|
+
* Forces the entity's next syncEntries to rebuild it in full. For writes core
|
|
1180
|
+
* cannot index itself, such as upsertGraph inside a host transaction.
|
|
1181
|
+
*/
|
|
1182
|
+
markStale(entityId) {
|
|
1183
|
+
if (this.indexStrategy.drain) return;
|
|
1184
|
+
this.staleEntities.add(entityId);
|
|
1185
|
+
this.staleEpochs.set(entityId, (this.staleEpochs.get(entityId) ?? 0) + 1);
|
|
1228
1186
|
}
|
|
1229
|
-
|
|
1230
|
-
|
|
1187
|
+
/**
|
|
1188
|
+
* Runs syncEntries(id, []) for each entity marked stale when this is called,
|
|
1189
|
+
* so only those entities are rebuilt. Each rebuild still takes its own turn
|
|
1190
|
+
* on the sync chain, and like syncEntries this never rejects. With nothing
|
|
1191
|
+
* stale it resolves at once, without waiting on work already on the chain.
|
|
1192
|
+
* Resolves false only when a ledger drain failed (see syncEntries).
|
|
1193
|
+
*/
|
|
1194
|
+
async syncStale(opts) {
|
|
1195
|
+
if (this.indexStrategy.drain) return this.drainTurn(void 0, { mode: "full", ids: opts?.ids });
|
|
1196
|
+
const ids = [...this.staleEntities];
|
|
1197
|
+
await Promise.all(ids.map((id) => this.syncEntries(id, [])));
|
|
1198
|
+
return true;
|
|
1231
1199
|
}
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
)
|
|
1200
|
+
/**
|
|
1201
|
+
* Clears the parsed vector cache. Useful for mid-loop flush guarantees
|
|
1202
|
+
* or memory pressure evictions.
|
|
1203
|
+
*/
|
|
1204
|
+
evictCache(entityId) {
|
|
1205
|
+
if (entityId) {
|
|
1206
|
+
this.vectorCache.delete(entityId);
|
|
1207
|
+
} else {
|
|
1208
|
+
this.vectorCache.clear();
|
|
1209
|
+
}
|
|
1238
1210
|
}
|
|
1239
|
-
|
|
1240
|
-
|
|
1211
|
+
/**
|
|
1212
|
+
* Fully resets the search service.
|
|
1213
|
+
*/
|
|
1214
|
+
async clearAll() {
|
|
1215
|
+
this.vectorCache.clear();
|
|
1216
|
+
await this.indexStrategy.replaceAll([]);
|
|
1217
|
+
this.staleEntities.clear();
|
|
1218
|
+
this.staleEpochs.clear();
|
|
1241
1219
|
}
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
)
|
|
1246
|
-
}
|
|
1247
|
-
|
|
1248
|
-
const closeChar = openChar === "{" ? "}" : "]";
|
|
1249
|
-
let depth = 0;
|
|
1250
|
-
let inString = false;
|
|
1251
|
-
let escape = false;
|
|
1252
|
-
let end = -1;
|
|
1253
|
-
for (let i = start; i < text.length; i++) {
|
|
1254
|
-
const ch = text[i];
|
|
1255
|
-
if (escape) {
|
|
1256
|
-
escape = false;
|
|
1257
|
-
continue;
|
|
1258
|
-
}
|
|
1259
|
-
if (ch === "\\" && inString) {
|
|
1260
|
-
escape = true;
|
|
1261
|
-
continue;
|
|
1262
|
-
}
|
|
1263
|
-
if (ch === '"') {
|
|
1264
|
-
inString = !inString;
|
|
1265
|
-
continue;
|
|
1266
|
-
}
|
|
1267
|
-
if (inString) continue;
|
|
1268
|
-
if (ch === openChar) {
|
|
1269
|
-
depth++;
|
|
1270
|
-
continue;
|
|
1271
|
-
}
|
|
1272
|
-
if (ch === closeChar) {
|
|
1273
|
-
depth--;
|
|
1274
|
-
if (depth === 0) {
|
|
1275
|
-
end = i;
|
|
1276
|
-
break;
|
|
1277
|
-
}
|
|
1278
|
-
}
|
|
1220
|
+
/**
|
|
1221
|
+
* Executes a keyword search against the active index strategy.
|
|
1222
|
+
*/
|
|
1223
|
+
async searchKeyword(query, entityIds, limit) {
|
|
1224
|
+
const results = finiteResults(await this.indexStrategy.search(query, { entityIds, limit }));
|
|
1225
|
+
return results.sort((a, b) => this._compareSearchResults(a, b)).slice(0, limit);
|
|
1279
1226
|
}
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
let out = "";
|
|
1290
|
-
let bareQuoteCount = 0;
|
|
1291
|
-
let bestFailed = null;
|
|
1292
|
-
let escapes = 0;
|
|
1293
|
-
let ambiguous = false;
|
|
1294
|
-
let bestSuccess = null;
|
|
1295
|
-
let attempts = 0;
|
|
1296
|
-
function tryEmitCandidate(candidate) {
|
|
1297
|
-
attempts++;
|
|
1298
|
-
try {
|
|
1299
|
-
bestSuccess = JSON.parse(candidate);
|
|
1300
|
-
return true;
|
|
1301
|
-
} catch (err) {
|
|
1302
|
-
if (bestFailed === null) {
|
|
1303
|
-
bestFailed = {
|
|
1304
|
-
candidate,
|
|
1305
|
-
position: extractParsePosition(err),
|
|
1306
|
-
message: safeErrorToString(err)
|
|
1307
|
-
};
|
|
1308
|
-
}
|
|
1309
|
-
return false;
|
|
1310
|
-
}
|
|
1227
|
+
/**
|
|
1228
|
+
* Pre-fetches keyword scores for candidate hydration, used during hybrid weighting.
|
|
1229
|
+
*/
|
|
1230
|
+
async getKeywordScores(query, entityIds, preFilterLimit) {
|
|
1231
|
+
const opts = preFilterLimit !== void 0 ? { entityIds, limit: preFilterLimit, preFilterLimit } : { entityIds, limit: Number.MAX_SAFE_INTEGER };
|
|
1232
|
+
const results = finiteResults(await this.indexStrategy.search(query, opts));
|
|
1233
|
+
if (results.length === 0) return /* @__PURE__ */ new Map();
|
|
1234
|
+
const maxScore = Math.max(1, results[0]?.score ?? 1);
|
|
1235
|
+
return new Map(results.map((r) => [r.id, r.score / maxScore]));
|
|
1311
1236
|
}
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
escape = false;
|
|
1325
|
-
i++;
|
|
1326
|
-
continue;
|
|
1327
|
-
}
|
|
1328
|
-
if (inString && ch === "\\") {
|
|
1329
|
-
out += ch;
|
|
1330
|
-
escape = true;
|
|
1331
|
-
i++;
|
|
1332
|
-
continue;
|
|
1237
|
+
/**
|
|
1238
|
+
* Score candidate rows using in-process JS cosine similarity.
|
|
1239
|
+
* Applies hybrid blending (if weight set) and tie-break sorting before returning.
|
|
1240
|
+
*/
|
|
1241
|
+
async rankSemantic(args) {
|
|
1242
|
+
const queryVec = args.queryVec instanceof Float32Array ? args.queryVec.slice() : Array.from(args.queryVec);
|
|
1243
|
+
const { entityId, candidateRows, weight, miniSearchScores, populateCache, limit, skipSort } = args;
|
|
1244
|
+
let entityCache = this.vectorCache.get(entityId);
|
|
1245
|
+
const tooLarge = populateCache && candidateRows.length > _SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY;
|
|
1246
|
+
if (tooLarge && entityCache) {
|
|
1247
|
+
this.vectorCache.delete(entityId);
|
|
1248
|
+
entityCache = void 0;
|
|
1333
1249
|
}
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
out += controlCharEscape(code);
|
|
1338
|
-
i++;
|
|
1339
|
-
continue;
|
|
1340
|
-
}
|
|
1250
|
+
const canCache = populateCache && !tooLarge;
|
|
1251
|
+
if (canCache && !entityCache) {
|
|
1252
|
+
entityCache = /* @__PURE__ */ new Map();
|
|
1341
1253
|
}
|
|
1342
|
-
|
|
1343
|
-
let
|
|
1344
|
-
|
|
1345
|
-
|
|
1254
|
+
const scored = candidateRows.map((row) => {
|
|
1255
|
+
let vector = entityCache?.get(row.id) ?? parseEmbedding(row.embedding_blob, row.embedding);
|
|
1256
|
+
if (vector && canCache && entityCache && !entityCache.has(row.id)) {
|
|
1257
|
+
entityCache.set(row.id, vector);
|
|
1346
1258
|
}
|
|
1347
|
-
|
|
1348
|
-
if (
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
if (tryEmitCandidate(out)) return { success: bestSuccess, failed: bestFailed, escapes, ambiguous };
|
|
1356
|
-
if (attempts >= MAX_REPAIR_CANDIDATES) break;
|
|
1357
|
-
out = "";
|
|
1358
|
-
}
|
|
1359
|
-
continue;
|
|
1259
|
+
let score = 0;
|
|
1260
|
+
if (vector && vector.length === queryVec.length) {
|
|
1261
|
+
const cosSim = cosineSimilarity(queryVec, vector);
|
|
1262
|
+
if (weight !== void 0) {
|
|
1263
|
+
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
1264
|
+
score = weight * Math.max(0, cosSim) + (1 - weight) * kwScore;
|
|
1265
|
+
} else {
|
|
1266
|
+
score = cosSim;
|
|
1360
1267
|
}
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
continue;
|
|
1268
|
+
} else if (weight !== void 0 && weight < 1) {
|
|
1269
|
+
const kwScore = miniSearchScores?.get(row.id) ?? 0;
|
|
1270
|
+
score = (1 - weight) * kwScore;
|
|
1271
|
+
} else {
|
|
1272
|
+
score = -2;
|
|
1367
1273
|
}
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
if (
|
|
1379
|
-
|
|
1274
|
+
return {
|
|
1275
|
+
id: row.id,
|
|
1276
|
+
entity_id: row.entity_id,
|
|
1277
|
+
score,
|
|
1278
|
+
updated_at: row.updated_at,
|
|
1279
|
+
access_count: row.access_count
|
|
1280
|
+
};
|
|
1281
|
+
});
|
|
1282
|
+
if (canCache && entityCache && entityCache.size > 0) {
|
|
1283
|
+
if (!this.vectorCache.has(entityId)) {
|
|
1284
|
+
if (this.vectorCache.size >= _SearchService.MAX_VECTOR_CACHE_ENTITIES) {
|
|
1285
|
+
const oldestKey = this.vectorCache.keys().next().value;
|
|
1286
|
+
if (oldestKey !== void 0) this.vectorCache.delete(oldestKey);
|
|
1380
1287
|
}
|
|
1381
|
-
|
|
1382
|
-
bareQuoteCount = 0;
|
|
1383
|
-
i++;
|
|
1384
|
-
continue;
|
|
1288
|
+
this.vectorCache.set(entityId, entityCache);
|
|
1385
1289
|
}
|
|
1386
|
-
out += "\\" + ch;
|
|
1387
|
-
bareQuoteCount++;
|
|
1388
|
-
escapes++;
|
|
1389
|
-
i++;
|
|
1390
|
-
continue;
|
|
1391
|
-
}
|
|
1392
|
-
if (!inString && (ch === "{" || ch === "[")) {
|
|
1393
|
-
stack.push({ container: ch === "{" ? "object" : "array", expectKey: true });
|
|
1394
|
-
out += ch;
|
|
1395
|
-
i++;
|
|
1396
|
-
continue;
|
|
1397
1290
|
}
|
|
1398
|
-
if (!
|
|
1399
|
-
|
|
1400
|
-
if (topFrame && topFrame.container === "object") {
|
|
1401
|
-
topFrame.expectKey = true;
|
|
1402
|
-
}
|
|
1403
|
-
out += ch;
|
|
1404
|
-
i++;
|
|
1405
|
-
continue;
|
|
1291
|
+
if (!skipSort) {
|
|
1292
|
+
this._tieBreakSort(scored);
|
|
1406
1293
|
}
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1294
|
+
return scored.slice(0, limit);
|
|
1295
|
+
}
|
|
1296
|
+
// --- Internal Index Management ---
|
|
1297
|
+
/**
|
|
1298
|
+
* Rebuild a single entity's index from scratch. Reads every row for
|
|
1299
|
+
* `entityId` from the repository (already filtered by `deleted_at IS NULL`)
|
|
1300
|
+
* and hands the lot to `indexStrategy.replaceEntity`, which owns the
|
|
1301
|
+
* drop-then-add bookkeeping so callers don't need to know the strategy's
|
|
1302
|
+
* id-tracking shape.
|
|
1303
|
+
*/
|
|
1304
|
+
async rebuildIndex(entityId) {
|
|
1305
|
+
const rows = await this.entryRepo.findMiniSearchRows(entityId);
|
|
1306
|
+
await this.indexStrategy.replaceEntity(entityId, rows.map(toIndexDoc));
|
|
1307
|
+
}
|
|
1308
|
+
_tieBreakSort(items) {
|
|
1309
|
+
items.sort((a, b) => this._compareScoredRows(a, b));
|
|
1310
|
+
}
|
|
1311
|
+
_compareScoredRows(a, b) {
|
|
1312
|
+
const scoreDiff = b.score - a.score;
|
|
1313
|
+
if (!Number.isNaN(scoreDiff) && scoreDiff !== 0) return scoreDiff;
|
|
1314
|
+
const accessCountDiff = (b.access_count ?? 0) - (a.access_count ?? 0);
|
|
1315
|
+
if (accessCountDiff !== 0) return accessCountDiff;
|
|
1316
|
+
const updatedAtDiff = (b.updated_at ?? 0) - (a.updated_at ?? 0);
|
|
1317
|
+
if (updatedAtDiff !== 0) return updatedAtDiff;
|
|
1318
|
+
return a.id.localeCompare(b.id);
|
|
1319
|
+
}
|
|
1320
|
+
/**
|
|
1321
|
+
* The keyword index breaks equal-score ties by internal insertion order,
|
|
1322
|
+
* which syncEntries changes: a discarded-and-re-added id moves to the end.
|
|
1323
|
+
* Every caller truncates these results to a limit, so which ids survive a
|
|
1324
|
+
* tie at the boundary must not depend on write history. Re-sort exact
|
|
1325
|
+
* score ties by id — the same final tie-break _compareScoredRows applies.
|
|
1326
|
+
*/
|
|
1327
|
+
_compareSearchResults(a, b) {
|
|
1328
|
+
const scoreDiff = b.score - a.score;
|
|
1329
|
+
if (!Number.isNaN(scoreDiff) && scoreDiff !== 0) return scoreDiff;
|
|
1330
|
+
return a.id.localeCompare(b.id);
|
|
1331
|
+
}
|
|
1332
|
+
};
|
|
1333
|
+
/**
|
|
1334
|
+
* Maximum number of entities whose parsed embedding vectors are held in
|
|
1335
|
+
* memory. This cap is intentionally conservative so the cache remains safe
|
|
1336
|
+
* on memory-constrained runtimes (e.g., mobile/Expo).
|
|
1337
|
+
*/
|
|
1338
|
+
_SearchService.MAX_VECTOR_CACHE_ENTITIES = 16;
|
|
1339
|
+
/**
|
|
1340
|
+
* Maximum number of fact vectors cached per entity. Keep this high enough to
|
|
1341
|
+
* preserve the parsed-embedding reuse optimization for common mid-sized
|
|
1342
|
+
* entities while still maintaining a bounded memory footprint.
|
|
1343
|
+
*/
|
|
1344
|
+
_SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY = 500;
|
|
1345
|
+
var SearchService = _SearchService;
|
|
1346
|
+
function finiteResults(results) {
|
|
1347
|
+
return results.every((r) => Number.isFinite(r.score)) ? results : results.filter((r) => Number.isFinite(r.score));
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
// src/services/JobManager.ts
|
|
1351
|
+
var JobManager = class {
|
|
1352
|
+
constructor(prefix) {
|
|
1353
|
+
this.prefix = prefix;
|
|
1354
|
+
this.activeMaintenanceJobs = /* @__PURE__ */ new Set();
|
|
1355
|
+
this.activeIngestJobs = /* @__PURE__ */ new Map();
|
|
1356
|
+
/**
|
|
1357
|
+
* Per-(entityId, sourceHash) promise-chain lock. Serializes ingest races
|
|
1358
|
+
* at the application level (the v9 partial UNIQUE index enforces it at
|
|
1359
|
+
* the DB level as defense-in-depth). Two callers holding the same hash
|
|
1360
|
+
* see FIFO order; different hashes never block one another.
|
|
1361
|
+
*
|
|
1362
|
+
* Keyed `${entityId}\0${sourceHash}` so distinct (entityId, sourceHash)
|
|
1363
|
+
* pairs can never share a lock. A raw concatenation would let
|
|
1364
|
+
* ("ab", "c…") collide with ("a", "bc…"); the NUL separator makes that
|
|
1365
|
+
* impossible because neither component can contain NUL (entityIds are
|
|
1366
|
+
* user-supplied identifiers; sourceHashes are hex digests).
|
|
1367
|
+
*/
|
|
1368
|
+
this.hashLocks = /* @__PURE__ */ new Map();
|
|
1369
|
+
this.statusSubscribers = /* @__PURE__ */ new Map();
|
|
1370
|
+
/**
|
|
1371
|
+
* Lookup table for acquireLock/releaseLock's dynamic-dispatch branch.
|
|
1372
|
+
* Excludes 'ingest' | 'global_reembed' | 'global_import', which those
|
|
1373
|
+
* methods already handle via explicit if/else branches before reaching
|
|
1374
|
+
* this table.
|
|
1375
|
+
*/
|
|
1376
|
+
this.lockKeyFns = {
|
|
1377
|
+
prune: (id) => this._pruneKey(id),
|
|
1378
|
+
librarian: (id) => this._librarianKey(id),
|
|
1379
|
+
heal: (id) => this._healKey(id),
|
|
1380
|
+
reembed: (id) => this._reembedKey(id),
|
|
1381
|
+
import: (id) => this._importKey(id),
|
|
1382
|
+
forget: (id) => this._forgetKey(id),
|
|
1383
|
+
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
1384
|
+
};
|
|
1385
|
+
}
|
|
1386
|
+
_hashLockKey(entityId, sourceHash) {
|
|
1387
|
+
return `${entityId}\0${sourceHash}`;
|
|
1388
|
+
}
|
|
1389
|
+
/**
|
|
1390
|
+
* Acquire a promise-chain lock scoped to one `(entityId, sourceHash)` pair.
|
|
1391
|
+
* Returns a zero-argument release closure that removes the current tail
|
|
1392
|
+
* from the chain. Releasing is idempotent so a holder that double-fires
|
|
1393
|
+
* (e.g. on a finally that races an explicit release) cannot unblock
|
|
1394
|
+
* the next holder prematurely.
|
|
1395
|
+
*
|
|
1396
|
+
* Documented ordering: ingest acquires the hash lock FIRST, then the
|
|
1397
|
+
* synchronous sourceRef ingest lock, and the release closure unwinds in
|
|
1398
|
+
* the opposite order (sourceRef first, hash second). Hash-then-sourceRef
|
|
1399
|
+
* is required because the v9 partial UNIQUE index conflicts on the
|
|
1400
|
+
* (entity_id, source_hash) pair regardless of source_ref, so two callers
|
|
1401
|
+
* with DIFFERENT source_refs racing the same hash are the exact case the
|
|
1402
|
+
* TOCTOU race fix needs to serialize.
|
|
1403
|
+
*
|
|
1404
|
+
* Mechanics: each holder owns a private `released` promise that resolves
|
|
1405
|
+
* only when THIS holder calls release(). The map stores the holder's
|
|
1406
|
+
* `released` promise as the "tail" — the NEXT caller awaits that tail
|
|
1407
|
+
* via `previous = this.hashLocks.get(key)`. The current caller gets
|
|
1408
|
+
* their unique release function once `previous` (the prior holder's
|
|
1409
|
+
* release signal) settles, so each holder is guaranteed a UNIQUE release
|
|
1410
|
+
* closure bound to their own `released` promise.
|
|
1411
|
+
*/
|
|
1412
|
+
acquireHashLock(entityId, sourceHash) {
|
|
1413
|
+
const key = this._hashLockKey(entityId, sourceHash);
|
|
1414
|
+
const previous = this.hashLocks.get(key) ?? Promise.resolve();
|
|
1415
|
+
let resolveReleased;
|
|
1416
|
+
const released = new Promise((resolve) => {
|
|
1417
|
+
resolveReleased = resolve;
|
|
1418
|
+
});
|
|
1419
|
+
let fired = false;
|
|
1420
|
+
const release = () => {
|
|
1421
|
+
if (fired) return;
|
|
1422
|
+
fired = true;
|
|
1423
|
+
if (this.hashLocks.get(key) === released) {
|
|
1424
|
+
this.hashLocks.delete(key);
|
|
1414
1425
|
}
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
i++;
|
|
1426
|
+
resolveReleased();
|
|
1427
|
+
};
|
|
1428
|
+
this.hashLocks.set(key, released);
|
|
1429
|
+
return previous.then(() => release);
|
|
1420
1430
|
}
|
|
1421
|
-
|
|
1422
|
-
}
|
|
1423
|
-
function controlCharEscape(code) {
|
|
1424
|
-
switch (code) {
|
|
1425
|
-
case 10:
|
|
1426
|
-
return "\\n";
|
|
1427
|
-
case 9:
|
|
1428
|
-
return "\\t";
|
|
1429
|
-
case 13:
|
|
1430
|
-
return "\\r";
|
|
1431
|
-
case 8:
|
|
1432
|
-
return "\\b";
|
|
1433
|
-
case 12:
|
|
1434
|
-
return "\\f";
|
|
1435
|
-
default:
|
|
1436
|
-
return "\\u" + code.toString(16).padStart(4, "0");
|
|
1431
|
+
_pruneKey(entityId) {
|
|
1432
|
+
return `${this.prefix}:${entityId}:prune`;
|
|
1437
1433
|
}
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
const match = /position\s+(\d+)/.exec(safeErrorToString(err));
|
|
1441
|
-
return match ? Number(match[1]) : null;
|
|
1442
|
-
}
|
|
1443
|
-
function safeErrorToString(e) {
|
|
1444
|
-
let isErrorLike = false;
|
|
1445
|
-
try {
|
|
1446
|
-
isErrorLike = e instanceof Error;
|
|
1447
|
-
} catch {
|
|
1434
|
+
_reembedKey(entityId) {
|
|
1435
|
+
return `${this.prefix}:${entityId}:reembed`;
|
|
1448
1436
|
}
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
if (typeof msg === "string" && msg.length > 0) return msg;
|
|
1452
|
-
const name = readErrorField(e, "name");
|
|
1453
|
-
if (typeof name === "string" && name.length > 0) return name;
|
|
1454
|
-
return "[Error]";
|
|
1437
|
+
_globalReembedKey() {
|
|
1438
|
+
return `${this.prefix}:reembed`;
|
|
1455
1439
|
}
|
|
1456
|
-
|
|
1457
|
-
return
|
|
1458
|
-
}
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1440
|
+
_importKey(entityId) {
|
|
1441
|
+
return `${this.prefix}:${entityId}:import`;
|
|
1442
|
+
}
|
|
1443
|
+
_globalImportKey() {
|
|
1444
|
+
return `${this.prefix}:import`;
|
|
1445
|
+
}
|
|
1446
|
+
_forgetKey(entityId) {
|
|
1447
|
+
return `${this.prefix}:${entityId}:forget`;
|
|
1448
|
+
}
|
|
1449
|
+
_librarianKey(entityId) {
|
|
1450
|
+
return `${this.prefix}:${entityId}:librarian`;
|
|
1451
|
+
}
|
|
1452
|
+
_healKey(entityId) {
|
|
1453
|
+
return `${this.prefix}:${entityId}:heal`;
|
|
1454
|
+
}
|
|
1455
|
+
_ontologyBackfillKey(entityId) {
|
|
1456
|
+
return `${this.prefix}:${entityId}:ontologyBackfill`;
|
|
1457
|
+
}
|
|
1458
|
+
_isReembedActive(entityId) {
|
|
1459
|
+
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey());
|
|
1460
|
+
}
|
|
1461
|
+
/**
|
|
1462
|
+
* True while any reembed sweep is in flight — per-entity or global.
|
|
1463
|
+
*
|
|
1464
|
+
* Read-only; acquires nothing. Callers that mutate embedding failure
|
|
1465
|
+
* markers across ALL entities (the dimension-promotion clear) must gate on
|
|
1466
|
+
* this and defer, rather than resurrect rows an in-flight sweep already
|
|
1467
|
+
* classified. See spec 2026-09-05-reembed-lock-scope-design.md §2.
|
|
1468
|
+
*
|
|
1469
|
+
* The global key `${prefix}:reembed` also ends in ':reembed', so the suffix
|
|
1470
|
+
* scan alone would already match it. The explicit check is kept because it
|
|
1471
|
+
* documents the intent and survives any future change to the global key's
|
|
1472
|
+
* format.
|
|
1473
|
+
*/
|
|
1474
|
+
isAnyReembedActive() {
|
|
1475
|
+
return this.activeMaintenanceJobs.has(this._globalReembedKey()) || this._isAnyMaintenanceActiveWithSuffix(":reembed");
|
|
1476
|
+
}
|
|
1477
|
+
_isImportActiveFor(entityId) {
|
|
1478
|
+
return this.activeMaintenanceJobs.has(this._importKey(entityId)) || this.activeMaintenanceJobs.has(this._globalImportKey());
|
|
1479
|
+
}
|
|
1480
|
+
_isForgetActiveFor(entityId) {
|
|
1481
|
+
return this.activeMaintenanceJobs.has(this._forgetKey(entityId));
|
|
1482
|
+
}
|
|
1483
|
+
_isAnyMaintenanceActiveWithSuffix(suffix) {
|
|
1484
|
+
const entityKeyPrefix = `${this.prefix}:`;
|
|
1485
|
+
for (const k of this.activeMaintenanceJobs) {
|
|
1486
|
+
if (k.startsWith(entityKeyPrefix) && k.endsWith(suffix)) return true;
|
|
1463
1487
|
}
|
|
1488
|
+
return false;
|
|
1464
1489
|
}
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
try {
|
|
1468
|
-
return e[key];
|
|
1469
|
-
} catch {
|
|
1470
|
-
return void 0;
|
|
1490
|
+
_hasIngestJob(entityId, sourceRef) {
|
|
1491
|
+
return this.activeIngestJobs.get(entityId)?.has(sourceRef ?? "") ?? false;
|
|
1471
1492
|
}
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1493
|
+
_addIngestJob(entityId, sourceRef) {
|
|
1494
|
+
const sourceKey = sourceRef ?? "";
|
|
1495
|
+
let refs = this.activeIngestJobs.get(entityId);
|
|
1496
|
+
if (!refs) {
|
|
1497
|
+
refs = /* @__PURE__ */ new Set();
|
|
1498
|
+
this.activeIngestJobs.set(entityId, refs);
|
|
1499
|
+
}
|
|
1500
|
+
refs.add(sourceKey);
|
|
1478
1501
|
}
|
|
1479
|
-
|
|
1480
|
-
|
|
1502
|
+
_removeIngestJob(entityId, sourceRef) {
|
|
1503
|
+
const sourceKey = sourceRef ?? "";
|
|
1504
|
+
const refs = this.activeIngestJobs.get(entityId);
|
|
1505
|
+
if (!refs) return;
|
|
1506
|
+
refs.delete(sourceKey);
|
|
1507
|
+
if (refs.size === 0) {
|
|
1508
|
+
this.activeIngestJobs.delete(entityId);
|
|
1509
|
+
}
|
|
1481
1510
|
}
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1511
|
+
_isIngestActiveFor(entityId) {
|
|
1512
|
+
return this.activeIngestJobs.has(entityId);
|
|
1513
|
+
}
|
|
1514
|
+
acquireLock(operation, entityId, sourceRef) {
|
|
1515
|
+
let blockingOperation = null;
|
|
1516
|
+
if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) {
|
|
1517
|
+
throw new WikiBusyError("import", "*");
|
|
1518
|
+
}
|
|
1519
|
+
switch (operation) {
|
|
1520
|
+
case "prune":
|
|
1521
|
+
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1522
|
+
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
1523
|
+
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
1524
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
1525
|
+
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
1526
|
+
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
1527
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1528
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1529
|
+
break;
|
|
1530
|
+
case "librarian":
|
|
1531
|
+
case "heal":
|
|
1532
|
+
case "ontologyBackfill": {
|
|
1533
|
+
const opKey = this.lockKeyFns[operation](entityId);
|
|
1534
|
+
if (this.activeMaintenanceJobs.has(opKey)) blockingOperation = operation;
|
|
1535
|
+
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1536
|
+
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
1537
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1538
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1539
|
+
break;
|
|
1540
|
+
}
|
|
1541
|
+
case "reembed":
|
|
1542
|
+
if (this.activeMaintenanceJobs.has(this._reembedKey(entityId))) blockingOperation = "reembed";
|
|
1543
|
+
else if (this.activeMaintenanceJobs.has(this._globalReembedKey())) blockingOperation = "reembed";
|
|
1544
|
+
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1545
|
+
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
1546
|
+
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
1547
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
1548
|
+
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
1549
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1550
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1551
|
+
break;
|
|
1552
|
+
case "global_reembed":
|
|
1553
|
+
if (this.activeMaintenanceJobs.has(this._globalReembedKey())) blockingOperation = "reembed";
|
|
1554
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":reembed")) blockingOperation = "reembed";
|
|
1555
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":prune")) blockingOperation = "prune";
|
|
1556
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":librarian")) blockingOperation = "librarian";
|
|
1557
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":heal")) blockingOperation = "heal";
|
|
1558
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill")) blockingOperation = "ontologyBackfill";
|
|
1559
|
+
else if (this.activeIngestJobs.size > 0) blockingOperation = "ingest";
|
|
1560
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":import")) blockingOperation = "import";
|
|
1561
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":forget")) blockingOperation = "forget";
|
|
1562
|
+
break;
|
|
1563
|
+
case "import":
|
|
1564
|
+
case "forget": {
|
|
1565
|
+
const selfKey = operation === "import" ? this._importKey(entityId) : this._forgetKey(entityId);
|
|
1566
|
+
if (this.activeMaintenanceJobs.has(selfKey)) blockingOperation = operation;
|
|
1567
|
+
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
1568
|
+
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
1569
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
1570
|
+
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1571
|
+
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
1572
|
+
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
1573
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1574
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1575
|
+
break;
|
|
1576
|
+
}
|
|
1577
|
+
case "global_import":
|
|
1578
|
+
if (this.activeMaintenanceJobs.has(this._globalImportKey())) blockingOperation = "import";
|
|
1579
|
+
break;
|
|
1580
|
+
case "ingest": {
|
|
1581
|
+
const sourceKey = sourceRef ?? "";
|
|
1582
|
+
if (this._hasIngestJob(entityId, sourceKey)) blockingOperation = "ingest";
|
|
1583
|
+
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
1584
|
+
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
1585
|
+
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
1586
|
+
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
1587
|
+
break;
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
if (blockingOperation) {
|
|
1591
|
+
throw new WikiBusyError(
|
|
1592
|
+
blockingOperation,
|
|
1593
|
+
operation === "global_reembed" || operation === "global_import" ? "*" : entityId
|
|
1594
|
+
);
|
|
1595
|
+
}
|
|
1596
|
+
if (operation === "ingest") {
|
|
1597
|
+
this._addIngestJob(entityId, sourceRef);
|
|
1598
|
+
} else if (operation === "global_reembed") {
|
|
1599
|
+
this.activeMaintenanceJobs.add(this._globalReembedKey());
|
|
1600
|
+
} else if (operation === "global_import") {
|
|
1601
|
+
this.activeMaintenanceJobs.add(this._globalImportKey());
|
|
1489
1602
|
} else {
|
|
1490
|
-
|
|
1603
|
+
this.activeMaintenanceJobs.add(this.lockKeyFns[operation](entityId));
|
|
1491
1604
|
}
|
|
1492
|
-
|
|
1493
|
-
typeName = isErrorLike ? "Error" : typeof err;
|
|
1605
|
+
this._notifyStatusSubscribers(entityId);
|
|
1494
1606
|
}
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
}
|
|
1501
|
-
|
|
1607
|
+
releaseLock(operation, entityId, sourceRef) {
|
|
1608
|
+
if (operation === "ingest") {
|
|
1609
|
+
this._removeIngestJob(entityId, sourceRef);
|
|
1610
|
+
} else if (operation === "global_reembed") {
|
|
1611
|
+
this.activeMaintenanceJobs.delete(this._globalReembedKey());
|
|
1612
|
+
} else if (operation === "global_import") {
|
|
1613
|
+
this.activeMaintenanceJobs.delete(this._globalImportKey());
|
|
1614
|
+
} else {
|
|
1615
|
+
this.activeMaintenanceJobs.delete(this.lockKeyFns[operation](entityId));
|
|
1502
1616
|
}
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1508
|
-
|
|
1509
|
-
|
|
1617
|
+
this._notifyStatusSubscribers(entityId);
|
|
1618
|
+
}
|
|
1619
|
+
/**
|
|
1620
|
+
* Returns true if acquireLock(operation, entityId) would throw WikiBusyError.
|
|
1621
|
+
* Use for non-throwing conflict checks (e.g. auto-trigger gating in write()).
|
|
1622
|
+
*/
|
|
1623
|
+
isBlocked(operation, entityId) {
|
|
1624
|
+
if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) return true;
|
|
1625
|
+
switch (operation) {
|
|
1626
|
+
case "librarian":
|
|
1627
|
+
case "heal":
|
|
1628
|
+
case "ontologyBackfill":
|
|
1629
|
+
return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1630
|
+
case "prune":
|
|
1631
|
+
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1632
|
+
case "reembed":
|
|
1633
|
+
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey()) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1634
|
+
case "global_reembed":
|
|
1635
|
+
return this.activeMaintenanceJobs.has(this._globalReembedKey()) || this._isAnyMaintenanceActiveWithSuffix(":reembed") || this._isAnyMaintenanceActiveWithSuffix(":prune") || this._isAnyMaintenanceActiveWithSuffix(":librarian") || this._isAnyMaintenanceActiveWithSuffix(":heal") || this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill") || this.activeIngestJobs.size > 0 || this._isAnyMaintenanceActiveWithSuffix(":import") || this._isAnyMaintenanceActiveWithSuffix(":forget");
|
|
1636
|
+
case "import":
|
|
1637
|
+
case "forget": {
|
|
1638
|
+
const selfKey = operation === "import" ? this._importKey(entityId) : this._forgetKey(entityId);
|
|
1639
|
+
return this.activeMaintenanceJobs.has(selfKey) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1510
1640
|
}
|
|
1511
|
-
|
|
1641
|
+
case "global_import":
|
|
1642
|
+
return this.activeMaintenanceJobs.has(this._globalImportKey());
|
|
1643
|
+
case "ingest":
|
|
1644
|
+
return this._hasIngestJob(entityId) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
1645
|
+
default:
|
|
1646
|
+
return false;
|
|
1512
1647
|
}
|
|
1513
1648
|
}
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
let safeEnd = end === void 0 ? length : end < 0 ? Math.max(length + end, 0) : Math.min(end, length);
|
|
1528
|
-
if (safeStart > safeEnd) {
|
|
1529
|
-
[safeStart, safeEnd] = [safeEnd, safeStart];
|
|
1530
|
-
}
|
|
1531
|
-
if (safeStart > 0 && safeStart < length && value.charCodeAt(safeStart) >= 56320 && value.charCodeAt(safeStart) <= 57343 && value.charCodeAt(safeStart - 1) >= 55296 && value.charCodeAt(safeStart - 1) <= 56319) {
|
|
1532
|
-
safeStart--;
|
|
1533
|
-
}
|
|
1534
|
-
if (safeEnd > 0 && safeEnd < length && value.charCodeAt(safeEnd - 1) >= 55296 && value.charCodeAt(safeEnd - 1) <= 56319 && value.charCodeAt(safeEnd) >= 56320 && value.charCodeAt(safeEnd) <= 57343) {
|
|
1535
|
-
safeEnd--;
|
|
1536
|
-
}
|
|
1537
|
-
return value.slice(safeStart, safeEnd);
|
|
1538
|
-
}
|
|
1539
|
-
function chunkText(input, maxChunkLength, overlap) {
|
|
1540
|
-
const text = input.trim();
|
|
1541
|
-
if (text.length === 0) return { chunks: [], truncated: false };
|
|
1542
|
-
if (!Number.isInteger(maxChunkLength) || maxChunkLength < 2) {
|
|
1543
|
-
throw new Error("maxChunkLength must be an integer >= 2");
|
|
1649
|
+
/**
|
|
1650
|
+
* Auto-heal historically only gated on the heal self-key. Keep that
|
|
1651
|
+
* permissiveness for write() auto-trigger paths — stricter checks stay in
|
|
1652
|
+
* acquireLock() — with one exception: reembed sweeps, which are excluded
|
|
1653
|
+
* structurally rather than by accident (see below).
|
|
1654
|
+
*/
|
|
1655
|
+
tryAcquireAutoHealLock(entityId) {
|
|
1656
|
+
if (this.isAnyReembedActive()) return false;
|
|
1657
|
+
const healKey = this._healKey(entityId);
|
|
1658
|
+
if (this.activeMaintenanceJobs.has(healKey)) return false;
|
|
1659
|
+
this.activeMaintenanceJobs.add(healKey);
|
|
1660
|
+
this._notifyStatusSubscribers(entityId);
|
|
1661
|
+
return true;
|
|
1544
1662
|
}
|
|
1545
|
-
|
|
1546
|
-
|
|
1663
|
+
/**
|
|
1664
|
+
* Centralized ingest lock acquisition. Acquires the hash lock FIRST
|
|
1665
|
+
* (FIFO across callers racing the same hash), then the synchronous
|
|
1666
|
+
* sourceRef ingest lock (rejects cross-entity conflicts and busy
|
|
1667
|
+
* operations). Returns a single zero-argument release closure that
|
|
1668
|
+
* unwinds in the opposite order — sourceRef first, hash second — so
|
|
1669
|
+
* the sourceRef lock is freed the instant its work is done and only
|
|
1670
|
+
* the hash lock keeps serializing the duplicate-content race window.
|
|
1671
|
+
*
|
|
1672
|
+
* On a sourceRef conflict (WikiBusyError thrown synchronously by the
|
|
1673
|
+
* underlying acquireLock) the hash lock is released BEFORE the error
|
|
1674
|
+
* propagates, so the next caller in the hash chain is not held up by
|
|
1675
|
+
* a request that never reached the DB.
|
|
1676
|
+
*/
|
|
1677
|
+
async acquireIngestLocks(entityId, sourceRef, sourceHash) {
|
|
1678
|
+
const releaseHash = await this.acquireHashLock(entityId, sourceHash);
|
|
1679
|
+
let sourceLockAcquired = false;
|
|
1680
|
+
try {
|
|
1681
|
+
this.acquireLock("ingest", entityId, sourceRef);
|
|
1682
|
+
sourceLockAcquired = true;
|
|
1683
|
+
} catch (err) {
|
|
1684
|
+
releaseHash();
|
|
1685
|
+
throw err;
|
|
1686
|
+
}
|
|
1687
|
+
return () => {
|
|
1688
|
+
try {
|
|
1689
|
+
if (sourceLockAcquired) this.releaseLock("ingest", entityId, sourceRef);
|
|
1690
|
+
} finally {
|
|
1691
|
+
releaseHash();
|
|
1692
|
+
}
|
|
1693
|
+
};
|
|
1547
1694
|
}
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
const
|
|
1554
|
-
|
|
1555
|
-
|
|
1556
|
-
|
|
1695
|
+
/**
|
|
1696
|
+
* Validates then acquires global + per-entity import locks atomically.
|
|
1697
|
+
* Validates all entities before acquiring any lock (same as current importDump semantics).
|
|
1698
|
+
*/
|
|
1699
|
+
acquireImportLocks(entityIds) {
|
|
1700
|
+
for (const entityId of entityIds) {
|
|
1701
|
+
if (this.activeMaintenanceJobs.has(this._importKey(entityId))) throw new WikiBusyError("import", entityId);
|
|
1702
|
+
if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) throw new WikiBusyError("librarian", entityId);
|
|
1703
|
+
if (this.activeMaintenanceJobs.has(this._healKey(entityId))) throw new WikiBusyError("heal", entityId);
|
|
1704
|
+
if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) throw new WikiBusyError("ontologyBackfill", entityId);
|
|
1705
|
+
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) throw new WikiBusyError("prune", entityId);
|
|
1706
|
+
if (this._isReembedActive(entityId)) throw new WikiBusyError("reembed", entityId);
|
|
1707
|
+
if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);
|
|
1708
|
+
if (this._isForgetActiveFor(entityId)) throw new WikiBusyError("forget", entityId);
|
|
1557
1709
|
}
|
|
1558
|
-
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
if (paraIdx >= minSplit && paraIdx + 2 <= windowEnd) {
|
|
1563
|
-
splitPoint = paraIdx + 2;
|
|
1710
|
+
if (this.activeMaintenanceJobs.has(this._globalImportKey())) throw new WikiBusyError("import", "*");
|
|
1711
|
+
this.activeMaintenanceJobs.add(this._globalImportKey());
|
|
1712
|
+
for (const entityId of entityIds) {
|
|
1713
|
+
this.activeMaintenanceJobs.add(this._importKey(entityId));
|
|
1564
1714
|
}
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
lastTerm = i + 2;
|
|
1571
|
-
}
|
|
1572
|
-
}
|
|
1573
|
-
if (lastTerm !== -1 && lastTerm <= windowEnd) splitPoint = lastTerm;
|
|
1715
|
+
}
|
|
1716
|
+
releaseImportLocks(entityIds) {
|
|
1717
|
+
this.activeMaintenanceJobs.delete(this._globalImportKey());
|
|
1718
|
+
for (const entityId of entityIds) {
|
|
1719
|
+
this.activeMaintenanceJobs.delete(this._importKey(entityId));
|
|
1574
1720
|
}
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1721
|
+
}
|
|
1722
|
+
getEntityStatus(entityId) {
|
|
1723
|
+
return {
|
|
1724
|
+
ingesting: this._isIngestActiveFor(entityId),
|
|
1725
|
+
librarian: this.activeMaintenanceJobs.has(this._librarianKey(entityId)),
|
|
1726
|
+
heal: this.activeMaintenanceJobs.has(this._healKey(entityId))
|
|
1727
|
+
};
|
|
1728
|
+
}
|
|
1729
|
+
subscribeEntityStatus(entityId, callback) {
|
|
1730
|
+
const initial = this.getEntityStatus(entityId);
|
|
1731
|
+
let set = this.statusSubscribers.get(entityId);
|
|
1732
|
+
if (!set) {
|
|
1733
|
+
set = /* @__PURE__ */ new Set();
|
|
1734
|
+
this.statusSubscribers.set(entityId, set);
|
|
1582
1735
|
}
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1736
|
+
const entry = { callback, last: this._copyEntityStatus(initial) };
|
|
1737
|
+
set.add(entry);
|
|
1738
|
+
try {
|
|
1739
|
+
callback(this._copyEntityStatus(initial));
|
|
1740
|
+
} catch (err) {
|
|
1741
|
+
console.error(`[JobManager] callback error for entityId="${entityId}" during initial emission`, err);
|
|
1586
1742
|
}
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1743
|
+
let active = true;
|
|
1744
|
+
return () => {
|
|
1745
|
+
if (!active) return;
|
|
1746
|
+
active = false;
|
|
1747
|
+
const s = this.statusSubscribers.get(entityId);
|
|
1748
|
+
if (!s) return;
|
|
1749
|
+
s.delete(entry);
|
|
1750
|
+
if (s.size === 0) this.statusSubscribers.delete(entityId);
|
|
1751
|
+
};
|
|
1590
1752
|
}
|
|
1591
|
-
|
|
1592
|
-
}
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
|
|
1598
|
-
|
|
1599
|
-
|
|
1600
|
-
const
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
}
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1753
|
+
_copyEntityStatus(s) {
|
|
1754
|
+
return { ingesting: s.ingesting, librarian: s.librarian, heal: s.heal };
|
|
1755
|
+
}
|
|
1756
|
+
_notifyStatusSubscribers(entityId) {
|
|
1757
|
+
if (entityId === "*") return;
|
|
1758
|
+
const set = this.statusSubscribers.get(entityId);
|
|
1759
|
+
if (!set || set.size === 0) return;
|
|
1760
|
+
for (const entry of Array.from(set)) {
|
|
1761
|
+
if (!set.has(entry)) continue;
|
|
1762
|
+
const next = this.getEntityStatus(entityId);
|
|
1763
|
+
if (entry.last.ingesting === next.ingesting && entry.last.librarian === next.librarian && entry.last.heal === next.heal) {
|
|
1764
|
+
continue;
|
|
1765
|
+
}
|
|
1766
|
+
entry.last = this._copyEntityStatus(next);
|
|
1767
|
+
try {
|
|
1768
|
+
entry.callback(this._copyEntityStatus(next));
|
|
1769
|
+
} catch (err) {
|
|
1770
|
+
console.error(`[JobManager] callback error for entityId="${entityId}" during transition emission`, err);
|
|
1609
1771
|
}
|
|
1610
1772
|
}
|
|
1611
1773
|
}
|
|
1612
|
-
|
|
1613
|
-
await Promise.allSettled(Array.from({ length: workerCount }, worker));
|
|
1614
|
-
if (failed) throw firstError;
|
|
1615
|
-
return results;
|
|
1616
|
-
}
|
|
1617
|
-
function clip(value, max) {
|
|
1618
|
-
if (typeof value !== "string") return "";
|
|
1619
|
-
const s = value.trim();
|
|
1620
|
-
return s.length <= max ? s : safeSlice(s, 0, max).trimEnd();
|
|
1621
|
-
}
|
|
1622
|
-
function validateTags(tags) {
|
|
1623
|
-
if (!Array.isArray(tags)) return [];
|
|
1624
|
-
return tags.filter((t) => typeof t === "string").map((t) => t.trim().toLowerCase()).filter((t) => t.length > 0 && t.length <= 40).slice(0, 6);
|
|
1625
|
-
}
|
|
1626
|
-
var MAX_EVIDENCE_QUOTES = 10;
|
|
1627
|
-
function normalizeEvidence(raw) {
|
|
1628
|
-
if (!Array.isArray(raw)) return void 0;
|
|
1629
|
-
const out = [];
|
|
1630
|
-
for (const entry of raw) {
|
|
1631
|
-
if (typeof entry !== "string") continue;
|
|
1632
|
-
const trimmed = entry.trim();
|
|
1633
|
-
if (!trimmed) continue;
|
|
1634
|
-
out.push(trimmed);
|
|
1635
|
-
if (out.length > MAX_EVIDENCE_QUOTES) break;
|
|
1636
|
-
}
|
|
1637
|
-
return out;
|
|
1638
|
-
}
|
|
1639
|
-
function validateFact(fact) {
|
|
1640
|
-
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return null;
|
|
1641
|
-
const title = clip(fact.title, 80);
|
|
1642
|
-
const body = clip(fact.body, 800);
|
|
1643
|
-
if (!title || !body) return null;
|
|
1644
|
-
let confidence = fact.confidence;
|
|
1645
|
-
if (confidence !== "certain" && confidence !== "tentative") confidence = "inferred";
|
|
1646
|
-
const valid = {
|
|
1647
|
-
...fact,
|
|
1648
|
-
title,
|
|
1649
|
-
body,
|
|
1650
|
-
confidence,
|
|
1651
|
-
tags: validateTags(fact.tags)
|
|
1652
|
-
};
|
|
1653
|
-
const evidence = normalizeEvidence(fact.evidence);
|
|
1654
|
-
if (evidence) valid.evidence = evidence;
|
|
1655
|
-
else delete valid.evidence;
|
|
1656
|
-
return valid;
|
|
1657
|
-
}
|
|
1658
|
-
function validateTask(task) {
|
|
1659
|
-
if (typeof task?.description !== "string") return null;
|
|
1660
|
-
const description = clip(task.description, 200);
|
|
1661
|
-
if (!description) return null;
|
|
1662
|
-
let priority = task.priority;
|
|
1663
|
-
if (typeof priority !== "number" || !isFinite(priority)) priority = 0;
|
|
1664
|
-
priority = Math.max(0, Math.min(10, Math.round(priority)));
|
|
1665
|
-
return {
|
|
1666
|
-
...task,
|
|
1667
|
-
description,
|
|
1668
|
-
priority
|
|
1669
|
-
};
|
|
1670
|
-
}
|
|
1671
|
-
function factRejectionReason(raw) {
|
|
1672
|
-
const fact = raw;
|
|
1673
|
-
if (typeof fact?.title !== "string" || typeof fact?.body !== "string") return "invalid_shape";
|
|
1674
|
-
if (!clip(fact.title, 80)) return "missing_title";
|
|
1675
|
-
return "missing_body";
|
|
1676
|
-
}
|
|
1677
|
-
function taskRejectionReason(raw) {
|
|
1678
|
-
const task = raw;
|
|
1679
|
-
if (typeof task?.description !== "string") return "invalid_shape";
|
|
1680
|
-
return "missing_description";
|
|
1681
|
-
}
|
|
1682
|
-
function normalizeSourceRef(value) {
|
|
1683
|
-
if (typeof value !== "string") return null;
|
|
1684
|
-
const cleaned = value.replace(/[^A-Za-z0-9._\- ]/g, "").trim().slice(0, 255);
|
|
1685
|
-
return cleaned.length > 0 ? cleaned : null;
|
|
1686
|
-
}
|
|
1687
|
-
function normalizeSourceHash(value) {
|
|
1688
|
-
if (typeof value !== "string") return null;
|
|
1689
|
-
return /^[0-9a-f]{64}$/i.test(value) ? value.toLowerCase() : null;
|
|
1690
|
-
}
|
|
1691
|
-
function titleTokens(title) {
|
|
1692
|
-
return new Set(title.toLowerCase().replace(/[^a-z0-9\s]/g, "").split(/\s+/).filter((t) => t.length >= 3));
|
|
1693
|
-
}
|
|
1694
|
-
function jaccardScore(a, b) {
|
|
1695
|
-
if (a.size === 0 && b.size === 0) return 0;
|
|
1696
|
-
const intersection = new Set([...a].filter((x) => b.has(x)));
|
|
1697
|
-
const union = /* @__PURE__ */ new Set([...a, ...b]);
|
|
1698
|
-
return intersection.size / union.size;
|
|
1699
|
-
}
|
|
1774
|
+
};
|
|
1700
1775
|
|
|
1701
1776
|
// src/prompts.ts
|
|
1702
1777
|
var LIBRARIAN_SYSTEM_PROMPT = `You are a knowledge extraction agent. Your job is to analyze recent episodic events and extract stable facts and actionable tasks about the user or entity.
|
|
@@ -2116,7 +2191,7 @@ var MESSAGE = {
|
|
|
2116
2191
|
embedding_failed: "A fact could not be embedded.",
|
|
2117
2192
|
hook_failed: "A host hook threw or rejected.",
|
|
2118
2193
|
background_job_failed: "A background maintenance job failed.",
|
|
2119
|
-
heal_skipped: "Heal skipped a candidate fact.",
|
|
2194
|
+
heal_skipped: "Heal skipped a candidate fact, or the whole LLM pass when keyword search is unavailable.",
|
|
2120
2195
|
grounding_missing: "A fact carried no qualifying evidence and was stored as a draft.",
|
|
2121
2196
|
grounding_failed: "A fact carried evidence not found in its source and was stored as a draft.",
|
|
2122
2197
|
classification_low_confidence: "A classifier answer was below the confidence threshold and was not applied.",
|
|
@@ -2610,7 +2685,8 @@ var IngestionService = class {
|
|
|
2610
2685
|
*
|
|
2611
2686
|
* @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
|
|
2612
2687
|
* (manifest-valid edges persisted), superseded (prior facts soft-deleted
|
|
2613
|
-
* plus prior source-ref edges hard-deleted)
|
|
2688
|
+
* plus prior source-ref edges hard-deleted), and supersededFactIds (the
|
|
2689
|
+
* ids of those soft-deleted facts, for the caller's post-commit index sync).
|
|
2614
2690
|
*
|
|
2615
2691
|
* Node-ownership pre-flight (spec §3, REQ-GRAPH-01): the first statement
|
|
2616
2692
|
* is `assertGraphNodeOwnership(entityId, params.nodes, tx)`, which runs
|
|
@@ -2623,6 +2699,9 @@ var IngestionService = class {
|
|
|
2623
2699
|
* ordering and counts so success counts match the pre-change contract.
|
|
2624
2700
|
*/
|
|
2625
2701
|
async upsertGraphCore(entityId, params, tx, opts) {
|
|
2702
|
+
const sourceRef = normalizeSourceRef(params.sourceRef);
|
|
2703
|
+
if (!sourceRef) throw new Error("Invalid sourceRef");
|
|
2704
|
+
params = { ...params, sourceRef };
|
|
2626
2705
|
await this.assertGraphNodeOwnership(entityId, params.nodes, tx);
|
|
2627
2706
|
const now = Date.now();
|
|
2628
2707
|
const ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
|
|
@@ -2735,7 +2814,8 @@ var IngestionService = class {
|
|
|
2735
2814
|
return {
|
|
2736
2815
|
nodesWritten: wikiFacts.length,
|
|
2737
2816
|
edgesWritten: edgesInsertedCount,
|
|
2738
|
-
superseded: deletedFactIds.length + deletedEdgeCount
|
|
2817
|
+
superseded: deletedFactIds.length + deletedEdgeCount,
|
|
2818
|
+
supersededFactIds: deletedFactIds
|
|
2739
2819
|
};
|
|
2740
2820
|
}
|
|
2741
2821
|
/**
|
|
@@ -3322,6 +3402,784 @@ async function runBatched(args) {
|
|
|
3322
3402
|
return { results, skipped, batches, unattempted, ...budgetStop ? { budgetStop } : {} };
|
|
3323
3403
|
}
|
|
3324
3404
|
|
|
3405
|
+
// src/repositories/rowMappers.ts
|
|
3406
|
+
function parseJsonArray(s, fallback) {
|
|
3407
|
+
if (Array.isArray(s)) return s;
|
|
3408
|
+
if (typeof s === "string" && s.length > 0) {
|
|
3409
|
+
try {
|
|
3410
|
+
const p = JSON.parse(s);
|
|
3411
|
+
if (Array.isArray(p)) return p;
|
|
3412
|
+
} catch {
|
|
3413
|
+
}
|
|
3414
|
+
}
|
|
3415
|
+
return fallback;
|
|
3416
|
+
}
|
|
3417
|
+
function parseJsonObject(s, fallback = null) {
|
|
3418
|
+
if (s && typeof s === "object" && !Array.isArray(s)) return s;
|
|
3419
|
+
if (typeof s === "string" && s.length > 0) {
|
|
3420
|
+
try {
|
|
3421
|
+
const p = JSON.parse(s);
|
|
3422
|
+
if (p && typeof p === "object" && !Array.isArray(p)) return p;
|
|
3423
|
+
} catch {
|
|
3424
|
+
}
|
|
3425
|
+
}
|
|
3426
|
+
return fallback;
|
|
3427
|
+
}
|
|
3428
|
+
|
|
3429
|
+
// src/db/schema.ts
|
|
3430
|
+
async function setupDatabase(db, prefix) {
|
|
3431
|
+
await db.execAsync(`
|
|
3432
|
+
CREATE TABLE IF NOT EXISTS ${prefix}entries (
|
|
3433
|
+
id TEXT PRIMARY KEY,
|
|
3434
|
+
entity_id TEXT NOT NULL,
|
|
3435
|
+
title TEXT NOT NULL,
|
|
3436
|
+
body TEXT NOT NULL,
|
|
3437
|
+
tags TEXT NOT NULL DEFAULT '[]',
|
|
3438
|
+
confidence TEXT NOT NULL DEFAULT 'inferred',
|
|
3439
|
+
source_type TEXT NOT NULL DEFAULT 'librarian_inferred',
|
|
3440
|
+
source_hash TEXT,
|
|
3441
|
+
source_ref TEXT,
|
|
3442
|
+
created_at INTEGER NOT NULL,
|
|
3443
|
+
updated_at INTEGER NOT NULL,
|
|
3444
|
+
last_accessed_at INTEGER,
|
|
3445
|
+
access_count INTEGER NOT NULL DEFAULT 0,
|
|
3446
|
+
deleted_at INTEGER,
|
|
3447
|
+
embedding TEXT,
|
|
3448
|
+
embedding_blob BLOB,
|
|
3449
|
+
okf_type TEXT,
|
|
3450
|
+
ontology_checked_at INTEGER,
|
|
3451
|
+
heal_checked_at INTEGER,
|
|
3452
|
+
lifecycle_status TEXT NOT NULL DEFAULT 'stable',
|
|
3453
|
+
stale_after INTEGER,
|
|
3454
|
+
generated_by TEXT,
|
|
3455
|
+
last_verified_at INTEGER,
|
|
3456
|
+
last_verified_by TEXT,
|
|
3457
|
+
okf_sources TEXT,
|
|
3458
|
+
okf_verified TEXT,
|
|
3459
|
+
okf_usage_window TEXT,
|
|
3460
|
+
embedding_failed_at INTEGER,
|
|
3461
|
+
embedding_failure_kind TEXT,
|
|
3462
|
+
embedding_attempts INTEGER NOT NULL DEFAULT 0,
|
|
3463
|
+
valid_from INTEGER,
|
|
3464
|
+
valid_to INTEGER,
|
|
3465
|
+
superseded_by TEXT,
|
|
3466
|
+
superseded_at INTEGER
|
|
3467
|
+
);
|
|
3468
|
+
|
|
3469
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
|
|
3470
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_source_ref_idx ON ${prefix}entries(entity_id, source_ref);
|
|
3471
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_source_hash_idx ON ${prefix}entries(entity_id, source_hash) WHERE source_hash IS NOT NULL;
|
|
3472
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_updated_idx ON ${prefix}entries(updated_at DESC);
|
|
3473
|
+
|
|
3474
|
+
-- source_ref_index: per-(entity, source_hash) record of the canonical sourceRef
|
|
3475
|
+
-- currently holding that hash. The partial UNIQUE index on (entity_id, source_hash)
|
|
3476
|
+
-- WHERE deleted_at IS NULL enforces the sourceRef-level TOCTOU-race invariant;
|
|
3477
|
+
-- entries-level uniqueness cannot express it because a single ingestDocument call
|
|
3478
|
+
-- writes N facts that all share (entity_id, source_ref, source_hash). See
|
|
3479
|
+
-- docs/superpowers/specs/2026-08-07-dependabot-concurrency-release-hygiene-design.md \xA7B1.
|
|
3480
|
+
CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
3481
|
+
id TEXT PRIMARY KEY,
|
|
3482
|
+
entity_id TEXT NOT NULL,
|
|
3483
|
+
source_hash TEXT NOT NULL,
|
|
3484
|
+
source_ref TEXT NOT NULL,
|
|
3485
|
+
created_at INTEGER NOT NULL,
|
|
3486
|
+
deleted_at INTEGER
|
|
3487
|
+
);
|
|
3488
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
3489
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
3490
|
+
WHERE deleted_at IS NULL;
|
|
3491
|
+
|
|
3492
|
+
CREATE TABLE IF NOT EXISTS ${prefix}tasks (
|
|
3493
|
+
id TEXT PRIMARY KEY,
|
|
3494
|
+
entity_id TEXT NOT NULL,
|
|
3495
|
+
description TEXT NOT NULL,
|
|
3496
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
3497
|
+
priority INTEGER NOT NULL DEFAULT 0,
|
|
3498
|
+
created_at INTEGER NOT NULL,
|
|
3499
|
+
updated_at INTEGER NOT NULL,
|
|
3500
|
+
resolved_at INTEGER,
|
|
3501
|
+
deleted_at INTEGER,
|
|
3502
|
+
okf_type TEXT,
|
|
3503
|
+
lifecycle_status TEXT NOT NULL DEFAULT 'stable',
|
|
3504
|
+
stale_after INTEGER,
|
|
3505
|
+
generated_by TEXT,
|
|
3506
|
+
last_verified_at INTEGER,
|
|
3507
|
+
last_verified_by TEXT,
|
|
3508
|
+
okf_sources TEXT,
|
|
3509
|
+
okf_verified TEXT,
|
|
3510
|
+
okf_usage_window TEXT
|
|
3511
|
+
);
|
|
3512
|
+
|
|
3513
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_entity_idx ON ${prefix}tasks(entity_id, status);
|
|
3514
|
+
|
|
3515
|
+
CREATE TABLE IF NOT EXISTS ${prefix}edges (
|
|
3516
|
+
id TEXT PRIMARY KEY,
|
|
3517
|
+
entity_id TEXT NOT NULL,
|
|
3518
|
+
source_id TEXT NOT NULL,
|
|
3519
|
+
target_id TEXT NOT NULL,
|
|
3520
|
+
edge_type TEXT NOT NULL,
|
|
3521
|
+
created_at INTEGER NOT NULL,
|
|
3522
|
+
UNIQUE(entity_id, source_id, target_id, edge_type)
|
|
3523
|
+
);
|
|
3524
|
+
|
|
3525
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
|
|
3526
|
+
|
|
3527
|
+
CREATE TABLE IF NOT EXISTS ${prefix}events (
|
|
3528
|
+
id TEXT PRIMARY KEY,
|
|
3529
|
+
entity_id TEXT NOT NULL,
|
|
3530
|
+
event_type TEXT NOT NULL,
|
|
3531
|
+
summary TEXT NOT NULL,
|
|
3532
|
+
related_entry_id TEXT,
|
|
3533
|
+
created_at INTEGER NOT NULL,
|
|
3534
|
+
occurred_at INTEGER
|
|
3535
|
+
);
|
|
3536
|
+
|
|
3537
|
+
CREATE INDEX IF NOT EXISTS ${prefix}events_entity_idx ON ${prefix}events(entity_id, created_at DESC);
|
|
3538
|
+
|
|
3539
|
+
CREATE TABLE IF NOT EXISTS ${prefix}checkpoints (
|
|
3540
|
+
entity_id TEXT PRIMARY KEY,
|
|
3541
|
+
heal_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
3542
|
+
memory_checkpoint INTEGER NOT NULL DEFAULT 0,
|
|
3543
|
+
librarian_watermark_at INTEGER,
|
|
3544
|
+
librarian_watermark_id TEXT
|
|
3545
|
+
);
|
|
3546
|
+
|
|
3547
|
+
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
3548
|
+
entity_id TEXT PRIMARY KEY,
|
|
3549
|
+
mode TEXT NOT NULL DEFAULT 'off',
|
|
3550
|
+
manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
|
|
3551
|
+
updated_at INTEGER NOT NULL
|
|
3552
|
+
);
|
|
3553
|
+
|
|
3554
|
+
CREATE TABLE IF NOT EXISTS ${prefix}meta (
|
|
3555
|
+
key TEXT PRIMARY KEY,
|
|
3556
|
+
value TEXT NOT NULL
|
|
3557
|
+
);
|
|
3558
|
+
|
|
3559
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
3560
|
+
id TEXT PRIMARY KEY,
|
|
3561
|
+
entity_id TEXT NOT NULL,
|
|
3562
|
+
table_name TEXT NOT NULL,
|
|
3563
|
+
record_id TEXT NOT NULL,
|
|
3564
|
+
operation TEXT NOT NULL,
|
|
3565
|
+
payload TEXT NOT NULL,
|
|
3566
|
+
created_at INTEGER NOT NULL
|
|
3567
|
+
);
|
|
3568
|
+
|
|
3569
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
3570
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
3571
|
+
|
|
3572
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_created_at
|
|
3573
|
+
ON ${prefix}outbox (created_at);
|
|
3574
|
+
`);
|
|
3575
|
+
await createOkfV02IndexesIfColumnsExist(db, prefix);
|
|
3576
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
3577
|
+
}
|
|
3578
|
+
async function createOkfV02IndexesIfColumnsExist(db, prefix) {
|
|
3579
|
+
for (const table of ["entries", "tasks"]) {
|
|
3580
|
+
const cols = await db.getAllAsync(
|
|
3581
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
3582
|
+
);
|
|
3583
|
+
const colNames = new Set(cols.map((c) => c.name));
|
|
3584
|
+
if (colNames.has("lifecycle_status")) {
|
|
3585
|
+
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_lifecycle_status_idx ON ${prefix}${table}(lifecycle_status);`);
|
|
3586
|
+
}
|
|
3587
|
+
if (colNames.has("stale_after")) {
|
|
3588
|
+
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_stale_after_idx ON ${prefix}${table}(stale_after);`);
|
|
3589
|
+
}
|
|
3590
|
+
if (colNames.has("last_verified_at")) {
|
|
3591
|
+
await db.execAsync(`CREATE INDEX IF NOT EXISTS ${prefix}${table}_last_verified_at_idx ON ${prefix}${table}(last_verified_at);`);
|
|
3592
|
+
}
|
|
3593
|
+
}
|
|
3594
|
+
}
|
|
3595
|
+
async function createTemporalIndexesIfColumnsExist(db, prefix) {
|
|
3596
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}entries)`);
|
|
3597
|
+
const names = new Set(cols.map((c) => c.name));
|
|
3598
|
+
if (names.has("superseded_by")) {
|
|
3599
|
+
await db.execAsync(
|
|
3600
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_superseded_idx ON ${prefix}entries(entity_id, superseded_by) WHERE superseded_by IS NOT NULL;`
|
|
3601
|
+
);
|
|
3602
|
+
}
|
|
3603
|
+
if (names.has("valid_from") && names.has("valid_to")) {
|
|
3604
|
+
await db.execAsync(
|
|
3605
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_temporal_idx ON ${prefix}entries(entity_id) WHERE valid_from IS NOT NULL OR valid_to IS NOT NULL;`
|
|
3606
|
+
);
|
|
3607
|
+
}
|
|
3608
|
+
}
|
|
3609
|
+
|
|
3610
|
+
// src/services/search/fts5Sql.ts
|
|
3611
|
+
var FTS5_STATE_KEY = "fts5_index_state";
|
|
3612
|
+
var FTS5_DRAIN_CHUNK = 500;
|
|
3613
|
+
var FTS5_SWEEP_CHUNK = 500;
|
|
3614
|
+
var FTS5_RETIRED_TABLES_KEY = "fts5_retired_tables";
|
|
3615
|
+
var FTS5_RETIRED_COUNTER_KEY = "fts5_retired_counter";
|
|
3616
|
+
var FTS5_RECLAIM_MERGE_PAGES = 500;
|
|
3617
|
+
function retiredTableName(p, n) {
|
|
3618
|
+
return `${p}entries_fts_retired_${n}`;
|
|
3619
|
+
}
|
|
3620
|
+
function isRetiredTableName(p, name) {
|
|
3621
|
+
return typeof name === "string" && name.startsWith(`${p}entries_fts_retired_`) && /^\d+$/.test(name.slice(`${p}entries_fts_retired_`.length));
|
|
3622
|
+
}
|
|
3623
|
+
var FTS5_REBUILD_WM_KEY = "fts5_rebuild_wm";
|
|
3624
|
+
var FTS5_REBUILD_SNAPSHOT_KEY = "fts5_rebuild_snapshot";
|
|
3625
|
+
var FTS5_SWEEP_CURSOR_KEY = "fts5_orphan_sweep_cursor";
|
|
3626
|
+
function fts5TriggerNames(prefix) {
|
|
3627
|
+
return [`${prefix}entries_fts_ai`, `${prefix}entries_fts_au`, `${prefix}entries_fts_ad`];
|
|
3628
|
+
}
|
|
3629
|
+
function tagsExpr(col) {
|
|
3630
|
+
return `CASE WHEN json_valid(${col}) THEN
|
|
3631
|
+
CASE WHEN json_type(${col}) = 'array' THEN
|
|
3632
|
+
(SELECT group_concat(CASE type WHEN 'true' THEN 'true' WHEN 'false' THEN 'false'
|
|
3633
|
+
WHEN 'null' THEN '' ELSE value END, ' ')
|
|
3634
|
+
FROM json_each(${col}))
|
|
3635
|
+
ELSE ${col} END
|
|
3636
|
+
ELSE ${col} END`;
|
|
3637
|
+
}
|
|
3638
|
+
function fts5TablesDdl(p) {
|
|
3639
|
+
return `
|
|
3640
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS ${p}entries_fts USING fts5(
|
|
3641
|
+
id UNINDEXED, entity_id UNINDEXED, title, body, tags,
|
|
3642
|
+
tokenize = 'porter unicode61'
|
|
3643
|
+
);
|
|
3644
|
+
CREATE TABLE IF NOT EXISTS ${p}fts_map (
|
|
3645
|
+
fts_rowid INTEGER PRIMARY KEY,
|
|
3646
|
+
id TEXT NOT NULL UNIQUE,
|
|
3647
|
+
entity_id TEXT NOT NULL
|
|
3648
|
+
);
|
|
3649
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_map_entity ON ${p}fts_map(entity_id);
|
|
3650
|
+
CREATE TABLE IF NOT EXISTS ${p}fts_pending (
|
|
3651
|
+
seq INTEGER PRIMARY KEY,
|
|
3652
|
+
entity_id TEXT NOT NULL,
|
|
3653
|
+
id TEXT NOT NULL
|
|
3654
|
+
);`;
|
|
3655
|
+
}
|
|
3656
|
+
function fts5TriggersDdl(p) {
|
|
3657
|
+
const [ai, au, ad] = fts5TriggerNames(p);
|
|
3658
|
+
return `
|
|
3659
|
+
CREATE TRIGGER IF NOT EXISTS ${ai} AFTER INSERT ON ${p}entries BEGIN
|
|
3660
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (NEW.entity_id, NEW.id);
|
|
3661
|
+
END;
|
|
3662
|
+
CREATE TRIGGER IF NOT EXISTS ${au}
|
|
3663
|
+
AFTER UPDATE OF id, entity_id, title, body, tags, deleted_at ON ${p}entries BEGIN
|
|
3664
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
3665
|
+
INSERT INTO ${p}fts_pending (entity_id, id)
|
|
3666
|
+
SELECT NEW.entity_id, NEW.id WHERE NEW.id <> OLD.id OR NEW.entity_id <> OLD.entity_id;
|
|
3667
|
+
END;
|
|
3668
|
+
CREATE TRIGGER IF NOT EXISTS ${ad} AFTER DELETE ON ${p}entries BEGIN
|
|
3669
|
+
INSERT INTO ${p}fts_pending (entity_id, id) VALUES (OLD.entity_id, OLD.id);
|
|
3670
|
+
END;`;
|
|
3671
|
+
}
|
|
3672
|
+
function drainHiSql(p) {
|
|
3673
|
+
return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
3674
|
+
}
|
|
3675
|
+
function ledgerChunkSql(p, chunkIds, ledgerWhere) {
|
|
3676
|
+
return [
|
|
3677
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
|
|
3678
|
+
`DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
|
|
3679
|
+
`INSERT INTO ${p}fts_map (id, entity_id)
|
|
3680
|
+
SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.id IN (${chunkIds}) AND e.deleted_at IS NULL`,
|
|
3681
|
+
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
3682
|
+
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
3683
|
+
FROM ${p}fts_map m JOIN ${p}entries e ON e.id = m.id
|
|
3684
|
+
WHERE m.id IN (${chunkIds})`,
|
|
3685
|
+
`DELETE FROM ${p}fts_pending WHERE ${ledgerWhere}`
|
|
3686
|
+
];
|
|
3687
|
+
}
|
|
3688
|
+
function drainChunkSql(p) {
|
|
3689
|
+
return ledgerChunkSql(p, `SELECT id FROM ${p}fts_pending WHERE seq <= ?`, "seq <= ?");
|
|
3690
|
+
}
|
|
3691
|
+
function fts5PendingIndexesDdl(p) {
|
|
3692
|
+
return `
|
|
3693
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_pending_id ON ${p}fts_pending(id);
|
|
3694
|
+
CREATE INDEX IF NOT EXISTS ${p}fts_pending_entity ON ${p}fts_pending(entity_id);`;
|
|
3695
|
+
}
|
|
3696
|
+
function idDrainChunkSql(p) {
|
|
3697
|
+
return ledgerChunkSql(
|
|
3698
|
+
p,
|
|
3699
|
+
`SELECT id FROM ${p}fts_pending WHERE id IN (SELECT value FROM json_each(?))`,
|
|
3700
|
+
"id IN (SELECT value FROM json_each(?))"
|
|
3701
|
+
);
|
|
3702
|
+
}
|
|
3703
|
+
function idReindexChunkSql(p) {
|
|
3704
|
+
return ledgerChunkSql(p, "SELECT value FROM json_each(?)", "id IN (SELECT value FROM json_each(?))");
|
|
3705
|
+
}
|
|
3706
|
+
function entityDrainHiSql(p) {
|
|
3707
|
+
return `SELECT max(seq) AS hi FROM (SELECT seq FROM ${p}fts_pending WHERE entity_id = ? ORDER BY seq LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
3708
|
+
}
|
|
3709
|
+
function entityDrainChunkSql(p) {
|
|
3710
|
+
return ledgerChunkSql(p, `SELECT id FROM ${p}fts_pending WHERE entity_id = ? AND seq <= ?`, "entity_id = ? AND seq <= ?");
|
|
3711
|
+
}
|
|
3712
|
+
function fts5LedgerDropSql(p) {
|
|
3713
|
+
return [
|
|
3714
|
+
...fts5TriggerNames(p).map((n) => `DROP TRIGGER IF EXISTS ${n};`),
|
|
3715
|
+
`DROP TABLE IF EXISTS ${p}fts_pending;`,
|
|
3716
|
+
`DROP TABLE IF EXISTS ${p}fts_map;`
|
|
3717
|
+
].join("\n");
|
|
3718
|
+
}
|
|
3719
|
+
function windowHiSql(p) {
|
|
3720
|
+
return `SELECT max(rowid) AS hi FROM (SELECT rowid FROM ${p}entries WHERE rowid > ? AND rowid <= ? ORDER BY rowid LIMIT ${FTS5_DRAIN_CHUNK})`;
|
|
3721
|
+
}
|
|
3722
|
+
function windowChunkSql(p) {
|
|
3723
|
+
const chunkIds = `SELECT id FROM ${p}entries WHERE rowid > ? AND rowid <= ?`;
|
|
3724
|
+
return [
|
|
3725
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (SELECT m.fts_rowid FROM ${p}fts_map m WHERE m.id IN (${chunkIds}))`,
|
|
3726
|
+
`DELETE FROM ${p}fts_map WHERE id IN (${chunkIds})`,
|
|
3727
|
+
`INSERT INTO ${p}fts_map (id, entity_id)
|
|
3728
|
+
SELECT e.id, e.entity_id FROM ${p}entries e WHERE e.rowid > ? AND e.rowid <= ? AND e.deleted_at IS NULL`,
|
|
3729
|
+
`INSERT INTO ${p}entries_fts (rowid, id, entity_id, title, body, tags)
|
|
3730
|
+
SELECT m.fts_rowid, e.id, e.entity_id, e.title, e.body, ${tagsExpr("e.tags")}
|
|
3731
|
+
FROM ${p}entries e JOIN ${p}fts_map m ON m.id = e.id
|
|
3732
|
+
WHERE e.rowid > ? AND e.rowid <= ?`
|
|
3733
|
+
];
|
|
3734
|
+
}
|
|
3735
|
+
function sweepHiSql(p) {
|
|
3736
|
+
return `SELECT max(fts_rowid) AS hi FROM (SELECT fts_rowid FROM ${p}fts_map WHERE fts_rowid > ? AND fts_rowid <= ? ORDER BY fts_rowid LIMIT ${FTS5_SWEEP_CHUNK})`;
|
|
3737
|
+
}
|
|
3738
|
+
function sweepChunkSql(p) {
|
|
3739
|
+
const orphans = `SELECT m.fts_rowid FROM ${p}fts_map m
|
|
3740
|
+
WHERE m.fts_rowid > ? AND m.fts_rowid <= ? AND NOT EXISTS (SELECT 1 FROM ${p}entries e WHERE e.id = m.id)`;
|
|
3741
|
+
return [
|
|
3742
|
+
`DELETE FROM ${p}entries_fts WHERE rowid IN (${orphans})`,
|
|
3743
|
+
`DELETE FROM ${p}fts_map WHERE fts_rowid IN (${orphans})`
|
|
3744
|
+
];
|
|
3745
|
+
}
|
|
3746
|
+
|
|
3747
|
+
// src/db/migrations.ts
|
|
3748
|
+
var MIGRATIONS = [
|
|
3749
|
+
{
|
|
3750
|
+
version: 1,
|
|
3751
|
+
description: "Rebuild FTS5 with porter unicode61 tokenizer (superseded by v2)",
|
|
3752
|
+
run: async (_db, _prefix) => {
|
|
3753
|
+
}
|
|
3754
|
+
},
|
|
3755
|
+
{
|
|
3756
|
+
version: 2,
|
|
3757
|
+
description: "Remove FTS5; add embedding column for semantic retrieval",
|
|
3758
|
+
run: async (db, prefix) => {
|
|
3759
|
+
await db.withTransactionAsync(async (tx) => {
|
|
3760
|
+
await tx.execAsync(`
|
|
3761
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_ai;
|
|
3762
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_ad;
|
|
3763
|
+
DROP TRIGGER IF EXISTS ${prefix}entries_au;
|
|
3764
|
+
DROP TABLE IF EXISTS ${prefix}entries_fts;
|
|
3765
|
+
`);
|
|
3766
|
+
});
|
|
3767
|
+
const cols = await db.getAllAsync(
|
|
3768
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3769
|
+
);
|
|
3770
|
+
if (!cols.some((c) => c.name === "embedding")) {
|
|
3771
|
+
await db.execAsync(`ALTER TABLE ${prefix}entries ADD COLUMN embedding TEXT`);
|
|
3772
|
+
}
|
|
3773
|
+
}
|
|
3774
|
+
},
|
|
3775
|
+
{
|
|
3776
|
+
version: 3,
|
|
3777
|
+
description: "Add embedding_blob BLOB column for Float32Array vector storage",
|
|
3778
|
+
run: async (db, prefix) => {
|
|
3779
|
+
const cols = await db.getAllAsync(
|
|
3780
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3781
|
+
);
|
|
3782
|
+
if (!cols.some((c) => c.name === "embedding_blob")) {
|
|
3783
|
+
await db.execAsync(
|
|
3784
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN embedding_blob BLOB`
|
|
3785
|
+
);
|
|
3786
|
+
}
|
|
3787
|
+
}
|
|
3788
|
+
},
|
|
3789
|
+
{
|
|
3790
|
+
version: 4,
|
|
3791
|
+
description: "Create outbox table for change data capture",
|
|
3792
|
+
run: async (db, prefix) => {
|
|
3793
|
+
await db.execAsync(`
|
|
3794
|
+
CREATE TABLE IF NOT EXISTS ${prefix}outbox (
|
|
3795
|
+
id TEXT PRIMARY KEY,
|
|
3796
|
+
entity_id TEXT NOT NULL,
|
|
3797
|
+
table_name TEXT NOT NULL,
|
|
3798
|
+
record_id TEXT NOT NULL,
|
|
3799
|
+
operation TEXT NOT NULL,
|
|
3800
|
+
payload TEXT NOT NULL,
|
|
3801
|
+
created_at INTEGER NOT NULL
|
|
3802
|
+
);
|
|
3803
|
+
CREATE INDEX IF NOT EXISTS ${prefix}outbox_entity_id_created_at
|
|
3804
|
+
ON ${prefix}outbox (entity_id, created_at);
|
|
3805
|
+
`);
|
|
3806
|
+
}
|
|
3807
|
+
},
|
|
3808
|
+
{
|
|
3809
|
+
version: 5,
|
|
3810
|
+
description: "Add okf_type to entries/tasks for OKF type fidelity; create edges table for OKF graph import",
|
|
3811
|
+
run: async (db, prefix) => {
|
|
3812
|
+
for (const table of ["entries", "tasks"]) {
|
|
3813
|
+
const cols = await db.getAllAsync(
|
|
3814
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
3815
|
+
);
|
|
3816
|
+
if (!cols.some((c) => c.name === "okf_type")) {
|
|
3817
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN okf_type TEXT`);
|
|
3818
|
+
}
|
|
3819
|
+
}
|
|
3820
|
+
await db.execAsync(`
|
|
3821
|
+
CREATE TABLE IF NOT EXISTS ${prefix}edges (
|
|
3822
|
+
id TEXT PRIMARY KEY,
|
|
3823
|
+
entity_id TEXT NOT NULL,
|
|
3824
|
+
source_id TEXT NOT NULL,
|
|
3825
|
+
target_id TEXT NOT NULL,
|
|
3826
|
+
edge_type TEXT NOT NULL,
|
|
3827
|
+
created_at INTEGER NOT NULL,
|
|
3828
|
+
UNIQUE(entity_id, source_id, target_id, edge_type)
|
|
3829
|
+
);
|
|
3830
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_idx ON ${prefix}edges (entity_id);
|
|
3831
|
+
`);
|
|
3832
|
+
}
|
|
3833
|
+
},
|
|
3834
|
+
{
|
|
3835
|
+
version: 6,
|
|
3836
|
+
description: "Add entity_manifests table for per-entity ontology state",
|
|
3837
|
+
run: async (db, prefix) => {
|
|
3838
|
+
await db.execAsync(`
|
|
3839
|
+
CREATE TABLE IF NOT EXISTS ${prefix}entity_manifests (
|
|
3840
|
+
entity_id TEXT PRIMARY KEY,
|
|
3841
|
+
mode TEXT NOT NULL DEFAULT 'off',
|
|
3842
|
+
manifest_json TEXT NOT NULL DEFAULT '{"node_types":[],"edge_types":[]}',
|
|
3843
|
+
updated_at INTEGER NOT NULL
|
|
3844
|
+
);
|
|
3845
|
+
`);
|
|
3846
|
+
}
|
|
3847
|
+
},
|
|
3848
|
+
{
|
|
3849
|
+
version: 7,
|
|
3850
|
+
description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
|
|
3851
|
+
run: async (db, prefix) => {
|
|
3852
|
+
const cols = await db.getAllAsync(
|
|
3853
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3854
|
+
);
|
|
3855
|
+
if (!cols.some((c) => c.name === "ontology_checked_at")) {
|
|
3856
|
+
await db.execAsync(
|
|
3857
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
|
|
3858
|
+
);
|
|
3859
|
+
}
|
|
3860
|
+
}
|
|
3861
|
+
},
|
|
3862
|
+
{
|
|
3863
|
+
version: 8,
|
|
3864
|
+
description: "Add heal_checked_at to entries for heal recheck cooldown",
|
|
3865
|
+
run: async (db, prefix) => {
|
|
3866
|
+
const cols = await db.getAllAsync(
|
|
3867
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3868
|
+
);
|
|
3869
|
+
if (!cols.some((c) => c.name === "heal_checked_at")) {
|
|
3870
|
+
await db.execAsync(
|
|
3871
|
+
`ALTER TABLE ${prefix}entries ADD COLUMN heal_checked_at INTEGER`
|
|
3872
|
+
);
|
|
3873
|
+
}
|
|
3874
|
+
}
|
|
3875
|
+
},
|
|
3876
|
+
{
|
|
3877
|
+
version: 9,
|
|
3878
|
+
description: "add_source_ref_index",
|
|
3879
|
+
run: async (db, prefix) => {
|
|
3880
|
+
const duplicates = await db.getAllAsync(
|
|
3881
|
+
`SELECT entity_id, source_hash, COUNT(DISTINCT source_ref) AS n_refs
|
|
3882
|
+
FROM ${prefix}entries
|
|
3883
|
+
WHERE deleted_at IS NULL AND source_hash IS NOT NULL
|
|
3884
|
+
GROUP BY entity_id, source_hash
|
|
3885
|
+
HAVING COUNT(DISTINCT source_ref) > 1`
|
|
3886
|
+
);
|
|
3887
|
+
if (duplicates.length > 0) {
|
|
3888
|
+
const sample = duplicates.slice(0, 5).map((d) => `(entity_id=${d.entity_id}, source_hash=${d.source_hash.slice(0, 12)}\u2026, n_refs=${d.n_refs})`).join(", ");
|
|
3889
|
+
throw new Error(
|
|
3890
|
+
`Migration v9 (add_source_ref_index) failed: existing live rows have multiple sourceRefs sharing a hash. Found ${duplicates.length} duplicate (entity_id, source_hash) groups. First ${Math.min(5, duplicates.length)}: ${sample}. Resolve each by calling forget({ sourceRef: <loser> }) for the offending sourceRef, then re-run setup.`
|
|
3891
|
+
);
|
|
3892
|
+
}
|
|
3893
|
+
await db.execAsync(
|
|
3894
|
+
`CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
|
|
3895
|
+
id TEXT PRIMARY KEY,
|
|
3896
|
+
entity_id TEXT NOT NULL,
|
|
3897
|
+
source_hash TEXT NOT NULL,
|
|
3898
|
+
source_ref TEXT NOT NULL,
|
|
3899
|
+
created_at INTEGER NOT NULL,
|
|
3900
|
+
deleted_at INTEGER
|
|
3901
|
+
);
|
|
3902
|
+
CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
|
|
3903
|
+
ON ${prefix}source_ref_index (entity_id, source_hash)
|
|
3904
|
+
WHERE deleted_at IS NULL;`
|
|
3905
|
+
);
|
|
3906
|
+
await db.execAsync(
|
|
3907
|
+
`INSERT OR IGNORE INTO ${prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
|
|
3908
|
+
SELECT
|
|
3909
|
+
'sri:' || entity_id || ':' || source_hash,
|
|
3910
|
+
entity_id,
|
|
3911
|
+
source_hash,
|
|
3912
|
+
source_ref,
|
|
3913
|
+
updated_at,
|
|
3914
|
+
NULL
|
|
3915
|
+
FROM (
|
|
3916
|
+
SELECT
|
|
3917
|
+
entity_id, source_hash, source_ref, updated_at,
|
|
3918
|
+
ROW_NUMBER() OVER (
|
|
3919
|
+
PARTITION BY entity_id, source_hash
|
|
3920
|
+
ORDER BY updated_at ASC, id ASC
|
|
3921
|
+
) AS rn
|
|
3922
|
+
FROM ${prefix}entries
|
|
3923
|
+
WHERE deleted_at IS NULL
|
|
3924
|
+
AND source_hash IS NOT NULL
|
|
3925
|
+
AND source_ref IS NOT NULL
|
|
3926
|
+
) ranked
|
|
3927
|
+
WHERE rn = 1;`
|
|
3928
|
+
);
|
|
3929
|
+
}
|
|
3930
|
+
},
|
|
3931
|
+
{
|
|
3932
|
+
version: 10,
|
|
3933
|
+
description: "OKF v0.2: add lifecycle_status, stale_after, generated_by, last_verified_*, okf_sources, okf_verified, okf_usage_window to entries and tasks",
|
|
3934
|
+
run: async (db, prefix) => {
|
|
3935
|
+
for (const table of ["entries", "tasks"]) {
|
|
3936
|
+
const cols = await db.getAllAsync(
|
|
3937
|
+
`PRAGMA table_info(${prefix}${table})`
|
|
3938
|
+
);
|
|
3939
|
+
const existing = new Set(cols.map((c) => c.name));
|
|
3940
|
+
const adds = [
|
|
3941
|
+
["lifecycle_status", `ALTER TABLE ${prefix}${table} ADD COLUMN lifecycle_status TEXT NOT NULL DEFAULT 'stable'`],
|
|
3942
|
+
["stale_after", `ALTER TABLE ${prefix}${table} ADD COLUMN stale_after INTEGER`],
|
|
3943
|
+
["generated_by", `ALTER TABLE ${prefix}${table} ADD COLUMN generated_by TEXT`],
|
|
3944
|
+
["last_verified_at", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_at INTEGER`],
|
|
3945
|
+
["last_verified_by", `ALTER TABLE ${prefix}${table} ADD COLUMN last_verified_by TEXT`],
|
|
3946
|
+
["okf_sources", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_sources TEXT`],
|
|
3947
|
+
["okf_verified", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_verified TEXT`],
|
|
3948
|
+
["okf_usage_window", `ALTER TABLE ${prefix}${table} ADD COLUMN okf_usage_window TEXT`]
|
|
3949
|
+
];
|
|
3950
|
+
for (const [name, sql] of adds) {
|
|
3951
|
+
if (!existing.has(name)) await db.execAsync(sql);
|
|
3952
|
+
}
|
|
3953
|
+
}
|
|
3954
|
+
await db.execAsync(`
|
|
3955
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_lifecycle_status_idx ON ${prefix}entries(lifecycle_status);
|
|
3956
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_stale_after_idx ON ${prefix}entries(stale_after);
|
|
3957
|
+
CREATE INDEX IF NOT EXISTS ${prefix}entries_last_verified_at_idx ON ${prefix}entries(last_verified_at);
|
|
3958
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_lifecycle_status_idx ON ${prefix}tasks(lifecycle_status);
|
|
3959
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_stale_after_idx ON ${prefix}tasks(stale_after);
|
|
3960
|
+
CREATE INDEX IF NOT EXISTS ${prefix}tasks_last_verified_at_idx ON ${prefix}tasks(last_verified_at);
|
|
3961
|
+
`);
|
|
3962
|
+
}
|
|
3963
|
+
},
|
|
3964
|
+
{
|
|
3965
|
+
version: 11,
|
|
3966
|
+
description: "Add embedding_failed_at, embedding_failure_kind, embedding_attempts to entries for embed retry state",
|
|
3967
|
+
run: async (db, prefix) => {
|
|
3968
|
+
const cols = await db.getAllAsync(
|
|
3969
|
+
`PRAGMA table_info(${prefix}entries)`
|
|
3970
|
+
);
|
|
3971
|
+
const existing = new Set(cols.map((c) => c.name));
|
|
3972
|
+
const adds = [
|
|
3973
|
+
["embedding_failed_at", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failed_at INTEGER`],
|
|
3974
|
+
["embedding_failure_kind", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_failure_kind TEXT`],
|
|
3975
|
+
["embedding_attempts", `ALTER TABLE ${prefix}entries ADD COLUMN embedding_attempts INTEGER NOT NULL DEFAULT 0`]
|
|
3976
|
+
];
|
|
3977
|
+
for (const [name, sql] of adds) {
|
|
3978
|
+
if (!existing.has(name)) await db.execAsync(sql);
|
|
3979
|
+
}
|
|
3980
|
+
}
|
|
3981
|
+
},
|
|
3982
|
+
{
|
|
3983
|
+
version: 12,
|
|
3984
|
+
description: "Replace edges_entity_idx with composite edges(entity_id, id) for keyset paging",
|
|
3985
|
+
run: async (db, prefix) => {
|
|
3986
|
+
await db.execAsync(`
|
|
3987
|
+
CREATE INDEX IF NOT EXISTS ${prefix}edges_entity_id_idx ON ${prefix}edges(entity_id, id);
|
|
3988
|
+
DROP INDEX IF EXISTS ${prefix}edges_entity_idx;
|
|
3989
|
+
`);
|
|
3990
|
+
}
|
|
3991
|
+
},
|
|
3992
|
+
{
|
|
3993
|
+
version: 13,
|
|
3994
|
+
description: "Temporal facts: valid_from/valid_to/superseded_by/superseded_at on entries, occurred_at on events, librarian watermark on checkpoints",
|
|
3995
|
+
run: async (db, prefix) => {
|
|
3996
|
+
const adds = [
|
|
3997
|
+
["entries", "valid_from", "INTEGER"],
|
|
3998
|
+
["entries", "valid_to", "INTEGER"],
|
|
3999
|
+
["entries", "superseded_by", "TEXT"],
|
|
4000
|
+
["entries", "superseded_at", "INTEGER"],
|
|
4001
|
+
["events", "occurred_at", "INTEGER"],
|
|
4002
|
+
["checkpoints", "librarian_watermark_at", "INTEGER"],
|
|
4003
|
+
["checkpoints", "librarian_watermark_id", "TEXT"]
|
|
4004
|
+
];
|
|
4005
|
+
const existingByTable = /* @__PURE__ */ new Map();
|
|
4006
|
+
for (const [table, column, type] of adds) {
|
|
4007
|
+
let existing = existingByTable.get(table);
|
|
4008
|
+
if (!existing) {
|
|
4009
|
+
const cols = await db.getAllAsync(`PRAGMA table_info(${prefix}${table})`);
|
|
4010
|
+
existing = new Set(cols.map((c) => c.name));
|
|
4011
|
+
existingByTable.set(table, existing);
|
|
4012
|
+
}
|
|
4013
|
+
if (!existing.has(column)) {
|
|
4014
|
+
await db.execAsync(`ALTER TABLE ${prefix}${table} ADD COLUMN ${column} ${type}`);
|
|
4015
|
+
existing.add(column);
|
|
4016
|
+
}
|
|
4017
|
+
}
|
|
4018
|
+
await createTemporalIndexesIfColumnsExist(db, prefix);
|
|
4019
|
+
}
|
|
4020
|
+
},
|
|
4021
|
+
{
|
|
4022
|
+
version: 14,
|
|
4023
|
+
description: "Partial index for the every-open legacy source_type probe",
|
|
4024
|
+
run: async (db, prefix) => {
|
|
4025
|
+
await createLegacySourceTypeIndex(db, prefix);
|
|
4026
|
+
}
|
|
4027
|
+
},
|
|
4028
|
+
{
|
|
4029
|
+
version: 15,
|
|
4030
|
+
description: "fts_pending(id) and fts_pending(entity_id) indexes for the ids-first and entity-targeted drains",
|
|
4031
|
+
run: async (db, prefix) => {
|
|
4032
|
+
const row = await db.getFirstAsync(
|
|
4033
|
+
`SELECT count(*) AS n FROM sqlite_master WHERE type = 'table' AND name = ?`,
|
|
4034
|
+
[`${prefix}fts_pending`]
|
|
4035
|
+
);
|
|
4036
|
+
if (row?.n) await db.execAsync(fts5PendingIndexesDdl(prefix));
|
|
4037
|
+
}
|
|
4038
|
+
}
|
|
4039
|
+
];
|
|
4040
|
+
async function createLegacySourceTypeIndex(db, prefix) {
|
|
4041
|
+
await db.execAsync(
|
|
4042
|
+
`CREATE INDEX IF NOT EXISTS ${prefix}entries_legacy_source_type_idx ON ${prefix}entries(source_type) WHERE source_type IN ('user_document', 'agent_inferred')`
|
|
4043
|
+
);
|
|
4044
|
+
}
|
|
4045
|
+
for (let i = 1; i < MIGRATIONS.length; i++) {
|
|
4046
|
+
if (MIGRATIONS[i].version <= MIGRATIONS[i - 1].version) {
|
|
4047
|
+
throw new Error(
|
|
4048
|
+
`migrations.ts: MIGRATIONS must be in strictly ascending version order. Found version ${MIGRATIONS[i].version} after ${MIGRATIONS[i - 1].version} at index ${i}.`
|
|
4049
|
+
);
|
|
4050
|
+
}
|
|
4051
|
+
}
|
|
4052
|
+
var CURRENT_SCHEMA_VERSION = MIGRATIONS.length > 0 ? MIGRATIONS[MIGRATIONS.length - 1].version : 0;
|
|
4053
|
+
|
|
4054
|
+
// src/db/sourceRefNormalize.ts
|
|
4055
|
+
var SOURCE_REF_NORMALIZE_ALG = 1;
|
|
4056
|
+
var NORMALIZE_STAMP_SCHEMA_VERSION = 14;
|
|
4057
|
+
var NORMALIZE_STAMP_KEY = "source_ref_normalize_stamp";
|
|
4058
|
+
var ENTRIES_WM_KEY = "normalize_v14_entries_wm";
|
|
4059
|
+
var SRI_WM_KEY = "normalize_v14_sri_wm";
|
|
4060
|
+
var CHUNK = 500;
|
|
4061
|
+
function stampMatches(raw) {
|
|
4062
|
+
return parseJsonObject(raw)?.normalize_alg === SOURCE_REF_NORMALIZE_ALG;
|
|
4063
|
+
}
|
|
4064
|
+
async function writeNormalizeStamp(meta, tx) {
|
|
4065
|
+
await meta.setMeta(NORMALIZE_STAMP_KEY, JSON.stringify({ schema_version: NORMALIZE_STAMP_SCHEMA_VERSION, normalize_alg: SOURCE_REF_NORMALIZE_ALG }), tx);
|
|
4066
|
+
}
|
|
4067
|
+
async function resumeRowid(db, prefix, meta, table, key) {
|
|
4068
|
+
const wm = parseJsonObject(await meta.getMeta(key));
|
|
4069
|
+
if (typeof wm?.rowid !== "number" || typeof wm.id !== "string") return 0;
|
|
4070
|
+
const row = await db.getFirstAsync(`SELECT id FROM ${prefix}${table} WHERE rowid = ?`, [wm.rowid]);
|
|
4071
|
+
return row?.id === wm.id ? wm.rowid : 0;
|
|
4072
|
+
}
|
|
4073
|
+
async function resetNormalizeWatermarks(meta) {
|
|
4074
|
+
await meta.deleteMeta(ENTRIES_WM_KEY);
|
|
4075
|
+
await meta.deleteMeta(SRI_WM_KEY);
|
|
4076
|
+
}
|
|
4077
|
+
async function detectSchemaVersion(meta, prefix) {
|
|
4078
|
+
const stored = await meta.getMeta("schema_version");
|
|
4079
|
+
if (stored) {
|
|
4080
|
+
const v = parseInt(stored, 10);
|
|
4081
|
+
return Number.isFinite(v) ? v : 0;
|
|
4082
|
+
}
|
|
4083
|
+
const ftsDdl = await meta.getTableDdl(`${prefix}entries_fts`);
|
|
4084
|
+
return /tokenize\s*=\s*['"]porter\s+unicode61['"]/i.test(ftsDdl ?? "") ? 1 : 0;
|
|
4085
|
+
}
|
|
4086
|
+
async function runPass(db, prefix, meta, table, key, stampOnCompletion) {
|
|
4087
|
+
let cursor = await resumeRowid(db, prefix, meta, table, key);
|
|
4088
|
+
while (cursor !== null) {
|
|
4089
|
+
const from = cursor;
|
|
4090
|
+
let next = from;
|
|
4091
|
+
await db.withTransactionAsync(async (tx) => {
|
|
4092
|
+
const rows = await tx.getAllAsync(
|
|
4093
|
+
`SELECT rowid, id, source_ref FROM ${prefix}${table} WHERE rowid > ? ORDER BY rowid LIMIT ${CHUNK}`,
|
|
4094
|
+
[from]
|
|
4095
|
+
);
|
|
4096
|
+
for (const r of rows) {
|
|
4097
|
+
if (r.source_ref == null) continue;
|
|
4098
|
+
const n = normalizeSourceRef(r.source_ref);
|
|
4099
|
+
if (n === r.source_ref) continue;
|
|
4100
|
+
if (table === "entries") {
|
|
4101
|
+
await tx.runAsync(`UPDATE ${prefix}entries SET source_ref = ? WHERE rowid = ?`, [n, r.rowid]);
|
|
4102
|
+
} else if (n === null) {
|
|
4103
|
+
await tx.runAsync(`UPDATE ${prefix}source_ref_index SET deleted_at = COALESCE(deleted_at, ?) WHERE rowid = ?`, [Date.now(), r.rowid]);
|
|
4104
|
+
} else {
|
|
4105
|
+
await tx.runAsync(`UPDATE ${prefix}source_ref_index SET source_ref = ? WHERE rowid = ?`, [n, r.rowid]);
|
|
4106
|
+
}
|
|
4107
|
+
}
|
|
4108
|
+
if (rows.length < CHUNK) {
|
|
4109
|
+
await meta.deleteMeta(key, tx);
|
|
4110
|
+
if (stampOnCompletion) await writeNormalizeStamp(meta, tx);
|
|
4111
|
+
next = null;
|
|
4112
|
+
return;
|
|
4113
|
+
}
|
|
4114
|
+
const last = rows[rows.length - 1];
|
|
4115
|
+
await meta.setMeta(key, JSON.stringify({ rowid: last.rowid, id: last.id }), tx);
|
|
4116
|
+
next = last.rowid;
|
|
4117
|
+
});
|
|
4118
|
+
cursor = next;
|
|
4119
|
+
}
|
|
4120
|
+
}
|
|
4121
|
+
async function probePendingWork(db, meta, prefix) {
|
|
4122
|
+
if (!await meta.tableExists(`${prefix}entries`)) return null;
|
|
4123
|
+
if (!await meta.tableExists(`${prefix}meta`)) {
|
|
4124
|
+
return { type: "schema_migration", fromVersion: 0, toVersion: CURRENT_SCHEMA_VERSION };
|
|
4125
|
+
}
|
|
4126
|
+
const version = await detectSchemaVersion(meta, prefix);
|
|
4127
|
+
if (version < CURRENT_SCHEMA_VERSION) {
|
|
4128
|
+
return { type: "schema_migration", fromVersion: version, toVersion: CURRENT_SCHEMA_VERSION };
|
|
4129
|
+
}
|
|
4130
|
+
if (!stampMatches(await meta.getMeta(NORMALIZE_STAMP_KEY))) {
|
|
4131
|
+
return {
|
|
4132
|
+
type: "source_ref_normalize",
|
|
4133
|
+
entriesWatermark: await resumeRowid(db, prefix, meta, "entries", ENTRIES_WM_KEY),
|
|
4134
|
+
sourceRefIndexWatermark: await resumeRowid(db, prefix, meta, "source_ref_index", SRI_WM_KEY)
|
|
4135
|
+
};
|
|
4136
|
+
}
|
|
4137
|
+
return null;
|
|
4138
|
+
}
|
|
4139
|
+
async function runSourceRefNormalize(db, prefix, meta) {
|
|
4140
|
+
if (stampMatches(await meta.getMeta(NORMALIZE_STAMP_KEY))) return;
|
|
4141
|
+
await runPass(db, prefix, meta, "entries", ENTRIES_WM_KEY, false);
|
|
4142
|
+
await runPass(db, prefix, meta, "source_ref_index", SRI_WM_KEY, true);
|
|
4143
|
+
}
|
|
4144
|
+
|
|
4145
|
+
// src/services/search/fts5Progress.ts
|
|
4146
|
+
async function writeRebuildStart(tx, prefix, meta) {
|
|
4147
|
+
const e = await tx.getFirstAsync(`SELECT coalesce(max(rowid), 0) AS hi FROM ${prefix}entries`);
|
|
4148
|
+
const m = await tx.getFirstAsync(`SELECT coalesce(max(fts_rowid), 0) AS hi FROM ${prefix}fts_map`);
|
|
4149
|
+
await meta.setMeta(FTS5_REBUILD_WM_KEY, JSON.stringify({ rowid: 0, id: null }), tx);
|
|
4150
|
+
await meta.setMeta(FTS5_REBUILD_SNAPSHOT_KEY, JSON.stringify({ entries: e?.hi ?? 0, map: m?.hi ?? 0 }), tx);
|
|
4151
|
+
await meta.setMeta(FTS5_SWEEP_CURSOR_KEY, "0", tx);
|
|
4152
|
+
}
|
|
4153
|
+
async function readRebuildProgress(meta, tx) {
|
|
4154
|
+
const wm = parseJsonObject(await meta.getMeta(FTS5_REBUILD_WM_KEY, tx));
|
|
4155
|
+
const snap = parseJsonObject(await meta.getMeta(FTS5_REBUILD_SNAPSHOT_KEY, tx));
|
|
4156
|
+
const rawCursor = await meta.getMeta(FTS5_SWEEP_CURSOR_KEY, tx);
|
|
4157
|
+
const cursor = rawCursor === null ? Number.NaN : Number(rawCursor);
|
|
4158
|
+
if (typeof wm?.rowid !== "number" || !(typeof wm.id === "string" || wm.id === null)) return null;
|
|
4159
|
+
if (typeof snap?.entries !== "number" || typeof snap.map !== "number" || !Number.isFinite(cursor)) return null;
|
|
4160
|
+
return { wm: { rowid: wm.rowid, id: wm.id }, snapshot: { entries: snap.entries, map: snap.map }, cursor };
|
|
4161
|
+
}
|
|
4162
|
+
async function reanchorRebuildFingerprint(tx, prefix, deletedIds) {
|
|
4163
|
+
if (deletedIds.length === 0) return;
|
|
4164
|
+
const metaValue = async (key) => (await tx.getFirstAsync(`SELECT value FROM ${prefix}meta WHERE key = ?`, [key]))?.value ?? null;
|
|
4165
|
+
if (await metaValue(FTS5_STATE_KEY) !== "rebuilding") return;
|
|
4166
|
+
const wm = parseJsonObject(await metaValue(FTS5_REBUILD_WM_KEY));
|
|
4167
|
+
if (typeof wm?.rowid !== "number" || typeof wm.id !== "string" || !deletedIds.includes(wm.id)) return;
|
|
4168
|
+
if ((await tx.getFirstAsync(`SELECT id FROM ${prefix}entries WHERE rowid = ?`, [wm.rowid]))?.id === wm.id) return;
|
|
4169
|
+
const below = await tx.getFirstAsync(
|
|
4170
|
+
`SELECT rowid, id FROM ${prefix}entries WHERE rowid < ? ORDER BY rowid DESC LIMIT 1`,
|
|
4171
|
+
[wm.rowid]
|
|
4172
|
+
);
|
|
4173
|
+
const next = below ? { rowid: below.rowid, id: below.id } : { rowid: 0, id: null };
|
|
4174
|
+
await tx.runAsync(`UPDATE ${prefix}meta SET value = ? WHERE key = ?`, [JSON.stringify(next), FTS5_REBUILD_WM_KEY]);
|
|
4175
|
+
}
|
|
4176
|
+
async function restartRebuildAfterVacuum(db, prefix, meta) {
|
|
4177
|
+
await db.withTransactionAsync(async (tx) => {
|
|
4178
|
+
if (await meta.getMeta(FTS5_STATE_KEY, tx) !== "rebuilding") return;
|
|
4179
|
+
await writeRebuildStart(tx, prefix, meta);
|
|
4180
|
+
});
|
|
4181
|
+
}
|
|
4182
|
+
|
|
3325
4183
|
// src/services/librarian/constants.ts
|
|
3326
4184
|
var FUZZY_THRESHOLD = 0.5;
|
|
3327
4185
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
@@ -3606,6 +4464,22 @@ function classifyCandidate(candidate, neighbours, cfg, mode) {
|
|
|
3606
4464
|
if (score < cfg.novelThreshold) return { kind: "add" };
|
|
3607
4465
|
return { kind: "ambiguous" };
|
|
3608
4466
|
}
|
|
4467
|
+
async function loadStoredVectors(deps, entityId, nonLive) {
|
|
4468
|
+
const stored = [];
|
|
4469
|
+
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
4470
|
+
if (nonLive.has(row.id)) continue;
|
|
4471
|
+
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
4472
|
+
if (vec) stored.push({ id: row.id, vec });
|
|
4473
|
+
}
|
|
4474
|
+
return stored;
|
|
4475
|
+
}
|
|
4476
|
+
async function gateNeedsKeyword(deps, entityId) {
|
|
4477
|
+
if (!await deps.entryRepo.hasLiveEntry(entityId)) return false;
|
|
4478
|
+
if (typeof deps.options.llmProvider.embed !== "function") return true;
|
|
4479
|
+
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
4480
|
+
const stored = await loadStoredVectors(deps, entityId, nonLive);
|
|
4481
|
+
return stored.length === 0 || stored.some((s) => s.vec.length !== stored[0].vec.length);
|
|
4482
|
+
}
|
|
3609
4483
|
async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
3610
4484
|
const { entityId, trigger } = ctx;
|
|
3611
4485
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -3621,14 +4495,8 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
|
3621
4495
|
}
|
|
3622
4496
|
const nonLive = await deps.entryRepo.findNonLiveIdsByEntityIds([entityId], "current", Date.now());
|
|
3623
4497
|
const canEmbed = typeof deps.options.llmProvider.embed === "function";
|
|
3624
|
-
const stored = [];
|
|
3625
|
-
|
|
3626
|
-
for (const row of await deps.entryRepo.findWithEmbeddingsByEntityIds([entityId])) {
|
|
3627
|
-
if (nonLive.has(row.id)) continue;
|
|
3628
|
-
const vec = parseEmbedding(row.embedding_blob, row.embedding);
|
|
3629
|
-
if (vec) stored.push({ id: row.id, vec });
|
|
3630
|
-
}
|
|
3631
|
-
}
|
|
4498
|
+
const stored = canEmbed && unique.length > 0 ? await loadStoredVectors(deps, entityId, nonLive) : [];
|
|
4499
|
+
const hasFacts = unique.length > 0 && await deps.entryRepo.hasLiveEntry(entityId);
|
|
3632
4500
|
const gated = [];
|
|
3633
4501
|
for (const c of unique) {
|
|
3634
4502
|
let ranked = null;
|
|
@@ -3649,7 +4517,7 @@ async function gateCandidates(deps, ctx, candidates, cfg) {
|
|
|
3649
4517
|
}
|
|
3650
4518
|
}
|
|
3651
4519
|
if (!ranked) {
|
|
3652
|
-
ranked = (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null }));
|
|
4520
|
+
ranked = hasFacts ? (await deps.searchService.searchKeyword(`${c.fact.title} ${c.fact.body}`, [entityId], cfg.k + nonLive.size)).filter((h) => !nonLive.has(h.id)).slice(0, cfg.k).map((h) => ({ id: h.id, score: null })) : [];
|
|
3653
4521
|
}
|
|
3654
4522
|
const facts = ranked.length > 0 ? await deps.entryRepo.findByIds(ranked.map((r) => r.id), [entityId]) : [];
|
|
3655
4523
|
const byId = new Map(facts.map((f) => [f.id, f]));
|
|
@@ -4010,6 +4878,9 @@ var runOpsLibrarianPass = async (deps, ctx) => {
|
|
|
4010
4878
|
const sel = await selectBatch(deps, entityId, wm, ctx.meter);
|
|
4011
4879
|
if ("budgetStop" in sel) return { processedThrough: null, factsWritten: 0, budgetStop: sel.budgetStop };
|
|
4012
4880
|
if (sel.batch.length === 0) return { processedThrough: null, factsWritten: 0 };
|
|
4881
|
+
if (!deps.searchService.isKeywordAvailable() && await gateNeedsKeyword(deps, entityId)) {
|
|
4882
|
+
throw new WikiSearchUnavailableError("librarian: keyword search unavailable and this pass may need it; retry the pass");
|
|
4883
|
+
}
|
|
4013
4884
|
let ex;
|
|
4014
4885
|
try {
|
|
4015
4886
|
ex = await runExtract(deps, ctx, sel.batch);
|
|
@@ -4239,7 +5110,9 @@ var MaintenanceService = class {
|
|
|
4239
5110
|
}
|
|
4240
5111
|
}
|
|
4241
5112
|
if (vacuum) {
|
|
5113
|
+
await resetNormalizeWatermarks(this.metadataRepo);
|
|
4242
5114
|
await this.metadataRepo.vacuum();
|
|
5115
|
+
await restartRebuildAfterVacuum(this.db, this.prefix, this.metadataRepo);
|
|
4243
5116
|
}
|
|
4244
5117
|
await this.searchService.syncEntries(entityId, syncedIds);
|
|
4245
5118
|
return { entries: deletedEntries, tasks: deletedTasks, events: deletedEvents };
|
|
@@ -4538,7 +5411,8 @@ var MaintenanceService = class {
|
|
|
4538
5411
|
*
|
|
4539
5412
|
* Bounded: at most `batchSize` candidates per pass (#67). Loop on
|
|
4540
5413
|
* `result.remaining > 0` for convergence — see {@link HealResult.remaining}
|
|
4541
|
-
* for what convergence means here
|
|
5414
|
+
* for what convergence means here, and for the no-progress flags
|
|
5415
|
+
* (`searchUnavailable`, `budgetStop`) a loop must also stop on.
|
|
4542
5416
|
*/
|
|
4543
5417
|
async doRunHeal(entityId, options) {
|
|
4544
5418
|
const promptOverride = options?.promptOverride;
|
|
@@ -4584,7 +5458,11 @@ var MaintenanceService = class {
|
|
|
4584
5458
|
}
|
|
4585
5459
|
}
|
|
4586
5460
|
const healCandidates = await this.entryRepo.findHealCandidatesByEntityId(entityId, batchSize, recheckCutoff);
|
|
4587
|
-
|
|
5461
|
+
const keywordUnavailable = healCandidates.length > 0 && !this.searchService.isKeywordAvailable();
|
|
5462
|
+
if (healCandidates.length === 0 || keywordUnavailable) {
|
|
5463
|
+
if (keywordUnavailable) {
|
|
5464
|
+
emitDiagnostic(this.options, { ...diagBase, code: "heal_skipped", detail: { reason: "keyword_search_unavailable", count: healCandidates.length } });
|
|
5465
|
+
}
|
|
4588
5466
|
await this.searchService.syncEntries(entityId, Array.from(/* @__PURE__ */ new Set([...orphanedIds, ...staleDowngradedIds])));
|
|
4589
5467
|
this.searchService.evictCache(entityId);
|
|
4590
5468
|
const counts2 = await this.entryRepo.countHealCandidatesByEntityId(entityId, recheckCutoff);
|
|
@@ -4596,7 +5474,10 @@ var MaintenanceService = class {
|
|
|
4596
5474
|
skipped: [],
|
|
4597
5475
|
degraded: [],
|
|
4598
5476
|
remaining: counts2.eligible,
|
|
4599
|
-
deferred: counts2.deferred
|
|
5477
|
+
deferred: counts2.deferred,
|
|
5478
|
+
// remaining stays > 0 so maybeRunHeal holds the checkpoint back and
|
|
5479
|
+
// retries after the outage; the flag lets a host loop stop instead.
|
|
5480
|
+
...keywordUnavailable ? { searchUnavailable: true } : {}
|
|
4600
5481
|
};
|
|
4601
5482
|
}
|
|
4602
5483
|
const allTasks = await this.taskRepo.findAllPending([entityId], HEAL_MAX_TASKS);
|
|
@@ -5479,7 +6360,7 @@ var ImportExportService = class {
|
|
|
5479
6360
|
if (merge) {
|
|
5480
6361
|
await this.searchService.syncEntries(entityId, [...upsertedFactIds]);
|
|
5481
6362
|
} else {
|
|
5482
|
-
await this.searchService.sync(entityId);
|
|
6363
|
+
await this.searchService.sync(entityId, { ids: [.../* @__PURE__ */ new Set([...upsertedFactIds, ...softDeletedFactIds])] });
|
|
5483
6364
|
}
|
|
5484
6365
|
for (const fact of bundle.facts) {
|
|
5485
6366
|
if (!fact.deleted_at && upsertedFactIds.has(fact.id) && !factsWithPreservedBlob.has(fact.id)) {
|
|
@@ -6129,7 +7010,12 @@ var RetrievalService = class {
|
|
|
6129
7010
|
candidateRows = this._withoutIds(await this.entryRepo.findWithEmbeddingsByEntityIds(scoredEntityIds), excludedIds);
|
|
6130
7011
|
}
|
|
6131
7012
|
if (weight !== void 0 && weight < 1) {
|
|
6132
|
-
|
|
7013
|
+
try {
|
|
7014
|
+
miniSearchScores = await this.searchService.getKeywordScores(trimmedQuery, scoredEntityIds);
|
|
7015
|
+
} catch (err) {
|
|
7016
|
+
if (!isWikiSearchUnavailableError(err)) throw err;
|
|
7017
|
+
miniSearchScores = /* @__PURE__ */ new Map();
|
|
7018
|
+
}
|
|
6133
7019
|
}
|
|
6134
7020
|
}
|
|
6135
7021
|
if (candidateRows === null) {
|
|
@@ -6624,6 +7510,7 @@ var WriteService = class {
|
|
|
6624
7510
|
this.jobManager = jobManager;
|
|
6625
7511
|
this.maintenanceService = maintenanceService;
|
|
6626
7512
|
this.inflight = /* @__PURE__ */ new Set();
|
|
7513
|
+
this.warnedSearchUnavailable = false;
|
|
6627
7514
|
}
|
|
6628
7515
|
track(p) {
|
|
6629
7516
|
this.inflight.add(p);
|
|
@@ -6720,8 +7607,7 @@ var WriteService = class {
|
|
|
6720
7607
|
this.jobManager.acquireLock("librarian", entityId);
|
|
6721
7608
|
const stage = { job: "librarian" };
|
|
6722
7609
|
const job = this.runLibrarianThenMaybeHeal(entityId, librarianCount, prevMemoryCheckpoint, stage).catch((err) => {
|
|
6723
|
-
|
|
6724
|
-
this.reportBackgroundFailure(entityId, stage.job);
|
|
7610
|
+
this.reportBackgroundFailure(entityId, stage.job, err);
|
|
6725
7611
|
}).finally(() => {
|
|
6726
7612
|
this.jobManager.releaseLock("librarian", entityId);
|
|
6727
7613
|
});
|
|
@@ -6732,8 +7618,7 @@ var WriteService = class {
|
|
|
6732
7618
|
}
|
|
6733
7619
|
} else if (!this.jobManager.isBlocked("librarian", entityId)) {
|
|
6734
7620
|
const healJob = this.maybeRunHeal(entityId, eventCount).catch((err) => {
|
|
6735
|
-
|
|
6736
|
-
this.reportBackgroundFailure(entityId, "heal");
|
|
7621
|
+
this.reportBackgroundFailure(entityId, "heal", err);
|
|
6737
7622
|
});
|
|
6738
7623
|
this.track(healJob);
|
|
6739
7624
|
}
|
|
@@ -6771,17 +7656,23 @@ var WriteService = class {
|
|
|
6771
7656
|
}
|
|
6772
7657
|
}
|
|
6773
7658
|
}
|
|
6774
|
-
reportBackgroundFailure(entityId, job) {
|
|
7659
|
+
reportBackgroundFailure(entityId, job, err) {
|
|
7660
|
+
const searchUnavailable = isWikiSearchUnavailableError(err);
|
|
7661
|
+
if (!searchUnavailable) console.error(err);
|
|
7662
|
+
else if (!this.warnedSearchUnavailable) {
|
|
7663
|
+
this.warnedSearchUnavailable = true;
|
|
7664
|
+
console.warn("[WikiMemory] Auto librarian/heal pass skipped: keyword search is unavailable. Work is held, not lost, until it recovers (a no-arg syncSearchIndex() retries the init); later skips report only via onDiagnostic.");
|
|
7665
|
+
}
|
|
6775
7666
|
emitDiagnostic(this.options, {
|
|
6776
7667
|
code: "background_job_failed",
|
|
6777
7668
|
operation: job,
|
|
6778
7669
|
trigger: "auto",
|
|
6779
7670
|
entityId,
|
|
6780
|
-
detail: { reason: "unhandled_rejection" }
|
|
7671
|
+
detail: { reason: searchUnavailable ? "keyword_search_unavailable" : "unhandled_rejection" }
|
|
6781
7672
|
});
|
|
6782
7673
|
}
|
|
6783
7674
|
};
|
|
6784
7675
|
|
|
6785
|
-
export { BaseRepository, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MAX_EMBED_ATTEMPTS, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, TemporalService, UsageMeter, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiParseError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, edgeTripleAllowed, emptyManifest, entitySummaryMetaKey, extractSqliteCode, factTokenCost, generateId, isLiveAt, liveAtSql, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, packFactsByBudget, parseEmbedding, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, safeSlice, typeSatisfies, validateInlineEdges, validateManifest };
|
|
6786
|
-
//# sourceMappingURL=chunk-
|
|
6787
|
-
//# sourceMappingURL=chunk-
|
|
7676
|
+
export { BaseRepository, CURRENT_SCHEMA_VERSION, DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, DEFAULT_MAX_EMBED_CHARS, DiagnosticBuffer, EMBED_CHARS_CEILING, EmbeddingService, FTS5_DRAIN_CHUNK, FTS5_REBUILD_SNAPSHOT_KEY, FTS5_REBUILD_WM_KEY, FTS5_RECLAIM_MERGE_PAGES, FTS5_RETIRED_COUNTER_KEY, FTS5_RETIRED_TABLES_KEY, FTS5_STATE_KEY, FTS5_SWEEP_CURSOR_KEY, HEAL_ANCHORS_PER_CANDIDATE, HEAL_BATCH_SIZE, HEAL_MAX_FACT_BODY_CHARS_L3, HEAL_MAX_TASKS, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MAX_EMBED_ATTEMPTS, MIGRATIONS, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, TemporalService, UnavailableIndexStrategy, UsageMeter, WikiBusyError, WikiDraftNotFound, WikiDuplicateHashError, WikiGraphNodeOwnershipConflict, WikiIngestEmptyError, WikiInvalidReadOptions, WikiMigrationRequiredError, WikiParseError, WikiSearchUnavailableError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiSupersedeError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, chunkText, configureRandomSource, createLegacySourceTypeIndex, detectSchemaVersion, drainChunkSql, drainHiSql, edgeTripleAllowed, emptyManifest, entityDrainChunkSql, entityDrainHiSql, entitySummaryMetaKey, extractSqliteCode, factTokenCost, fts5LedgerDropSql, fts5PendingIndexesDdl, fts5TablesDdl, fts5TriggerNames, fts5TriggersDdl, generateId, idDrainChunkSql, idReindexChunkSql, isLiveAt, isRetiredTableName, isWikiSearchUnavailableError, liveAtSql, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, packFactsByBudget, parseEmbedding, parseJsonArray, parseJsonObject, probePendingWork, readRebuildProgress, reanchorRebuildFingerprint, resolveEdgeDefinitions, resolveGrounding, resolveNodeType, retiredTableName, runSourceRefNormalize, safeSlice, setupDatabase, sweepChunkSql, sweepHiSql, typeSatisfies, validateInlineEdges, validateManifest, windowChunkSql, windowHiSql, writeNormalizeStamp, writeRebuildStart };
|
|
7677
|
+
//# sourceMappingURL=chunk-PCRYIUV5.mjs.map
|
|
7678
|
+
//# sourceMappingURL=chunk-PCRYIUV5.mjs.map
|