@equationalapplications/core-llm-wiki 4.19.0 → 4.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -0
- package/dist/{chunk-3GMXRLKU.mjs → chunk-UYLYN5N4.mjs} +247 -32
- package/dist/chunk-UYLYN5N4.mjs.map +1 -0
- package/dist/index.d.mts +5 -3
- package/dist/index.d.ts +5 -3
- package/dist/index.js +416 -41
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +169 -14
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-DP3lus54.d.mts → testing-i91HR1TG.d.mts} +122 -7
- package/dist/{testing-DP3lus54.d.ts → testing-i91HR1TG.d.ts} +122 -7
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +167 -6
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-3GMXRLKU.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -541,6 +541,40 @@ In **Strict** and **Emergent** modes, librarian and ingest JSON may include type
|
|
|
541
541
|
|
|
542
542
|
See the design spec: [`docs/superpowers/specs/2026-06-23-per-entity-seeded-ontology-design.md`](https://github.com/equationalapplications/expo-llm-wiki/blob/main/docs/superpowers/specs/2026-06-23-per-entity-seeded-ontology-design.md).
|
|
543
543
|
|
|
544
|
+
### Ontology backfill
|
|
545
|
+
|
|
546
|
+
Facts that enter the store without passing through the librarian (synced-down
|
|
547
|
+
remote facts via `importDump`, or facts created before the ontology feature)
|
|
548
|
+
have `okf_type = NULL` and no edges, so `traverseGraph` cannot reach them.
|
|
549
|
+
`runOntologyBackfill` types them in place:
|
|
550
|
+
|
|
551
|
+
```ts
|
|
552
|
+
const result = await wiki.runOntologyBackfill(entityId);
|
|
553
|
+
// { scanned, typed, failedValidation, edgesAdded, remaining, deferred }
|
|
554
|
+
```
|
|
555
|
+
|
|
556
|
+
- **When to call it:** you own the trigger — the library provides the operation,
|
|
557
|
+
not a scheduler (same as `runLibrarian`/`runHeal`). A good default is after
|
|
558
|
+
each sync/import completes. `WikiBusyError` under concurrency is safe to
|
|
559
|
+
swallow; the next trigger retries.
|
|
560
|
+
- **Cost model:** one LLM call only when eligible untyped facts exist; otherwise
|
|
561
|
+
one SELECT. Each run scans at most 25 facts (override via
|
|
562
|
+
`options.batchSize`), oldest first, with the full serialized prompt
|
|
563
|
+
(system + user) capped at 40k chars. One exception: a single fact whose
|
|
564
|
+
prompt alone exceeds the cap is still sent by itself so it is never starved.
|
|
565
|
+
- **Convergence:** loop `while (result.remaining > 0)` to drain a backlog.
|
|
566
|
+
`remaining` counts only *eligible* untyped facts, so the loop terminates even
|
|
567
|
+
when unclassifiable facts exist; those are cooldown-stamped and retried after
|
|
568
|
+
7 days (`deferred` reports them). Because of that weekly retry, host
|
|
569
|
+
dashboards may see small recurring bumps in `scanned`/`failedValidation` as
|
|
570
|
+
the unclassifiable backlog is re-tested — expected, not a regression.
|
|
571
|
+
- **Strictly additive:** never creates, deletes, or rewrites facts, and never
|
|
572
|
+
overwrites an existing `okf_type` (guarded in SQL, not just convention).
|
|
573
|
+
`updated_at` is never touched, so sync merge resolution is unaffected.
|
|
574
|
+
- **Prompt customization:** per-call `options.promptOverride`, or global
|
|
575
|
+
`config.prompts.ontologyBackfillSystemPrompt` (template may use `{{facts}}`
|
|
576
|
+
and the ontology placeholders).
|
|
577
|
+
|
|
544
578
|
## OKF Import/Export
|
|
545
579
|
|
|
546
580
|
The core package integrates with `@equationalapplications/core-okf` to seamlessly adapt wiki data dumps to and from Open Knowledge Format (OKF) v0.1 bundles.
|
|
@@ -846,6 +880,31 @@ const adapter: SQLiteAdapter = {
|
|
|
846
880
|
};
|
|
847
881
|
```
|
|
848
882
|
|
|
883
|
+
## Concurrency
|
|
884
|
+
|
|
885
|
+
All write APIs are safe to call from concurrent async contexts. Transactions are
|
|
886
|
+
serialized internally on the single database connection — you never need to know that
|
|
887
|
+
SQLite forbids nested `BEGIN`, and you never need to throttle callers yourself.
|
|
888
|
+
|
|
889
|
+
- One connection per database file per process is the supported topology.
|
|
890
|
+
- Non-transactional reads are **not** serialized, so read latency is unaffected.
|
|
891
|
+
- Inside a transaction callback, use only the provided `tx` handle — never the outer
|
|
892
|
+
database handle. Using the outer handle deadlocks; opening a nested transaction throws.
|
|
893
|
+
|
|
894
|
+
### `WikiTransactionError`
|
|
895
|
+
|
|
896
|
+
Thrown when a SQLite driver error escapes a transaction (nested `BEGIN`,
|
|
897
|
+
`SQLITE_BUSY`, constraint violation). Stable `instanceof` target for observability:
|
|
898
|
+
|
|
899
|
+
- `error.cause` — the original driver error (the chain bottoms out here, not at a
|
|
900
|
+
rollback red herring).
|
|
901
|
+
- `error.sqliteErrorCode` — best-effort SQLite code (e.g. `'SQLITE_BUSY'`), present
|
|
902
|
+
when it can be parsed. Group on this in Sentry/Datadog instead of inspecting
|
|
903
|
+
`error.cause` recursively.
|
|
904
|
+
|
|
905
|
+
Domain errors thrown from your own callback logic (validation, `WikiBusyError`)
|
|
906
|
+
pass through unwrapped with their original type intact.
|
|
907
|
+
|
|
849
908
|
## How It Works
|
|
850
909
|
|
|
851
910
|
```mermaid
|
|
@@ -8,6 +8,83 @@ var __privateGet = (obj, member, getter) => (__accessCheck(obj, member, "read fr
|
|
|
8
8
|
var __privateAdd = (obj, member, value) => member.has(obj) ? __typeError("Cannot add the same private member more than once") : member instanceof WeakSet ? member.add(obj) : member.set(obj, value);
|
|
9
9
|
var __privateSet = (obj, member, value, setter) => (__accessCheck(obj, member, "write to private field"), setter ? setter.call(obj, value) : member.set(obj, value), value);
|
|
10
10
|
|
|
11
|
+
// src/db/sqliteCodes.ts
|
|
12
|
+
var SQLITE_RESULT_CODE_NAMES = {
|
|
13
|
+
1: "SQLITE_ERROR",
|
|
14
|
+
2: "SQLITE_INTERNAL",
|
|
15
|
+
3: "SQLITE_PERM",
|
|
16
|
+
4: "SQLITE_ABORT",
|
|
17
|
+
5: "SQLITE_BUSY",
|
|
18
|
+
6: "SQLITE_LOCKED",
|
|
19
|
+
7: "SQLITE_NOMEM",
|
|
20
|
+
8: "SQLITE_READONLY",
|
|
21
|
+
9: "SQLITE_INTERRUPT",
|
|
22
|
+
10: "SQLITE_IOERR",
|
|
23
|
+
11: "SQLITE_CORRUPT",
|
|
24
|
+
12: "SQLITE_NOTFOUND",
|
|
25
|
+
13: "SQLITE_FULL",
|
|
26
|
+
14: "SQLITE_CANTOPEN",
|
|
27
|
+
15: "SQLITE_PROTOCOL",
|
|
28
|
+
16: "SQLITE_EMPTY",
|
|
29
|
+
17: "SQLITE_SCHEMA",
|
|
30
|
+
18: "SQLITE_TOOBIG",
|
|
31
|
+
19: "SQLITE_CONSTRAINT",
|
|
32
|
+
20: "SQLITE_MISMATCH",
|
|
33
|
+
21: "SQLITE_MISUSE",
|
|
34
|
+
22: "SQLITE_NOLFS",
|
|
35
|
+
23: "SQLITE_AUTH",
|
|
36
|
+
24: "SQLITE_FORMAT",
|
|
37
|
+
25: "SQLITE_RANGE",
|
|
38
|
+
26: "SQLITE_NOTADB",
|
|
39
|
+
27: "SQLITE_NOTICE",
|
|
40
|
+
28: "SQLITE_WARNING",
|
|
41
|
+
100: "SQLITE_ROW",
|
|
42
|
+
101: "SQLITE_DONE"
|
|
43
|
+
};
|
|
44
|
+
function nameForSqliteResultCode(n) {
|
|
45
|
+
return SQLITE_RESULT_CODE_NAMES[n] ?? `SQLITE_${n}`;
|
|
46
|
+
}
|
|
47
|
+
function extractSqliteCode(err) {
|
|
48
|
+
if (typeof err !== "object" || err === null) return void 0;
|
|
49
|
+
const { code, message } = err;
|
|
50
|
+
if (typeof code === "string" && code.startsWith("SQLITE_")) return code;
|
|
51
|
+
if (typeof message === "string") {
|
|
52
|
+
const m = /^Error code (\d+):/.exec(message);
|
|
53
|
+
if (m) return nameForSqliteResultCode(Number(m[1]));
|
|
54
|
+
}
|
|
55
|
+
return void 0;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// src/types.ts
|
|
59
|
+
var WikiBusyError = class extends Error {
|
|
60
|
+
constructor(operation, entityId) {
|
|
61
|
+
super(`${operation} already running for entity ${entityId}`);
|
|
62
|
+
this.name = "WikiBusyError";
|
|
63
|
+
this.operation = operation;
|
|
64
|
+
this.entityId = entityId;
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
var WikiTransactionError = class extends Error {
|
|
68
|
+
constructor(message, options) {
|
|
69
|
+
super(message, options);
|
|
70
|
+
this.name = "WikiTransactionError";
|
|
71
|
+
this.sqliteErrorCode = extractSqliteCode(options.cause);
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
var PrunePartialFailureError = class extends Error {
|
|
75
|
+
constructor(deleted, failedAt, remaining, cause, deletedTasks = 0, deletedEvents = 0) {
|
|
76
|
+
super(`Prune partially failed: deleted ${deleted}, failed at ${failedAt}, ${remaining} remaining`);
|
|
77
|
+
this.name = "PrunePartialFailureError";
|
|
78
|
+
this.deleted = deleted;
|
|
79
|
+
this.failedAt = failedAt;
|
|
80
|
+
this.remaining = remaining;
|
|
81
|
+
this.deletedTasks = deletedTasks;
|
|
82
|
+
this.deletedEvents = deletedEvents;
|
|
83
|
+
this.cause = cause;
|
|
84
|
+
}
|
|
85
|
+
};
|
|
86
|
+
var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
|
|
87
|
+
|
|
11
88
|
// src/utils/ids.ts
|
|
12
89
|
var _injectedGetRandomValues = null;
|
|
13
90
|
function configureRandomSource(fn) {
|
|
@@ -278,29 +355,6 @@ _SearchService.MAX_VECTOR_CACHE_ENTITIES = 16;
|
|
|
278
355
|
_SearchService.MAX_VECTOR_CACHE_FACTS_PER_ENTITY = 500;
|
|
279
356
|
var SearchService = _SearchService;
|
|
280
357
|
|
|
281
|
-
// src/types.ts
|
|
282
|
-
var WikiBusyError = class extends Error {
|
|
283
|
-
constructor(operation, entityId) {
|
|
284
|
-
super(`${operation} already running for entity ${entityId}`);
|
|
285
|
-
this.name = "WikiBusyError";
|
|
286
|
-
this.operation = operation;
|
|
287
|
-
this.entityId = entityId;
|
|
288
|
-
}
|
|
289
|
-
};
|
|
290
|
-
var PrunePartialFailureError = class extends Error {
|
|
291
|
-
constructor(deleted, failedAt, remaining, cause, deletedTasks = 0, deletedEvents = 0) {
|
|
292
|
-
super(`Prune partially failed: deleted ${deleted}, failed at ${failedAt}, ${remaining} remaining`);
|
|
293
|
-
this.name = "PrunePartialFailureError";
|
|
294
|
-
this.deleted = deleted;
|
|
295
|
-
this.failedAt = failedAt;
|
|
296
|
-
this.remaining = remaining;
|
|
297
|
-
this.deletedTasks = deletedTasks;
|
|
298
|
-
this.deletedEvents = deletedEvents;
|
|
299
|
-
this.cause = cause;
|
|
300
|
-
}
|
|
301
|
-
};
|
|
302
|
-
var HOOK_TIMEOUT_MARKER = /* @__PURE__ */ Symbol("WikiMemoryHookTimeout");
|
|
303
|
-
|
|
304
358
|
// src/services/JobManager.ts
|
|
305
359
|
var JobManager = class {
|
|
306
360
|
constructor(prefix) {
|
|
@@ -320,7 +374,8 @@ var JobManager = class {
|
|
|
320
374
|
heal: (id) => this._healKey(id),
|
|
321
375
|
reembed: (id) => this._reembedKey(id),
|
|
322
376
|
import: (id) => this._importKey(id),
|
|
323
|
-
forget: (id) => this._forgetKey(id)
|
|
377
|
+
forget: (id) => this._forgetKey(id),
|
|
378
|
+
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
324
379
|
};
|
|
325
380
|
}
|
|
326
381
|
_pruneKey(entityId) {
|
|
@@ -347,6 +402,9 @@ var JobManager = class {
|
|
|
347
402
|
_healKey(entityId) {
|
|
348
403
|
return `${this.prefix}:${entityId}:heal`;
|
|
349
404
|
}
|
|
405
|
+
_ontologyBackfillKey(entityId) {
|
|
406
|
+
return `${this.prefix}:${entityId}:ontologyBackfill`;
|
|
407
|
+
}
|
|
350
408
|
_isReembedActive(entityId) {
|
|
351
409
|
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey());
|
|
352
410
|
}
|
|
@@ -397,14 +455,16 @@ var JobManager = class {
|
|
|
397
455
|
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
398
456
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
399
457
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
458
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
400
459
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
401
460
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
402
461
|
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
403
462
|
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
404
463
|
break;
|
|
405
464
|
case "librarian":
|
|
406
|
-
case "heal":
|
|
407
|
-
|
|
465
|
+
case "heal":
|
|
466
|
+
case "ontologyBackfill": {
|
|
467
|
+
const opKey = this.lockKeyFns[operation](entityId);
|
|
408
468
|
if (this.activeMaintenanceJobs.has(opKey)) blockingOperation = operation;
|
|
409
469
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
410
470
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
@@ -418,6 +478,7 @@ var JobManager = class {
|
|
|
418
478
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
419
479
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
420
480
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
481
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
421
482
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
422
483
|
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
423
484
|
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
@@ -428,6 +489,7 @@ var JobManager = class {
|
|
|
428
489
|
else if (this._isAnyMaintenanceActiveWithSuffix(":prune")) blockingOperation = "prune";
|
|
429
490
|
else if (this._isAnyMaintenanceActiveWithSuffix(":librarian")) blockingOperation = "librarian";
|
|
430
491
|
else if (this._isAnyMaintenanceActiveWithSuffix(":heal")) blockingOperation = "heal";
|
|
492
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill")) blockingOperation = "ontologyBackfill";
|
|
431
493
|
else if (this.activeIngestJobs.size > 0) blockingOperation = "ingest";
|
|
432
494
|
else if (this._isAnyMaintenanceActiveWithSuffix(":import")) blockingOperation = "import";
|
|
433
495
|
else if (this._isAnyMaintenanceActiveWithSuffix(":forget")) blockingOperation = "forget";
|
|
@@ -438,6 +500,7 @@ var JobManager = class {
|
|
|
438
500
|
if (this.activeMaintenanceJobs.has(selfKey)) blockingOperation = operation;
|
|
439
501
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
440
502
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
503
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
441
504
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
442
505
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
443
506
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
@@ -495,11 +558,11 @@ var JobManager = class {
|
|
|
495
558
|
if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) return true;
|
|
496
559
|
switch (operation) {
|
|
497
560
|
case "librarian":
|
|
498
|
-
return this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
499
561
|
case "heal":
|
|
500
|
-
|
|
562
|
+
case "ontologyBackfill":
|
|
563
|
+
return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
501
564
|
case "prune":
|
|
502
|
-
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
565
|
+
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
503
566
|
default:
|
|
504
567
|
return false;
|
|
505
568
|
}
|
|
@@ -524,6 +587,7 @@ var JobManager = class {
|
|
|
524
587
|
if (this.activeMaintenanceJobs.has(this._importKey(entityId))) throw new WikiBusyError("import", entityId);
|
|
525
588
|
if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) throw new WikiBusyError("librarian", entityId);
|
|
526
589
|
if (this.activeMaintenanceJobs.has(this._healKey(entityId))) throw new WikiBusyError("heal", entityId);
|
|
590
|
+
if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) throw new WikiBusyError("ontologyBackfill", entityId);
|
|
527
591
|
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) throw new WikiBusyError("prune", entityId);
|
|
528
592
|
if (this._isReembedActive(entityId)) throw new WikiBusyError("reembed", entityId);
|
|
529
593
|
if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);
|
|
@@ -617,6 +681,15 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
617
681
|
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
618
682
|
}
|
|
619
683
|
Extract verbatim factual content. Do not return markdown, just raw JSON.`;
|
|
684
|
+
var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
|
|
685
|
+
Return ONLY a valid JSON object matching this schema:
|
|
686
|
+
{
|
|
687
|
+
"classifications": [
|
|
688
|
+
{ "id": "string (input fact id, copied verbatim)", "okf_type": "string (manifest node type slug)", "edges": [{ "edge_type": "string", "target_title": "string" }] }
|
|
689
|
+
]
|
|
690
|
+
}
|
|
691
|
+
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
692
|
+
Do not return markdown, just raw JSON.`;
|
|
620
693
|
|
|
621
694
|
// src/services/PromptService.ts
|
|
622
695
|
var PromptService = class {
|
|
@@ -706,6 +779,22 @@ ${JSON.stringify(recentEvents, null, 2)}
|
|
|
706
779
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
707
780
|
};
|
|
708
781
|
}
|
|
782
|
+
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
783
|
+
const template = runtimeOverride ?? this.globalOverrides?.ontologyBackfillSystemPrompt ?? ONTOLOGY_BACKFILL_SYSTEM_PROMPT;
|
|
784
|
+
const hasFacts = /\{\{\s*facts\s*\}\}/.test(template);
|
|
785
|
+
if (hasFacts || this.hasOntologyPlaceholders(template)) {
|
|
786
|
+
return {
|
|
787
|
+
systemPrompt: this.buildSystemPrompt(template, { facts }, ontologyContext),
|
|
788
|
+
userPrompt: hasFacts ? "Please classify the facts." : `Facts:
|
|
789
|
+
${JSON.stringify(facts, null, 2)}`
|
|
790
|
+
};
|
|
791
|
+
}
|
|
792
|
+
return {
|
|
793
|
+
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
794
|
+
userPrompt: `Facts:
|
|
795
|
+
${JSON.stringify(facts, null, 2)}`
|
|
796
|
+
};
|
|
797
|
+
}
|
|
709
798
|
};
|
|
710
799
|
|
|
711
800
|
// src/utils/pure.ts
|
|
@@ -1322,6 +1411,9 @@ var MetadataRepository = class extends BaseRepository {
|
|
|
1322
1411
|
// src/services/MaintenanceService.ts
|
|
1323
1412
|
var FUZZY_THRESHOLD = 0.5;
|
|
1324
1413
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
1414
|
+
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
1415
|
+
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
1416
|
+
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
1325
1417
|
var MaintenanceService = class {
|
|
1326
1418
|
constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1327
1419
|
this.db = db;
|
|
@@ -1427,6 +1519,14 @@ var MaintenanceService = class {
|
|
|
1427
1519
|
this.jobManager.releaseLock("heal", entityId);
|
|
1428
1520
|
}
|
|
1429
1521
|
}
|
|
1522
|
+
async runOntologyBackfill(entityId, options) {
|
|
1523
|
+
this.jobManager.acquireLock("ontologyBackfill", entityId);
|
|
1524
|
+
try {
|
|
1525
|
+
return await this.doRunOntologyBackfill(entityId, options);
|
|
1526
|
+
} finally {
|
|
1527
|
+
this.jobManager.releaseLock("ontologyBackfill", entityId);
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1430
1530
|
async runReembed(entityId, opts) {
|
|
1431
1531
|
const embedFn = this.options.llmProvider.embed;
|
|
1432
1532
|
if (!embedFn) return { embedded: 0, skipped: 0, failed: 0 };
|
|
@@ -1778,6 +1878,121 @@ var MaintenanceService = class {
|
|
|
1778
1878
|
}
|
|
1779
1879
|
this.searchService.evictCache(entityId);
|
|
1780
1880
|
}
|
|
1881
|
+
/** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
|
|
1882
|
+
async doRunOntologyBackfill(entityId, options) {
|
|
1883
|
+
const batchSize = options?.batchSize ?? ONTOLOGY_BACKFILL_BATCH_SIZE;
|
|
1884
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
1885
|
+
throw new Error("Invalid batchSize: must be an integer >= 1");
|
|
1886
|
+
}
|
|
1887
|
+
const now = Date.now();
|
|
1888
|
+
const recheckCutoff = now - ONTOLOGY_BACKFILL_RECHECK_MS;
|
|
1889
|
+
const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0 };
|
|
1890
|
+
const ontologyService = this.ontologyService;
|
|
1891
|
+
if (!ontologyService) {
|
|
1892
|
+
return { ...zeroed, remaining: 0, deferred: 0 };
|
|
1893
|
+
}
|
|
1894
|
+
const { mode } = await ontologyService.getEffectiveState(entityId);
|
|
1895
|
+
if (mode === "off") {
|
|
1896
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1897
|
+
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
1898
|
+
}
|
|
1899
|
+
const candidates = await this.entryRepo.findUntypedByEntityId(entityId, batchSize, recheckCutoff);
|
|
1900
|
+
if (candidates.length === 0) {
|
|
1901
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1902
|
+
return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
|
|
1903
|
+
}
|
|
1904
|
+
const ontologyContext = await ontologyService.buildPromptContext(entityId);
|
|
1905
|
+
const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
|
|
1906
|
+
const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
|
|
1907
|
+
facts.map(toPromptShape),
|
|
1908
|
+
options?.promptOverride,
|
|
1909
|
+
ontologyContext
|
|
1910
|
+
);
|
|
1911
|
+
const batch = [candidates[0]];
|
|
1912
|
+
let built = buildPrompt(batch);
|
|
1913
|
+
for (let i = 1; i < candidates.length; i++) {
|
|
1914
|
+
const next = buildPrompt([...batch, candidates[i]]);
|
|
1915
|
+
if (next.systemPrompt.length + next.userPrompt.length > ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS) break;
|
|
1916
|
+
batch.push(candidates[i]);
|
|
1917
|
+
built = next;
|
|
1918
|
+
}
|
|
1919
|
+
const { systemPrompt, userPrompt } = built;
|
|
1920
|
+
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
1921
|
+
const parsed = parseJsonResponse(responseText);
|
|
1922
|
+
const classifications = Array.isArray(parsed.classifications) ? parsed.classifications : [];
|
|
1923
|
+
let typed = 0;
|
|
1924
|
+
let failedValidation = 0;
|
|
1925
|
+
let edgesAdded = 0;
|
|
1926
|
+
let abortedOntologyOff = false;
|
|
1927
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
1928
|
+
let { mode: txMode, manifest } = await ontologyService.getEffectiveState(entityId, tx);
|
|
1929
|
+
if (txMode === "off") {
|
|
1930
|
+
abortedOntologyOff = true;
|
|
1931
|
+
return;
|
|
1932
|
+
}
|
|
1933
|
+
if (txMode === "emergent" && parsed.ontology_updates) {
|
|
1934
|
+
manifest = await ontologyService.mergeEmergentUpdates(entityId, parsed.ontology_updates, tx);
|
|
1935
|
+
}
|
|
1936
|
+
const titleRows = await this.entryRepo.findTitleIndexByEntityId(entityId, tx);
|
|
1937
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
1938
|
+
for (const row of titleRows) {
|
|
1939
|
+
titleIndex.set(normalizeTitleKey(row.title), { id: row.id, okf_type: row.okf_type });
|
|
1940
|
+
}
|
|
1941
|
+
const batchById = new Map(batch.map((f) => [f.id, f]));
|
|
1942
|
+
const applied = /* @__PURE__ */ new Set();
|
|
1943
|
+
const pendingEdges = [];
|
|
1944
|
+
for (const classification of classifications) {
|
|
1945
|
+
const fact = typeof classification.id === "string" ? batchById.get(classification.id) : void 0;
|
|
1946
|
+
if (!fact || applied.has(fact.id)) {
|
|
1947
|
+
failedValidation++;
|
|
1948
|
+
continue;
|
|
1949
|
+
}
|
|
1950
|
+
applied.add(fact.id);
|
|
1951
|
+
const normalized = ontologyService.validateAndNormalizeFact(
|
|
1952
|
+
{
|
|
1953
|
+
title: fact.title,
|
|
1954
|
+
body: fact.body,
|
|
1955
|
+
tags: fact.tags,
|
|
1956
|
+
confidence: fact.confidence,
|
|
1957
|
+
okf_type: classification.okf_type,
|
|
1958
|
+
edges: classification.edges
|
|
1959
|
+
},
|
|
1960
|
+
manifest
|
|
1961
|
+
);
|
|
1962
|
+
if (!normalized.okf_type) {
|
|
1963
|
+
failedValidation++;
|
|
1964
|
+
continue;
|
|
1965
|
+
}
|
|
1966
|
+
const result = await this.entryRepo.updateOkfType(fact.id, entityId, normalized.okf_type, tx);
|
|
1967
|
+
if (result.changes === 0) continue;
|
|
1968
|
+
typed++;
|
|
1969
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id: fact.id, okf_type: normalized.okf_type });
|
|
1970
|
+
if (normalized.edges.length > 0) {
|
|
1971
|
+
pendingEdges.push({ sourceId: fact.id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1974
|
+
for (const item of pendingEdges) {
|
|
1975
|
+
edgesAdded += await ontologyService.resolveAndPersistEdges(
|
|
1976
|
+
entityId,
|
|
1977
|
+
item.sourceId,
|
|
1978
|
+
item.sourceType,
|
|
1979
|
+
item.edges,
|
|
1980
|
+
manifest,
|
|
1981
|
+
titleIndex,
|
|
1982
|
+
tx,
|
|
1983
|
+
now
|
|
1984
|
+
);
|
|
1985
|
+
}
|
|
1986
|
+
await this.entryRepo.markOntologyChecked(batch.map((f) => f.id), entityId, now, tx);
|
|
1987
|
+
});
|
|
1988
|
+
if (abortedOntologyOff) {
|
|
1989
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1990
|
+
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
1991
|
+
}
|
|
1992
|
+
this.searchService.evictCache(entityId);
|
|
1993
|
+
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1994
|
+
return { scanned: batch.length, typed, failedValidation, edgesAdded, remaining: counts.eligible, deferred: counts.deferred };
|
|
1995
|
+
}
|
|
1781
1996
|
_validatePruneDuration(value, name) {
|
|
1782
1997
|
if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
|
|
1783
1998
|
throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
|
|
@@ -2991,6 +3206,6 @@ var WriteService = class {
|
|
|
2991
3206
|
}
|
|
2992
3207
|
};
|
|
2993
3208
|
|
|
2994
|
-
export { BaseRepository, EmbeddingService, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WriteService, __privateAdd, __privateGet, __privateSet, configureRandomSource, emptyManifest, entitySummaryMetaKey, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinition, resolveNodeType, validateInlineEdges };
|
|
2995
|
-
//# sourceMappingURL=chunk-
|
|
2996
|
-
//# sourceMappingURL=chunk-
|
|
3209
|
+
export { BaseRepository, EmbeddingService, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinition, resolveNodeType, validateInlineEdges };
|
|
3210
|
+
//# sourceMappingURL=chunk-UYLYN5N4.mjs.map
|
|
3211
|
+
//# sourceMappingURL=chunk-UYLYN5N4.mjs.map
|