@equationalapplications/core-llm-wiki 4.20.0 → 4.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -0
- package/dist/{chunk-HKB5FEJG.mjs → chunk-LCAU6NYC.mjs} +270 -94
- package/dist/chunk-LCAU6NYC.mjs.map +1 -0
- package/dist/index.d.mts +7 -3
- package/dist/index.d.ts +7 -3
- package/dist/index.js +328 -32
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +126 -11
- package/dist/index.mjs.map +1 -1
- package/dist/{testing-BVN_g414.d.mts → testing-Bk8J6QLX.d.mts} +100 -7
- package/dist/{testing-BVN_g414.d.ts → testing-Bk8J6QLX.d.ts} +100 -7
- package/dist/testing.d.mts +1 -1
- package/dist/testing.d.ts +1 -1
- package/dist/testing.js +167 -6
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +1 -1
- package/package.json +2 -2
- package/dist/chunk-HKB5FEJG.mjs.map +0 -1
package/README.md
CHANGED
|
@@ -541,6 +541,40 @@ In **Strict** and **Emergent** modes, librarian and ingest JSON may include type
|
|
|
541
541
|
|
|
542
542
|
See the design spec: [`docs/superpowers/specs/2026-06-23-per-entity-seeded-ontology-design.md`](https://github.com/equationalapplications/expo-llm-wiki/blob/main/docs/superpowers/specs/2026-06-23-per-entity-seeded-ontology-design.md).
|
|
543
543
|
|
|
544
|
+
### Ontology backfill
|
|
545
|
+
|
|
546
|
+
Facts that enter the store without passing through the librarian (synced-down
|
|
547
|
+
remote facts via `importDump`, or facts created before the ontology feature)
|
|
548
|
+
have `okf_type = NULL` and no edges, so `traverseGraph` cannot reach them.
|
|
549
|
+
`runOntologyBackfill` types them in place:
|
|
550
|
+
|
|
551
|
+
```ts
|
|
552
|
+
const result = await wiki.runOntologyBackfill(entityId);
|
|
553
|
+
// { scanned, typed, failedValidation, edgesAdded, remaining, deferred }
|
|
554
|
+
```
|
|
555
|
+
|
|
556
|
+
- **When to call it:** you own the trigger — the library provides the operation,
|
|
557
|
+
not a scheduler (same as `runLibrarian`/`runHeal`). A good default is after
|
|
558
|
+
each sync/import completes. `WikiBusyError` under concurrency is safe to
|
|
559
|
+
swallow; the next trigger retries.
|
|
560
|
+
- **Cost model:** one LLM call only when eligible untyped facts exist; otherwise
|
|
561
|
+
one SELECT. Each run scans at most 25 facts (override via
|
|
562
|
+
`options.batchSize`), oldest first, with the full serialized prompt
|
|
563
|
+
(system + user) capped at 40k chars. One exception: a single fact whose
|
|
564
|
+
prompt alone exceeds the cap is still sent by itself so it is never starved.
|
|
565
|
+
- **Convergence:** loop `while (result.remaining > 0)` to drain a backlog.
|
|
566
|
+
`remaining` counts only *eligible* untyped facts, so the loop terminates even
|
|
567
|
+
when unclassifiable facts exist; those are cooldown-stamped and retried after
|
|
568
|
+
7 days (`deferred` reports them). Because of that weekly retry, host
|
|
569
|
+
dashboards may see small recurring bumps in `scanned`/`failedValidation` as
|
|
570
|
+
the unclassifiable backlog is re-tested — expected, not a regression.
|
|
571
|
+
- **Strictly additive:** never creates, deletes, or rewrites facts, and never
|
|
572
|
+
overwrites an existing `okf_type` (guarded in SQL, not just convention).
|
|
573
|
+
`updated_at` is never touched, so sync merge resolution is unaffected.
|
|
574
|
+
- **Prompt customization:** per-call `options.promptOverride`, or global
|
|
575
|
+
`config.prompts.ontologyBackfillSystemPrompt` (template may use `{{facts}}`
|
|
576
|
+
and the ontology placeholders).
|
|
577
|
+
|
|
544
578
|
## OKF Import/Export
|
|
545
579
|
|
|
546
580
|
The core package integrates with `@equationalapplications/core-okf` to seamlessly adapt wiki data dumps to and from Open Knowledge Format (OKF) v0.1 bundles.
|
|
@@ -917,6 +951,7 @@ The flowchart shows:
|
|
|
917
951
|
| [@equationalapplications/prisma-outbox](https://github.com/equationalapplications/expo-llm-wiki/blob/main/packages/prisma-outbox/README.md) | Sync SQLite outbox events to Prisma |
|
|
918
952
|
| [@equationalapplications/core-llm-tools](https://github.com/equationalapplications/expo-llm-wiki/blob/main/packages/core-llm-tools/README.md) | Gemini tool schemas and capability injector |
|
|
919
953
|
| [@equationalapplications/core-okf](https://github.com/equationalapplications/expo-llm-wiki/blob/main/packages/okf/README.md) | Zero-dependency Open Knowledge Format (OKF) v0.1 primitives — parse and produce interoperable knowledge bundles. |
|
|
954
|
+
| [@equationalapplications/schema-org-llm-wiki](https://github.com/equationalapplications/expo-llm-wiki/blob/main/packages/schema-org/README.md) | Curated schema.org warm-agent ontology manifest |
|
|
920
955
|
|
|
921
956
|
## License
|
|
922
957
|
|
|
@@ -109,6 +109,106 @@ function generateId(prefix = "") {
|
|
|
109
109
|
);
|
|
110
110
|
}
|
|
111
111
|
|
|
112
|
+
// src/utils/ontology.ts
|
|
113
|
+
function emptyManifest() {
|
|
114
|
+
return { node_types: [], edge_types: [] };
|
|
115
|
+
}
|
|
116
|
+
function normalizeTitleKey(title) {
|
|
117
|
+
return title.trim().toLowerCase().replace(/\s+/g, " ");
|
|
118
|
+
}
|
|
119
|
+
function resolveNodeType(raw, manifest) {
|
|
120
|
+
const slug = raw.trim();
|
|
121
|
+
if (!slug) return null;
|
|
122
|
+
const hit = manifest.node_types.find((n) => n.type.toLowerCase() === slug.toLowerCase());
|
|
123
|
+
return hit?.type ?? null;
|
|
124
|
+
}
|
|
125
|
+
function resolveEdgeDefinitions(rawEdgeType, manifest) {
|
|
126
|
+
const slug = rawEdgeType.trim();
|
|
127
|
+
if (!slug) return [];
|
|
128
|
+
return manifest.edge_types.filter((e) => e.type.toLowerCase() === slug.toLowerCase());
|
|
129
|
+
}
|
|
130
|
+
function edgeTripleKey(type, sourceType, targetType) {
|
|
131
|
+
return `${type.trim().toLowerCase()}|${sourceType.trim().toLowerCase()}|${targetType.trim().toLowerCase()}`;
|
|
132
|
+
}
|
|
133
|
+
function validateManifest(manifest) {
|
|
134
|
+
const nodeSlugs = /* @__PURE__ */ new Set();
|
|
135
|
+
for (const node of manifest.node_types ?? []) {
|
|
136
|
+
const type = node.type?.trim();
|
|
137
|
+
if (!type) throw new Error("Ontology node type slug must be non-empty");
|
|
138
|
+
const key = type.toLowerCase();
|
|
139
|
+
if (nodeSlugs.has(key)) throw new Error(`Duplicate node type: ${type}`);
|
|
140
|
+
nodeSlugs.add(key);
|
|
141
|
+
}
|
|
142
|
+
const edgeKeys = /* @__PURE__ */ new Set();
|
|
143
|
+
const edgeNames = /* @__PURE__ */ new Map();
|
|
144
|
+
for (const edge of manifest.edge_types ?? []) {
|
|
145
|
+
const edgeType = edge.type?.trim();
|
|
146
|
+
const sourceType = edge.source_type?.trim();
|
|
147
|
+
const targetType = edge.target_type?.trim();
|
|
148
|
+
if (!edgeType) throw new Error("Ontology edge type slug must be non-empty");
|
|
149
|
+
if (!sourceType || !targetType || !nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) {
|
|
150
|
+
throw new Error(`Edge type ${edgeType} references unknown node type`);
|
|
151
|
+
}
|
|
152
|
+
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
153
|
+
if (edgeKeys.has(edgeKey)) {
|
|
154
|
+
throw new Error(`Duplicate edge definition: ${edgeType} (${sourceType} \u2192 ${targetType})`);
|
|
155
|
+
}
|
|
156
|
+
edgeKeys.add(edgeKey);
|
|
157
|
+
const canonical = edgeNames.get(edgeType.toLowerCase());
|
|
158
|
+
if (canonical === void 0) {
|
|
159
|
+
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
160
|
+
} else if (canonical !== edgeType) {
|
|
161
|
+
throw new Error(`Inconsistent casing for edge type: ${edgeType} conflicts with ${canonical}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
function mergeOntologyUpdates(current, updates) {
|
|
166
|
+
const node_types = [...current.node_types];
|
|
167
|
+
const edge_types = [...current.edge_types];
|
|
168
|
+
const nodeSlugs = new Set(node_types.map((n) => n.type.trim().toLowerCase()));
|
|
169
|
+
const edgeKeys = new Set(edge_types.map((e) => edgeTripleKey(e.type, e.source_type, e.target_type)));
|
|
170
|
+
const edgeNames = new Map(edge_types.map((e) => [e.type.trim().toLowerCase(), e.type.trim()]));
|
|
171
|
+
for (const node of updates.node_types ?? []) {
|
|
172
|
+
const type = node?.type?.trim();
|
|
173
|
+
if (!type) continue;
|
|
174
|
+
const key = type.toLowerCase();
|
|
175
|
+
if (nodeSlugs.has(key)) continue;
|
|
176
|
+
node_types.push({ type, description: String(node.description ?? "") });
|
|
177
|
+
nodeSlugs.add(key);
|
|
178
|
+
}
|
|
179
|
+
for (const edge of updates.edge_types ?? []) {
|
|
180
|
+
const rawEdgeType = edge?.type?.trim();
|
|
181
|
+
const sourceType = edge?.source_type?.trim();
|
|
182
|
+
const targetType = edge?.target_type?.trim();
|
|
183
|
+
if (!rawEdgeType || !sourceType || !targetType) continue;
|
|
184
|
+
const edgeType = edgeNames.get(rawEdgeType.toLowerCase()) ?? rawEdgeType;
|
|
185
|
+
const edgeKey = edgeTripleKey(edgeType, sourceType, targetType);
|
|
186
|
+
if (edgeKeys.has(edgeKey)) continue;
|
|
187
|
+
if (!nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) continue;
|
|
188
|
+
edgeNames.set(edgeType.toLowerCase(), edgeType);
|
|
189
|
+
edge_types.push({
|
|
190
|
+
type: edgeType,
|
|
191
|
+
source_type: sourceType,
|
|
192
|
+
target_type: targetType,
|
|
193
|
+
description: String(edge.description ?? "")
|
|
194
|
+
});
|
|
195
|
+
edgeKeys.add(edgeKey);
|
|
196
|
+
}
|
|
197
|
+
return { node_types, edge_types };
|
|
198
|
+
}
|
|
199
|
+
function validateInlineEdges(sourceType, _targetType, edges, manifest) {
|
|
200
|
+
if (!Array.isArray(edges)) return [];
|
|
201
|
+
const valid = [];
|
|
202
|
+
for (const edge of edges) {
|
|
203
|
+
if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string") continue;
|
|
204
|
+
const defs = resolveEdgeDefinitions(edge.edge_type, manifest);
|
|
205
|
+
const match = defs.find((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
|
|
206
|
+
if (!match) continue;
|
|
207
|
+
valid.push({ edge_type: match.type, target_title: edge.target_title });
|
|
208
|
+
}
|
|
209
|
+
return valid;
|
|
210
|
+
}
|
|
211
|
+
|
|
112
212
|
// src/utils/embedding.ts
|
|
113
213
|
function parseEmbedding(blob, text) {
|
|
114
214
|
if (blob && blob.byteLength > 0) {
|
|
@@ -374,7 +474,8 @@ var JobManager = class {
|
|
|
374
474
|
heal: (id) => this._healKey(id),
|
|
375
475
|
reembed: (id) => this._reembedKey(id),
|
|
376
476
|
import: (id) => this._importKey(id),
|
|
377
|
-
forget: (id) => this._forgetKey(id)
|
|
477
|
+
forget: (id) => this._forgetKey(id),
|
|
478
|
+
ontologyBackfill: (id) => this._ontologyBackfillKey(id)
|
|
378
479
|
};
|
|
379
480
|
}
|
|
380
481
|
_pruneKey(entityId) {
|
|
@@ -401,6 +502,9 @@ var JobManager = class {
|
|
|
401
502
|
_healKey(entityId) {
|
|
402
503
|
return `${this.prefix}:${entityId}:heal`;
|
|
403
504
|
}
|
|
505
|
+
_ontologyBackfillKey(entityId) {
|
|
506
|
+
return `${this.prefix}:${entityId}:ontologyBackfill`;
|
|
507
|
+
}
|
|
404
508
|
_isReembedActive(entityId) {
|
|
405
509
|
return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey());
|
|
406
510
|
}
|
|
@@ -451,14 +555,16 @@ var JobManager = class {
|
|
|
451
555
|
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
452
556
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
453
557
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
558
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
454
559
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
455
560
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
456
561
|
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
457
562
|
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
458
563
|
break;
|
|
459
564
|
case "librarian":
|
|
460
|
-
case "heal":
|
|
461
|
-
|
|
565
|
+
case "heal":
|
|
566
|
+
case "ontologyBackfill": {
|
|
567
|
+
const opKey = this.lockKeyFns[operation](entityId);
|
|
462
568
|
if (this.activeMaintenanceJobs.has(opKey)) blockingOperation = operation;
|
|
463
569
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
464
570
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
@@ -472,6 +578,7 @@ var JobManager = class {
|
|
|
472
578
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
473
579
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
474
580
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
581
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
475
582
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
476
583
|
else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
|
|
477
584
|
else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
|
|
@@ -482,6 +589,7 @@ var JobManager = class {
|
|
|
482
589
|
else if (this._isAnyMaintenanceActiveWithSuffix(":prune")) blockingOperation = "prune";
|
|
483
590
|
else if (this._isAnyMaintenanceActiveWithSuffix(":librarian")) blockingOperation = "librarian";
|
|
484
591
|
else if (this._isAnyMaintenanceActiveWithSuffix(":heal")) blockingOperation = "heal";
|
|
592
|
+
else if (this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill")) blockingOperation = "ontologyBackfill";
|
|
485
593
|
else if (this.activeIngestJobs.size > 0) blockingOperation = "ingest";
|
|
486
594
|
else if (this._isAnyMaintenanceActiveWithSuffix(":import")) blockingOperation = "import";
|
|
487
595
|
else if (this._isAnyMaintenanceActiveWithSuffix(":forget")) blockingOperation = "forget";
|
|
@@ -492,6 +600,7 @@ var JobManager = class {
|
|
|
492
600
|
if (this.activeMaintenanceJobs.has(selfKey)) blockingOperation = operation;
|
|
493
601
|
else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
|
|
494
602
|
else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
|
|
603
|
+
else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
|
|
495
604
|
else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
|
|
496
605
|
else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
|
|
497
606
|
else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
|
|
@@ -549,11 +658,11 @@ var JobManager = class {
|
|
|
549
658
|
if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) return true;
|
|
550
659
|
switch (operation) {
|
|
551
660
|
case "librarian":
|
|
552
|
-
return this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
553
661
|
case "heal":
|
|
554
|
-
|
|
662
|
+
case "ontologyBackfill":
|
|
663
|
+
return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
555
664
|
case "prune":
|
|
556
|
-
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
665
|
+
return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
|
|
557
666
|
default:
|
|
558
667
|
return false;
|
|
559
668
|
}
|
|
@@ -578,6 +687,7 @@ var JobManager = class {
|
|
|
578
687
|
if (this.activeMaintenanceJobs.has(this._importKey(entityId))) throw new WikiBusyError("import", entityId);
|
|
579
688
|
if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) throw new WikiBusyError("librarian", entityId);
|
|
580
689
|
if (this.activeMaintenanceJobs.has(this._healKey(entityId))) throw new WikiBusyError("heal", entityId);
|
|
690
|
+
if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) throw new WikiBusyError("ontologyBackfill", entityId);
|
|
581
691
|
if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) throw new WikiBusyError("prune", entityId);
|
|
582
692
|
if (this._isReembedActive(entityId)) throw new WikiBusyError("reembed", entityId);
|
|
583
693
|
if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);
|
|
@@ -671,6 +781,15 @@ Return ONLY a valid JSON object matching this schema:
|
|
|
671
781
|
"facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
|
|
672
782
|
}
|
|
673
783
|
Extract verbatim factual content. Do not return markdown, just raw JSON.`;
|
|
784
|
+
var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
|
|
785
|
+
Return ONLY a valid JSON object matching this schema:
|
|
786
|
+
{
|
|
787
|
+
"classifications": [
|
|
788
|
+
{ "id": "string (input fact id, copied verbatim)", "okf_type": "string (manifest node type slug)", "edges": [{ "edge_type": "string", "target_title": "string" }] }
|
|
789
|
+
]
|
|
790
|
+
}
|
|
791
|
+
If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
|
|
792
|
+
Do not return markdown, just raw JSON.`;
|
|
674
793
|
|
|
675
794
|
// src/services/PromptService.ts
|
|
676
795
|
var PromptService = class {
|
|
@@ -760,6 +879,22 @@ ${JSON.stringify(recentEvents, null, 2)}
|
|
|
760
879
|
The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
|
|
761
880
|
};
|
|
762
881
|
}
|
|
882
|
+
buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
|
|
883
|
+
const template = runtimeOverride ?? this.globalOverrides?.ontologyBackfillSystemPrompt ?? ONTOLOGY_BACKFILL_SYSTEM_PROMPT;
|
|
884
|
+
const hasFacts = /\{\{\s*facts\s*\}\}/.test(template);
|
|
885
|
+
if (hasFacts || this.hasOntologyPlaceholders(template)) {
|
|
886
|
+
return {
|
|
887
|
+
systemPrompt: this.buildSystemPrompt(template, { facts }, ontologyContext),
|
|
888
|
+
userPrompt: hasFacts ? "Please classify the facts." : `Facts:
|
|
889
|
+
${JSON.stringify(facts, null, 2)}`
|
|
890
|
+
};
|
|
891
|
+
}
|
|
892
|
+
return {
|
|
893
|
+
systemPrompt: this.appendOntology(template, ontologyContext),
|
|
894
|
+
userPrompt: `Facts:
|
|
895
|
+
${JSON.stringify(facts, null, 2)}`
|
|
896
|
+
};
|
|
897
|
+
}
|
|
763
898
|
};
|
|
764
899
|
|
|
765
900
|
// src/utils/pure.ts
|
|
@@ -977,91 +1112,6 @@ function jaccardScore(a, b) {
|
|
|
977
1112
|
return intersection.size / union.size;
|
|
978
1113
|
}
|
|
979
1114
|
|
|
980
|
-
// src/utils/ontology.ts
|
|
981
|
-
function emptyManifest() {
|
|
982
|
-
return { node_types: [], edge_types: [] };
|
|
983
|
-
}
|
|
984
|
-
function normalizeTitleKey(title) {
|
|
985
|
-
return title.trim().toLowerCase().replace(/\s+/g, " ");
|
|
986
|
-
}
|
|
987
|
-
function resolveNodeType(raw, manifest) {
|
|
988
|
-
const slug = raw.trim();
|
|
989
|
-
if (!slug) return null;
|
|
990
|
-
const hit = manifest.node_types.find((n) => n.type.toLowerCase() === slug.toLowerCase());
|
|
991
|
-
return hit?.type ?? null;
|
|
992
|
-
}
|
|
993
|
-
function resolveEdgeDefinition(rawEdgeType, manifest) {
|
|
994
|
-
const slug = rawEdgeType.trim();
|
|
995
|
-
if (!slug) return null;
|
|
996
|
-
return manifest.edge_types.find((e) => e.type.toLowerCase() === slug.toLowerCase()) ?? null;
|
|
997
|
-
}
|
|
998
|
-
function validateManifest(manifest) {
|
|
999
|
-
const nodeSlugs = /* @__PURE__ */ new Set();
|
|
1000
|
-
for (const node of manifest.node_types ?? []) {
|
|
1001
|
-
const type = node.type?.trim();
|
|
1002
|
-
if (!type) throw new Error("Ontology node type slug must be non-empty");
|
|
1003
|
-
const key = type.toLowerCase();
|
|
1004
|
-
if (nodeSlugs.has(key)) throw new Error(`Duplicate node type: ${type}`);
|
|
1005
|
-
nodeSlugs.add(key);
|
|
1006
|
-
}
|
|
1007
|
-
const edgeSlugs = /* @__PURE__ */ new Set();
|
|
1008
|
-
for (const edge of manifest.edge_types ?? []) {
|
|
1009
|
-
const edgeType = edge.type?.trim();
|
|
1010
|
-
const sourceType = edge.source_type?.trim();
|
|
1011
|
-
const targetType = edge.target_type?.trim();
|
|
1012
|
-
if (!edgeType) throw new Error("Ontology edge type slug must be non-empty");
|
|
1013
|
-
const edgeKey = edgeType.toLowerCase();
|
|
1014
|
-
if (edgeSlugs.has(edgeKey)) throw new Error(`Duplicate edge type: ${edgeType}`);
|
|
1015
|
-
edgeSlugs.add(edgeKey);
|
|
1016
|
-
if (!sourceType || !targetType || !nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) {
|
|
1017
|
-
throw new Error(`Edge type ${edgeType} references unknown node type`);
|
|
1018
|
-
}
|
|
1019
|
-
}
|
|
1020
|
-
}
|
|
1021
|
-
function mergeOntologyUpdates(current, updates) {
|
|
1022
|
-
const node_types = [...current.node_types];
|
|
1023
|
-
const edge_types = [...current.edge_types];
|
|
1024
|
-
const nodeSlugs = new Set(node_types.map((n) => n.type.trim().toLowerCase()));
|
|
1025
|
-
const edgeSlugs = new Set(edge_types.map((e) => e.type.trim().toLowerCase()));
|
|
1026
|
-
for (const node of updates.node_types ?? []) {
|
|
1027
|
-
const type = node?.type?.trim();
|
|
1028
|
-
if (!type) continue;
|
|
1029
|
-
const key = type.toLowerCase();
|
|
1030
|
-
if (nodeSlugs.has(key)) continue;
|
|
1031
|
-
node_types.push({ type, description: String(node.description ?? "") });
|
|
1032
|
-
nodeSlugs.add(key);
|
|
1033
|
-
}
|
|
1034
|
-
for (const edge of updates.edge_types ?? []) {
|
|
1035
|
-
const edgeType = edge?.type?.trim();
|
|
1036
|
-
const sourceType = edge?.source_type?.trim();
|
|
1037
|
-
const targetType = edge?.target_type?.trim();
|
|
1038
|
-
if (!edgeType || !sourceType || !targetType) continue;
|
|
1039
|
-
const edgeKey = edgeType.toLowerCase();
|
|
1040
|
-
if (edgeSlugs.has(edgeKey)) continue;
|
|
1041
|
-
if (!nodeSlugs.has(sourceType.toLowerCase()) || !nodeSlugs.has(targetType.toLowerCase())) continue;
|
|
1042
|
-
edge_types.push({
|
|
1043
|
-
type: edgeType,
|
|
1044
|
-
source_type: sourceType,
|
|
1045
|
-
target_type: targetType,
|
|
1046
|
-
description: String(edge.description ?? "")
|
|
1047
|
-
});
|
|
1048
|
-
edgeSlugs.add(edgeKey);
|
|
1049
|
-
}
|
|
1050
|
-
return { node_types, edge_types };
|
|
1051
|
-
}
|
|
1052
|
-
function validateInlineEdges(sourceType, _targetType, edges, manifest) {
|
|
1053
|
-
if (!Array.isArray(edges)) return [];
|
|
1054
|
-
const valid = [];
|
|
1055
|
-
for (const edge of edges) {
|
|
1056
|
-
if (typeof edge?.edge_type !== "string" || typeof edge?.target_title !== "string") continue;
|
|
1057
|
-
const def = resolveEdgeDefinition(edge.edge_type, manifest);
|
|
1058
|
-
if (!def) continue;
|
|
1059
|
-
if (def.source_type.toLowerCase() !== sourceType.toLowerCase()) continue;
|
|
1060
|
-
valid.push({ edge_type: def.type, target_title: edge.target_title });
|
|
1061
|
-
}
|
|
1062
|
-
return valid;
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1065
1115
|
// src/services/IngestionService.ts
|
|
1066
1116
|
var IngestionService = class {
|
|
1067
1117
|
constructor(db, prefix, options, entryRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
@@ -1376,6 +1426,9 @@ var MetadataRepository = class extends BaseRepository {
|
|
|
1376
1426
|
// src/services/MaintenanceService.ts
|
|
1377
1427
|
var FUZZY_THRESHOLD = 0.5;
|
|
1378
1428
|
var MIN_TOKENS_TO_QUALIFY = 3;
|
|
1429
|
+
var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
|
|
1430
|
+
var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
|
|
1431
|
+
var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
|
|
1379
1432
|
var MaintenanceService = class {
|
|
1380
1433
|
constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
|
|
1381
1434
|
this.db = db;
|
|
@@ -1481,6 +1534,14 @@ var MaintenanceService = class {
|
|
|
1481
1534
|
this.jobManager.releaseLock("heal", entityId);
|
|
1482
1535
|
}
|
|
1483
1536
|
}
|
|
1537
|
+
async runOntologyBackfill(entityId, options) {
|
|
1538
|
+
this.jobManager.acquireLock("ontologyBackfill", entityId);
|
|
1539
|
+
try {
|
|
1540
|
+
return await this.doRunOntologyBackfill(entityId, options);
|
|
1541
|
+
} finally {
|
|
1542
|
+
this.jobManager.releaseLock("ontologyBackfill", entityId);
|
|
1543
|
+
}
|
|
1544
|
+
}
|
|
1484
1545
|
async runReembed(entityId, opts) {
|
|
1485
1546
|
const embedFn = this.options.llmProvider.embed;
|
|
1486
1547
|
if (!embedFn) return { embedded: 0, skipped: 0, failed: 0 };
|
|
@@ -1832,6 +1893,121 @@ var MaintenanceService = class {
|
|
|
1832
1893
|
}
|
|
1833
1894
|
this.searchService.evictCache(entityId);
|
|
1834
1895
|
}
|
|
1896
|
+
/** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
|
|
1897
|
+
async doRunOntologyBackfill(entityId, options) {
|
|
1898
|
+
const batchSize = options?.batchSize ?? ONTOLOGY_BACKFILL_BATCH_SIZE;
|
|
1899
|
+
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
1900
|
+
throw new Error("Invalid batchSize: must be an integer >= 1");
|
|
1901
|
+
}
|
|
1902
|
+
const now = Date.now();
|
|
1903
|
+
const recheckCutoff = now - ONTOLOGY_BACKFILL_RECHECK_MS;
|
|
1904
|
+
const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0 };
|
|
1905
|
+
const ontologyService = this.ontologyService;
|
|
1906
|
+
if (!ontologyService) {
|
|
1907
|
+
return { ...zeroed, remaining: 0, deferred: 0 };
|
|
1908
|
+
}
|
|
1909
|
+
const { mode } = await ontologyService.getEffectiveState(entityId);
|
|
1910
|
+
if (mode === "off") {
|
|
1911
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1912
|
+
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
1913
|
+
}
|
|
1914
|
+
const candidates = await this.entryRepo.findUntypedByEntityId(entityId, batchSize, recheckCutoff);
|
|
1915
|
+
if (candidates.length === 0) {
|
|
1916
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
1917
|
+
return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
|
|
1918
|
+
}
|
|
1919
|
+
const ontologyContext = await ontologyService.buildPromptContext(entityId);
|
|
1920
|
+
const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
|
|
1921
|
+
const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
|
|
1922
|
+
facts.map(toPromptShape),
|
|
1923
|
+
options?.promptOverride,
|
|
1924
|
+
ontologyContext
|
|
1925
|
+
);
|
|
1926
|
+
const batch = [candidates[0]];
|
|
1927
|
+
let built = buildPrompt(batch);
|
|
1928
|
+
for (let i = 1; i < candidates.length; i++) {
|
|
1929
|
+
const next = buildPrompt([...batch, candidates[i]]);
|
|
1930
|
+
if (next.systemPrompt.length + next.userPrompt.length > ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS) break;
|
|
1931
|
+
batch.push(candidates[i]);
|
|
1932
|
+
built = next;
|
|
1933
|
+
}
|
|
1934
|
+
const { systemPrompt, userPrompt } = built;
|
|
1935
|
+
const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
|
|
1936
|
+
const parsed = parseJsonResponse(responseText);
|
|
1937
|
+
const classifications = Array.isArray(parsed.classifications) ? parsed.classifications : [];
|
|
1938
|
+
let typed = 0;
|
|
1939
|
+
let failedValidation = 0;
|
|
1940
|
+
let edgesAdded = 0;
|
|
1941
|
+
let abortedOntologyOff = false;
|
|
1942
|
+
await this.db.withTransactionAsync(async (tx) => {
|
|
1943
|
+
let { mode: txMode, manifest } = await ontologyService.getEffectiveState(entityId, tx);
|
|
1944
|
+
if (txMode === "off") {
|
|
1945
|
+
abortedOntologyOff = true;
|
|
1946
|
+
return;
|
|
1947
|
+
}
|
|
1948
|
+
if (txMode === "emergent" && parsed.ontology_updates) {
|
|
1949
|
+
manifest = await ontologyService.mergeEmergentUpdates(entityId, parsed.ontology_updates, tx);
|
|
1950
|
+
}
|
|
1951
|
+
const titleRows = await this.entryRepo.findTitleIndexByEntityId(entityId, tx);
|
|
1952
|
+
const titleIndex = /* @__PURE__ */ new Map();
|
|
1953
|
+
for (const row of titleRows) {
|
|
1954
|
+
titleIndex.set(normalizeTitleKey(row.title), { id: row.id, okf_type: row.okf_type });
|
|
1955
|
+
}
|
|
1956
|
+
const batchById = new Map(batch.map((f) => [f.id, f]));
|
|
1957
|
+
const applied = /* @__PURE__ */ new Set();
|
|
1958
|
+
const pendingEdges = [];
|
|
1959
|
+
for (const classification of classifications) {
|
|
1960
|
+
const fact = typeof classification.id === "string" ? batchById.get(classification.id) : void 0;
|
|
1961
|
+
if (!fact || applied.has(fact.id)) {
|
|
1962
|
+
failedValidation++;
|
|
1963
|
+
continue;
|
|
1964
|
+
}
|
|
1965
|
+
applied.add(fact.id);
|
|
1966
|
+
const normalized = ontologyService.validateAndNormalizeFact(
|
|
1967
|
+
{
|
|
1968
|
+
title: fact.title,
|
|
1969
|
+
body: fact.body,
|
|
1970
|
+
tags: fact.tags,
|
|
1971
|
+
confidence: fact.confidence,
|
|
1972
|
+
okf_type: classification.okf_type,
|
|
1973
|
+
edges: classification.edges
|
|
1974
|
+
},
|
|
1975
|
+
manifest
|
|
1976
|
+
);
|
|
1977
|
+
if (!normalized.okf_type) {
|
|
1978
|
+
failedValidation++;
|
|
1979
|
+
continue;
|
|
1980
|
+
}
|
|
1981
|
+
const result = await this.entryRepo.updateOkfType(fact.id, entityId, normalized.okf_type, tx);
|
|
1982
|
+
if (result.changes === 0) continue;
|
|
1983
|
+
typed++;
|
|
1984
|
+
titleIndex.set(normalizeTitleKey(fact.title), { id: fact.id, okf_type: normalized.okf_type });
|
|
1985
|
+
if (normalized.edges.length > 0) {
|
|
1986
|
+
pendingEdges.push({ sourceId: fact.id, sourceType: normalized.okf_type, edges: normalized.edges });
|
|
1987
|
+
}
|
|
1988
|
+
}
|
|
1989
|
+
for (const item of pendingEdges) {
|
|
1990
|
+
edgesAdded += await ontologyService.resolveAndPersistEdges(
|
|
1991
|
+
entityId,
|
|
1992
|
+
item.sourceId,
|
|
1993
|
+
item.sourceType,
|
|
1994
|
+
item.edges,
|
|
1995
|
+
manifest,
|
|
1996
|
+
titleIndex,
|
|
1997
|
+
tx,
|
|
1998
|
+
now
|
|
1999
|
+
);
|
|
2000
|
+
}
|
|
2001
|
+
await this.entryRepo.markOntologyChecked(batch.map((f) => f.id), entityId, now, tx);
|
|
2002
|
+
});
|
|
2003
|
+
if (abortedOntologyOff) {
|
|
2004
|
+
const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
2005
|
+
return { ...zeroed, remaining: 0, deferred: counts2.deferred };
|
|
2006
|
+
}
|
|
2007
|
+
this.searchService.evictCache(entityId);
|
|
2008
|
+
const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
|
|
2009
|
+
return { scanned: batch.length, typed, failedValidation, edgesAdded, remaining: counts.eligible, deferred: counts.deferred };
|
|
2010
|
+
}
|
|
1835
2011
|
_validatePruneDuration(value, name) {
|
|
1836
2012
|
if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
|
|
1837
2013
|
throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
|
|
@@ -3045,6 +3221,6 @@ var WriteService = class {
|
|
|
3045
3221
|
}
|
|
3046
3222
|
};
|
|
3047
3223
|
|
|
3048
|
-
export { BaseRepository, EmbeddingService, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding,
|
|
3049
|
-
//# sourceMappingURL=chunk-
|
|
3050
|
-
//# sourceMappingURL=chunk-
|
|
3224
|
+
export { BaseRepository, EmbeddingService, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinitions, resolveNodeType, validateInlineEdges, validateManifest };
|
|
3225
|
+
//# sourceMappingURL=chunk-LCAU6NYC.mjs.map
|
|
3226
|
+
//# sourceMappingURL=chunk-LCAU6NYC.mjs.map
|