@equationalapplications/core-llm-wiki 4.20.0 → 4.21.0

package/README.md

@@ -541,6 +541,40 @@ In **Strict** and **Emergent** modes, librarian and ingest JSON may include type
 
 See the design spec: [`docs/superpowers/specs/2026-06-23-per-entity-seeded-ontology-design.md`](https://github.com/equationalapplications/expo-llm-wiki/blob/main/docs/superpowers/specs/2026-06-23-per-entity-seeded-ontology-design.md).
 
+### Ontology backfill
+
+Facts that enter the store without passing through the librarian (synced-down
+remote facts via `importDump`, or facts created before the ontology feature)
+have `okf_type = NULL` and no edges, so `traverseGraph` cannot reach them.
+`runOntologyBackfill` types them in place:
+
+```ts
+const result = await wiki.runOntologyBackfill(entityId);
+// { scanned, typed, failedValidation, edgesAdded, remaining, deferred }
+```
+
+- **When to call it:** you own the trigger — the library provides the operation,
+  not a scheduler (same as `runLibrarian`/`runHeal`). A good default is after
+  each sync/import completes. `WikiBusyError` under concurrency is safe to
+  swallow; the next trigger retries.
+- **Cost model:** one LLM call only when eligible untyped facts exist; otherwise
+  one SELECT. Each run scans at most 25 facts (override via
+  `options.batchSize`), oldest first, with the full serialized prompt
+  (system + user) capped at 40k chars. One exception: a single fact whose
+  prompt alone exceeds the cap is still sent by itself so it is never starved.
+- **Convergence:** loop `while (result.remaining > 0)` to drain a backlog.
+  `remaining` counts only *eligible* untyped facts, so the loop terminates even
+  when unclassifiable facts exist; those are cooldown-stamped and retried after
+  7 days (`deferred` reports them). Because of that weekly retry, host
+  dashboards may see small recurring bumps in `scanned`/`failedValidation` as
+  the unclassifiable backlog is re-tested — expected, not a regression.
+- **Strictly additive:** never creates, deletes, or rewrites facts, and never
+  overwrites an existing `okf_type` (guarded in SQL, not just convention).
+  `updated_at` is never touched, so sync merge resolution is unaffected.
+- **Prompt customization:** per-call `options.promptOverride`, or global
+  `config.prompts.ontologyBackfillSystemPrompt` (template may use `{{facts}}`
+  and the ontology placeholders).
+
 ## OKF Import/Export
 
 The core package integrates with `@equationalapplications/core-okf` to seamlessly adapt wiki data dumps to and from Open Knowledge Format (OKF) v0.1 bundles.

package/dist/{chunk-HKB5FEJG.mjs → chunk-UYLYN5N4.mjs}

@@ -374,7 +374,8 @@ var JobManager = class {
       heal: (id) => this._healKey(id),
       reembed: (id) => this._reembedKey(id),
       import: (id) => this._importKey(id),
-      forget: (id) => this._forgetKey(id)
+      forget: (id) => this._forgetKey(id),
+      ontologyBackfill: (id) => this._ontologyBackfillKey(id)
     };
   }
   _pruneKey(entityId) {
@@ -401,6 +402,9 @@ var JobManager = class {
   _healKey(entityId) {
     return `${this.prefix}:${entityId}:heal`;
   }
+  _ontologyBackfillKey(entityId) {
+    return `${this.prefix}:${entityId}:ontologyBackfill`;
+  }
   _isReembedActive(entityId) {
     return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey());
   }
@@ -451,14 +455,16 @@ var JobManager = class {
         if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
         else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
         else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
+        else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
         else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
         else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
         else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
         else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
         break;
       case "librarian":
-      case "heal": {
-        const opKey = operation === "librarian" ? this._librarianKey(entityId) : this._healKey(entityId);
+      case "heal":
+      case "ontologyBackfill": {
+        const opKey = this.lockKeyFns[operation](entityId);
         if (this.activeMaintenanceJobs.has(opKey)) blockingOperation = operation;
         else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
         else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
@@ -472,6 +478,7 @@ var JobManager = class {
         else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
         else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
         else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
+        else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
         else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
         else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
         else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
@@ -482,6 +489,7 @@ var JobManager = class {
         else if (this._isAnyMaintenanceActiveWithSuffix(":prune")) blockingOperation = "prune";
         else if (this._isAnyMaintenanceActiveWithSuffix(":librarian")) blockingOperation = "librarian";
         else if (this._isAnyMaintenanceActiveWithSuffix(":heal")) blockingOperation = "heal";
+        else if (this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill")) blockingOperation = "ontologyBackfill";
         else if (this.activeIngestJobs.size > 0) blockingOperation = "ingest";
         else if (this._isAnyMaintenanceActiveWithSuffix(":import")) blockingOperation = "import";
         else if (this._isAnyMaintenanceActiveWithSuffix(":forget")) blockingOperation = "forget";
@@ -492,6 +500,7 @@ var JobManager = class {
         if (this.activeMaintenanceJobs.has(selfKey)) blockingOperation = operation;
         else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
         else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
+        else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
         else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
         else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
         else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
@@ -549,11 +558,11 @@ var JobManager = class {
     if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) return true;
     switch (operation) {
       case "librarian":
-        return this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
       case "heal":
-        return this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
+      case "ontologyBackfill":
+        return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
       case "prune":
-        return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
+        return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
       default:
         return false;
     }
@@ -578,6 +587,7 @@ var JobManager = class {
       if (this.activeMaintenanceJobs.has(this._importKey(entityId))) throw new WikiBusyError("import", entityId);
       if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) throw new WikiBusyError("librarian", entityId);
       if (this.activeMaintenanceJobs.has(this._healKey(entityId))) throw new WikiBusyError("heal", entityId);
+      if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) throw new WikiBusyError("ontologyBackfill", entityId);
       if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) throw new WikiBusyError("prune", entityId);
       if (this._isReembedActive(entityId)) throw new WikiBusyError("reembed", entityId);
       if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);
@@ -671,6 +681,15 @@ Return ONLY a valid JSON object matching this schema:
   "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
 }
 Extract verbatim factual content. Do not return markdown, just raw JSON.`;
+var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
+Return ONLY a valid JSON object matching this schema:
+{
+  "classifications": [
+    { "id": "string (input fact id, copied verbatim)", "okf_type": "string (manifest node type slug)", "edges": [{ "edge_type": "string", "target_title": "string" }] }
+  ]
+}
+If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
+Do not return markdown, just raw JSON.`;
 
 // src/services/PromptService.ts
 var PromptService = class {
@@ -760,6 +779,22 @@ ${JSON.stringify(recentEvents, null, 2)}
 The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
     };
   }
+  buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
+    const template = runtimeOverride ?? this.globalOverrides?.ontologyBackfillSystemPrompt ?? ONTOLOGY_BACKFILL_SYSTEM_PROMPT;
+    const hasFacts = /\{\{\s*facts\s*\}\}/.test(template);
+    if (hasFacts || this.hasOntologyPlaceholders(template)) {
+      return {
+        systemPrompt: this.buildSystemPrompt(template, { facts }, ontologyContext),
+        userPrompt: hasFacts ? "Please classify the facts." : `Facts:
+${JSON.stringify(facts, null, 2)}`
+      };
+    }
+    return {
+      systemPrompt: this.appendOntology(template, ontologyContext),
+      userPrompt: `Facts:
+${JSON.stringify(facts, null, 2)}`
+    };
+  }
 };
 
 // src/utils/pure.ts
@@ -1376,6 +1411,9 @@ var MetadataRepository = class extends BaseRepository {
 // src/services/MaintenanceService.ts
 var FUZZY_THRESHOLD = 0.5;
 var MIN_TOKENS_TO_QUALIFY = 3;
+var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
+var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
+var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
 var MaintenanceService = class {
   constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
     this.db = db;
@@ -1481,6 +1519,14 @@ var MaintenanceService = class {
       this.jobManager.releaseLock("heal", entityId);
     }
   }
+  async runOntologyBackfill(entityId, options) {
+    this.jobManager.acquireLock("ontologyBackfill", entityId);
+    try {
+      return await this.doRunOntologyBackfill(entityId, options);
+    } finally {
+      this.jobManager.releaseLock("ontologyBackfill", entityId);
+    }
+  }
   async runReembed(entityId, opts) {
     const embedFn = this.options.llmProvider.embed;
     if (!embedFn) return { embedded: 0, skipped: 0, failed: 0 };
@@ -1832,6 +1878,121 @@ var MaintenanceService = class {
     }
     this.searchService.evictCache(entityId);
   }
+  /** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
+  async doRunOntologyBackfill(entityId, options) {
+    const batchSize = options?.batchSize ?? ONTOLOGY_BACKFILL_BATCH_SIZE;
+    if (!Number.isInteger(batchSize) || batchSize < 1) {
+      throw new Error("Invalid batchSize: must be an integer >= 1");
+    }
+    const now = Date.now();
+    const recheckCutoff = now - ONTOLOGY_BACKFILL_RECHECK_MS;
+    const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0 };
+    const ontologyService = this.ontologyService;
+    if (!ontologyService) {
+      return { ...zeroed, remaining: 0, deferred: 0 };
+    }
+    const { mode } = await ontologyService.getEffectiveState(entityId);
+    if (mode === "off") {
+      const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+      return { ...zeroed, remaining: 0, deferred: counts2.deferred };
+    }
+    const candidates = await this.entryRepo.findUntypedByEntityId(entityId, batchSize, recheckCutoff);
+    if (candidates.length === 0) {
+      const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+      return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
+    }
+    const ontologyContext = await ontologyService.buildPromptContext(entityId);
+    const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
+    const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
+      facts.map(toPromptShape),
+      options?.promptOverride,
+      ontologyContext
+    );
+    const batch = [candidates[0]];
+    let built = buildPrompt(batch);
+    for (let i = 1; i < candidates.length; i++) {
+      const next = buildPrompt([...batch, candidates[i]]);
+      if (next.systemPrompt.length + next.userPrompt.length > ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS) break;
+      batch.push(candidates[i]);
+      built = next;
+    }
+    const { systemPrompt, userPrompt } = built;
+    const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
+    const parsed = parseJsonResponse(responseText);
+    const classifications = Array.isArray(parsed.classifications) ? parsed.classifications : [];
+    let typed = 0;
+    let failedValidation = 0;
+    let edgesAdded = 0;
+    let abortedOntologyOff = false;
+    await this.db.withTransactionAsync(async (tx) => {
+      let { mode: txMode, manifest } = await ontologyService.getEffectiveState(entityId, tx);
+      if (txMode === "off") {
+        abortedOntologyOff = true;
+        return;
+      }
+      if (txMode === "emergent" && parsed.ontology_updates) {
+        manifest = await ontologyService.mergeEmergentUpdates(entityId, parsed.ontology_updates, tx);
+      }
+      const titleRows = await this.entryRepo.findTitleIndexByEntityId(entityId, tx);
+      const titleIndex = /* @__PURE__ */ new Map();
+      for (const row of titleRows) {
+        titleIndex.set(normalizeTitleKey(row.title), { id: row.id, okf_type: row.okf_type });
+      }
+      const batchById = new Map(batch.map((f) => [f.id, f]));
+      const applied = /* @__PURE__ */ new Set();
+      const pendingEdges = [];
+      for (const classification of classifications) {
+        const fact = typeof classification.id === "string" ? batchById.get(classification.id) : void 0;
+        if (!fact || applied.has(fact.id)) {
+          failedValidation++;
+          continue;
+        }
+        applied.add(fact.id);
+        const normalized = ontologyService.validateAndNormalizeFact(
+          {
+            title: fact.title,
+            body: fact.body,
+            tags: fact.tags,
+            confidence: fact.confidence,
+            okf_type: classification.okf_type,
+            edges: classification.edges
+          },
+          manifest
+        );
+        if (!normalized.okf_type) {
+          failedValidation++;
+          continue;
+        }
+        const result = await this.entryRepo.updateOkfType(fact.id, entityId, normalized.okf_type, tx);
+        if (result.changes === 0) continue;
+        typed++;
+        titleIndex.set(normalizeTitleKey(fact.title), { id: fact.id, okf_type: normalized.okf_type });
+        if (normalized.edges.length > 0) {
+          pendingEdges.push({ sourceId: fact.id, sourceType: normalized.okf_type, edges: normalized.edges });
+        }
+      }
+      for (const item of pendingEdges) {
+        edgesAdded += await ontologyService.resolveAndPersistEdges(
+          entityId,
+          item.sourceId,
+          item.sourceType,
+          item.edges,
+          manifest,
+          titleIndex,
+          tx,
+          now
+        );
+      }
+      await this.entryRepo.markOntologyChecked(batch.map((f) => f.id), entityId, now, tx);
+    });
+    if (abortedOntologyOff) {
+      const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+      return { ...zeroed, remaining: 0, deferred: counts2.deferred };
+    }
+    this.searchService.evictCache(entityId);
+    const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+    return { scanned: batch.length, typed, failedValidation, edgesAdded, remaining: counts.eligible, deferred: counts.deferred };
+  }
   _validatePruneDuration(value, name) {
     if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
       throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
@@ -3045,6 +3206,6 @@ var WriteService = class {
   }
 };
 
-export { BaseRepository, EmbeddingService, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinition, resolveNodeType, validateInlineEdges };
-//# sourceMappingURL=chunk-HKB5FEJG.mjs.map
-//# sourceMappingURL=chunk-HKB5FEJG.mjs.map
+export { BaseRepository, EmbeddingService, HOOK_TIMEOUT_MARKER, ImportExportService, IngestionService, JobManager, MaintenanceService, MetadataRepository, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, RetrievalService, SearchService, WikiBusyError, WikiTransactionError, WriteService, __privateAdd, __privateGet, __privateSet, configureRandomSource, emptyManifest, entitySummaryMetaKey, extractSqliteCode, generateId, normalizeSourceHash, normalizeSourceRef, normalizeTitleKey, parseEmbedding, resolveEdgeDefinition, resolveNodeType, validateInlineEdges };
+//# sourceMappingURL=chunk-UYLYN5N4.mjs.map
+//# sourceMappingURL=chunk-UYLYN5N4.mjs.map