@equationalapplications/core-llm-wiki 4.20.0 → 4.22.0

package/dist/testing.js

@@ -853,6 +853,15 @@ Return ONLY a valid JSON object matching this schema:
   "facts": [{ "title": "string (max 80 chars)", "body": "string (max 800 chars)", "tags": ["string"], "confidence": "certain|inferred|tentative" }]
 }
 Extract verbatim factual content. Do not return markdown, just raw JSON.`;
+var ONTOLOGY_BACKFILL_SYSTEM_PROMPT = `You are a knowledge classification agent. You will receive existing memory facts that currently have no ontology type. For each input fact { "id", "title", "body", "tags" }, assign the best matching okf_type from the ontology manifest and optionally propose edges to related facts by title.
+Return ONLY a valid JSON object matching this schema:
+{
+  "classifications": [
+    { "id": "string (input fact id, copied verbatim)", "okf_type": "string (manifest node type slug)", "edges": [{ "edge_type": "string", "target_title": "string" }] }
+  ]
+}
+If no manifest type fits a fact, omit that fact from "classifications" entirely \u2014 do not guess.
+Do not return markdown, just raw JSON.`;
 
 // src/services/PromptService.ts
 var PromptService = class {
@@ -942,6 +951,22 @@ ${JSON.stringify(recentEvents, null, 2)}
 The following document anchors are provided for contradiction detection only. Do not include them in \`downgraded\`, \`deleted\`, or \`newFacts\`.`
     };
   }
+  buildOntologyBackfillPrompt(facts, runtimeOverride, ontologyContext) {
+    const template = runtimeOverride ?? this.globalOverrides?.ontologyBackfillSystemPrompt ?? ONTOLOGY_BACKFILL_SYSTEM_PROMPT;
+    const hasFacts = /\{\{\s*facts\s*\}\}/.test(template);
+    if (hasFacts || this.hasOntologyPlaceholders(template)) {
+      return {
+        systemPrompt: this.buildSystemPrompt(template, { facts }, ontologyContext),
+        userPrompt: hasFacts ? "Please classify the facts." : `Facts:
+${JSON.stringify(facts, null, 2)}`
+      };
+    }
+    return {
+      systemPrompt: this.appendOntology(template, ontologyContext),
+      userPrompt: `Facts:
+${JSON.stringify(facts, null, 2)}`
+    };
+  }
 };
 
 // src/services/IngestionService.ts
@@ -1123,6 +1148,9 @@ function parseEmbedding(blob, text) {
 // src/services/MaintenanceService.ts
 var FUZZY_THRESHOLD = 0.5;
 var MIN_TOKENS_TO_QUALIFY = 3;
+var ONTOLOGY_BACKFILL_BATCH_SIZE = 25;
+var ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS = 4e4;
+var ONTOLOGY_BACKFILL_RECHECK_MS = 7 * 24 * 60 * 60 * 1e3;
 var MaintenanceService = class {
   constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
     this.db = db;
@@ -1228,6 +1256,14 @@ var MaintenanceService = class {
       this.jobManager.releaseLock("heal", entityId);
     }
   }
+  async runOntologyBackfill(entityId, options) {
+    this.jobManager.acquireLock("ontologyBackfill", entityId);
+    try {
+      return await this.doRunOntologyBackfill(entityId, options);
+    } finally {
+      this.jobManager.releaseLock("ontologyBackfill", entityId);
+    }
+  }
   async runReembed(entityId, opts) {
     const embedFn = this.options.llmProvider.embed;
     if (!embedFn) return { embedded: 0, skipped: 0, failed: 0 };
@@ -1579,6 +1615,121 @@ var MaintenanceService = class {
     }
     this.searchService.evictCache(entityId);
   }
+  /** Core ontology backfill pass (locks handled by {@link runOntologyBackfill}). Package-internal orchestration hook. */
+  async doRunOntologyBackfill(entityId, options) {
+    const batchSize = options?.batchSize ?? ONTOLOGY_BACKFILL_BATCH_SIZE;
+    if (!Number.isInteger(batchSize) || batchSize < 1) {
+      throw new Error("Invalid batchSize: must be an integer >= 1");
+    }
+    const now = Date.now();
+    const recheckCutoff = now - ONTOLOGY_BACKFILL_RECHECK_MS;
+    const zeroed = { scanned: 0, typed: 0, failedValidation: 0, edgesAdded: 0 };
+    const ontologyService = this.ontologyService;
+    if (!ontologyService) {
+      return { ...zeroed, remaining: 0, deferred: 0 };
+    }
+    const { mode } = await ontologyService.getEffectiveState(entityId);
+    if (mode === "off") {
+      const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+      return { ...zeroed, remaining: 0, deferred: counts2.deferred };
+    }
+    const candidates = await this.entryRepo.findUntypedByEntityId(entityId, batchSize, recheckCutoff);
+    if (candidates.length === 0) {
+      const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+      return { ...zeroed, remaining: counts2.eligible, deferred: counts2.deferred };
+    }
+    const ontologyContext = await ontologyService.buildPromptContext(entityId);
+    const toPromptShape = (f) => ({ id: f.id, title: f.title, body: f.body, tags: f.tags });
+    const buildPrompt = (facts) => this.promptService.buildOntologyBackfillPrompt(
+      facts.map(toPromptShape),
+      options?.promptOverride,
+      ontologyContext
+    );
+    const batch = [candidates[0]];
+    let built = buildPrompt(batch);
+    for (let i = 1; i < candidates.length; i++) {
+      const next = buildPrompt([...batch, candidates[i]]);
+      if (next.systemPrompt.length + next.userPrompt.length > ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS) break;
+      batch.push(candidates[i]);
+      built = next;
+    }
+    const { systemPrompt, userPrompt } = built;
+    const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
+    const parsed = parseJsonResponse(responseText);
+    const classifications = Array.isArray(parsed.classifications) ? parsed.classifications : [];
+    let typed = 0;
+    let failedValidation = 0;
+    let edgesAdded = 0;
+    let abortedOntologyOff = false;
+    await this.db.withTransactionAsync(async (tx) => {
+      let { mode: txMode, manifest } = await ontologyService.getEffectiveState(entityId, tx);
+      if (txMode === "off") {
+        abortedOntologyOff = true;
+        return;
+      }
+      if (txMode === "emergent" && parsed.ontology_updates) {
+        manifest = await ontologyService.mergeEmergentUpdates(entityId, parsed.ontology_updates, tx);
+      }
+      const titleRows = await this.entryRepo.findTitleIndexByEntityId(entityId, tx);
+      const titleIndex = /* @__PURE__ */ new Map();
+      for (const row of titleRows) {
+        titleIndex.set(normalizeTitleKey(row.title), { id: row.id, okf_type: row.okf_type });
+      }
+      const batchById = new Map(batch.map((f) => [f.id, f]));
+      const applied = /* @__PURE__ */ new Set();
+      const pendingEdges = [];
+      for (const classification of classifications) {
+        const fact = typeof classification.id === "string" ? batchById.get(classification.id) : void 0;
+        if (!fact || applied.has(fact.id)) {
+          failedValidation++;
+          continue;
+        }
+        applied.add(fact.id);
+        const normalized = ontologyService.validateAndNormalizeFact(
+          {
+            title: fact.title,
+            body: fact.body,
+            tags: fact.tags,
+            confidence: fact.confidence,
+            okf_type: classification.okf_type,
+            edges: classification.edges
+          },
+          manifest
+        );
+        if (!normalized.okf_type) {
+          failedValidation++;
+          continue;
+        }
+        const result = await this.entryRepo.updateOkfType(fact.id, entityId, normalized.okf_type, tx);
+        if (result.changes === 0) continue;
+        typed++;
+        titleIndex.set(normalizeTitleKey(fact.title), { id: fact.id, okf_type: normalized.okf_type });
+        if (normalized.edges.length > 0) {
+          pendingEdges.push({ sourceId: fact.id, sourceType: normalized.okf_type, edges: normalized.edges });
+        }
+      }
+      for (const item of pendingEdges) {
+        edgesAdded += await ontologyService.resolveAndPersistEdges(
+          entityId,
+          item.sourceId,
+          item.sourceType,
+          item.edges,
+          manifest,
+          titleIndex,
+          tx,
+          now
+        );
+      }
+      await this.entryRepo.markOntologyChecked(batch.map((f) => f.id), entityId, now, tx);
+    });
+    if (abortedOntologyOff) {
+      const counts2 = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+      return { ...zeroed, remaining: 0, deferred: counts2.deferred };
+    }
+    this.searchService.evictCache(entityId);
+    const counts = await this.entryRepo.countUntypedByEntityId(entityId, recheckCutoff);
+    return { scanned: batch.length, typed, failedValidation, edgesAdded, remaining: counts.eligible, deferred: counts.deferred };
+  }
   _validatePruneDuration(value, name) {
     if (value !== null && value !== void 0 && (typeof value !== "number" || !isFinite(value) || value < 0)) {
       throw new Error(`Invalid ${name}: must be a non-negative finite number or null`);
@@ -2360,7 +2511,8 @@ var JobManager = class {
       heal: (id) => this._healKey(id),
       reembed: (id) => this._reembedKey(id),
       import: (id) => this._importKey(id),
-      forget: (id) => this._forgetKey(id)
+      forget: (id) => this._forgetKey(id),
+      ontologyBackfill: (id) => this._ontologyBackfillKey(id)
     };
   }
   _pruneKey(entityId) {
@@ -2387,6 +2539,9 @@ var JobManager = class {
   _healKey(entityId) {
     return `${this.prefix}:${entityId}:heal`;
   }
+  _ontologyBackfillKey(entityId) {
+    return `${this.prefix}:${entityId}:ontologyBackfill`;
+  }
   _isReembedActive(entityId) {
     return this.activeMaintenanceJobs.has(this._reembedKey(entityId)) || this.activeMaintenanceJobs.has(this._globalReembedKey());
   }
@@ -2437,14 +2592,16 @@ var JobManager = class {
         if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
         else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
         else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
+        else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
         else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
         else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
         else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
         else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
         break;
       case "librarian":
-      case "heal": {
-        const opKey = operation === "librarian" ? this._librarianKey(entityId) : this._healKey(entityId);
+      case "heal":
+      case "ontologyBackfill": {
+        const opKey = this.lockKeyFns[operation](entityId);
         if (this.activeMaintenanceJobs.has(opKey)) blockingOperation = operation;
         else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
         else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
@@ -2458,6 +2615,7 @@ var JobManager = class {
         else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
         else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
         else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
+        else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
         else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
         else if (this._isImportActiveFor(entityId)) blockingOperation = "import";
         else if (this._isForgetActiveFor(entityId)) blockingOperation = "forget";
@@ -2468,6 +2626,7 @@ var JobManager = class {
         else if (this._isAnyMaintenanceActiveWithSuffix(":prune")) blockingOperation = "prune";
         else if (this._isAnyMaintenanceActiveWithSuffix(":librarian")) blockingOperation = "librarian";
         else if (this._isAnyMaintenanceActiveWithSuffix(":heal")) blockingOperation = "heal";
+        else if (this._isAnyMaintenanceActiveWithSuffix(":ontologyBackfill")) blockingOperation = "ontologyBackfill";
         else if (this.activeIngestJobs.size > 0) blockingOperation = "ingest";
         else if (this._isAnyMaintenanceActiveWithSuffix(":import")) blockingOperation = "import";
         else if (this._isAnyMaintenanceActiveWithSuffix(":forget")) blockingOperation = "forget";
@@ -2478,6 +2637,7 @@ var JobManager = class {
         if (this.activeMaintenanceJobs.has(selfKey)) blockingOperation = operation;
         else if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) blockingOperation = "librarian";
         else if (this.activeMaintenanceJobs.has(this._healKey(entityId))) blockingOperation = "heal";
+        else if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) blockingOperation = "ontologyBackfill";
         else if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) blockingOperation = "prune";
         else if (this._isReembedActive(entityId)) blockingOperation = "reembed";
         else if (this._isIngestActiveFor(entityId)) blockingOperation = "ingest";
@@ -2535,11 +2695,11 @@ var JobManager = class {
     if (operation !== "global_import" && this.activeMaintenanceJobs.has(this._globalImportKey())) return true;
     switch (operation) {
       case "librarian":
-        return this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
       case "heal":
-        return this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
+      case "ontologyBackfill":
+        return this.activeMaintenanceJobs.has(this.lockKeyFns[operation](entityId)) || this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this._isReembedActive(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
       case "prune":
-        return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
+        return this.activeMaintenanceJobs.has(this._pruneKey(entityId)) || this.activeMaintenanceJobs.has(this._librarianKey(entityId)) || this.activeMaintenanceJobs.has(this._healKey(entityId)) || this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId)) || this._isReembedActive(entityId) || this._isIngestActiveFor(entityId) || this._isImportActiveFor(entityId) || this._isForgetActiveFor(entityId);
       default:
         return false;
     }
@@ -2564,6 +2724,7 @@ var JobManager = class {
       if (this.activeMaintenanceJobs.has(this._importKey(entityId))) throw new WikiBusyError("import", entityId);
       if (this.activeMaintenanceJobs.has(this._librarianKey(entityId))) throw new WikiBusyError("librarian", entityId);
       if (this.activeMaintenanceJobs.has(this._healKey(entityId))) throw new WikiBusyError("heal", entityId);
+      if (this.activeMaintenanceJobs.has(this._ontologyBackfillKey(entityId))) throw new WikiBusyError("ontologyBackfill", entityId);
       if (this.activeMaintenanceJobs.has(this._pruneKey(entityId))) throw new WikiBusyError("prune", entityId);
       if (this._isReembedActive(entityId)) throw new WikiBusyError("reembed", entityId);
       if (this._isIngestActiveFor(entityId)) throw new WikiBusyError("ingest", entityId);