@equationalapplications/core-llm-wiki 4.15.3 → 4.16.0

package/dist/testing.js

@@ -796,6 +796,11 @@ After running the migration SQL, restart your application.`
   }
 };
 
+// src/utils/ontology.ts
+function normalizeTitleKey(title) {
+  return title.trim().toLowerCase().replace(/\s+/g, " ");
+}
+
 // src/utils/ids.ts
 function generateId(prefix = "") {
   if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
@@ -846,30 +851,54 @@ var PromptService = class {
       return typeof value === "string" ? value : JSON.stringify(value, null, 2);
     });
   }
-  buildIngestPrompt(documentChunk, runtimeOverride) {
+  hasOntologyPlaceholders(template) {
+    return /\{\{\s*ontology(?:Manifest|ModeInstructions)\s*\}\}/.test(template);
+  }
+  buildSystemPrompt(template, variables, ontologyContext) {
+    const shouldHydrate = Object.keys(variables).some(
+      (key) => new RegExp(`\\{\\{\\s*${key}\\s*\\}\\}`).test(template)
+    ) || ontologyContext != null && this.hasOntologyPlaceholders(template);
+    const hydrated = shouldHydrate ? this.hydrate(template, { ...variables, ...ontologyContext ?? {} }) : template;
+    return this.hasOntologyPlaceholders(template) ? ontologyContext != null ? hydrated : hydrated.replace(/\{\{\s*ontology(?:Manifest|ModeInstructions)\s*\}\}/g, "") : this.appendOntology(hydrated, ontologyContext);
+  }
+  appendOntology(systemPrompt, ctx) {
+    if (!ctx) return systemPrompt;
+    return `${systemPrompt}
+
+${ctx.ontologyModeInstructions}`;
+  }
+  buildIngestPrompt(documentChunk, runtimeOverride, ontologyContext) {
     const template = runtimeOverride ?? this.globalOverrides?.ingestSystemPrompt ?? INGEST_SYSTEM_PROMPT;
-    if (/\{\{\s*documentChunk\s*\}\}/.test(template)) {
+    const hasDocumentChunk = /\{\{\s*documentChunk\s*\}\}/.test(template);
+    if (hasDocumentChunk || this.hasOntologyPlaceholders(template)) {
       return {
-        systemPrompt: this.hydrate(template, { documentChunk }),
-        userPrompt: "Please extract the facts."
+        systemPrompt: this.buildSystemPrompt(template, { documentChunk }, ontologyContext),
+        userPrompt: hasDocumentChunk ? "Please extract the facts." : `Document Chunk:
+${documentChunk}`
       };
     }
     return {
-      systemPrompt: template,
+      systemPrompt: this.appendOntology(template, ontologyContext),
       userPrompt: `Document Chunk:
 ${documentChunk}`
     };
   }
-  buildLibrarianPrompt(events, currentFacts, runtimeOverride) {
+  buildLibrarianPrompt(events, currentFacts, runtimeOverride, ontologyContext) {
     const template = runtimeOverride ?? this.globalOverrides?.librarianSystemPrompt ?? LIBRARIAN_SYSTEM_PROMPT;
-    if (/\{\{\s*events\s*\}\}/.test(template) || /\{\{\s*currentFacts\s*\}\}/.test(template)) {
+    const hasEvents = /\{\{\s*events\s*\}\}/.test(template);
+    const hasCurrentFacts = /\{\{\s*currentFacts\s*\}\}/.test(template);
+    if (hasEvents || hasCurrentFacts || this.hasOntologyPlaceholders(template)) {
       return {
-        systemPrompt: this.hydrate(template, { events, currentFacts }),
-        userPrompt: "Please synthesize the context."
+        systemPrompt: this.buildSystemPrompt(template, { events, currentFacts }, ontologyContext),
+        userPrompt: hasEvents || hasCurrentFacts ? "Please synthesize the context." : `Events:
+${JSON.stringify(events, null, 2)}
+
+Current Facts:
+${JSON.stringify(currentFacts, null, 2)}`
       };
     }
     return {
-      systemPrompt: template,
+      systemPrompt: this.appendOntology(template, ontologyContext),
       userPrompt: `Events:
 ${JSON.stringify(events, null, 2)}
 
@@ -902,7 +931,7 @@ The following document anchors are provided for contradiction detection only. Do
 
 // src/services/IngestionService.ts
 var IngestionService = class {
-  constructor(db, prefix, options, entryRepo, searchService, jobManager, embeddingService, promptService) {
+  constructor(db, prefix, options, entryRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
     this.db = db;
     this.prefix = prefix;
     this.options = options;
@@ -910,6 +939,7 @@ var IngestionService = class {
     this.searchService = searchService;
     this.jobManager = jobManager;
     this.embeddingService = embeddingService;
+    this.ontologyService = ontologyService;
     this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
   }
   async ingestDocument(entityId, params) {
@@ -934,23 +964,33 @@ var IngestionService = class {
       if (chunks.length === 0) return { truncated: false, chunks: 0 };
       const chunkResults = await withConcurrency(
         chunks.map((chunk) => async () => {
-          const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(chunk, params.promptOverride);
+          const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
+          const { systemPrompt, userPrompt } = this.promptService.buildIngestPrompt(
+            chunk,
+            params.promptOverride,
+            ontologyContext
+          );
           const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
           const result = parseJsonResponse(responseText);
-          return (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null);
+          return {
+            facts: (Array.isArray(result.facts) ? result.facts : []).map(validateFact).filter((f) => f !== null),
+            ontology_updates: result.ontology_updates
+          };
         }),
         chunkConcurrency
       );
       const seen = /* @__PURE__ */ new Set();
-      const allValidFacts = [];
-      for (const facts of chunkResults) {
-        for (const fact of facts) {
-          const normalized = fact.title.trim().toLowerCase().replace(/\s+/g, " ");
-          if (!seen.has(normalized)) {
-            seen.add(normalized);
-            allValidFacts.push(fact);
+      const orderedChunkFacts = [];
+      for (const chunkResult of chunkResults) {
+        const dedupedFacts = [];
+        for (const fact of chunkResult.facts) {
+          const normalizedTitle = normalizeTitleKey(fact.title);
+          if (!seen.has(normalizedTitle)) {
+            seen.add(normalizedTitle);
+            dedupedFacts.push(fact);
           }
         }
+        orderedChunkFacts.push({ facts: dedupedFacts, ontology_updates: chunkResult.ontology_updates });
       }
       const now = Date.now();
       const insertedFacts = [];
@@ -958,26 +998,63 @@ var IngestionService = class {
       await this.db.withTransactionAsync(async (tx) => {
         deletedSourceFactIds.push(...await this.entryRepo.findIdsBySource(entityId, sourceRef, null, tx, false));
         await this.entryRepo.softDeleteBySource(entityId, tx, sourceRef, null);
-        for (const fact of allValidFacts) {
-          const id = generateId("fact_");
-          const wikiFact = {
-            id,
-            entity_id: entityId,
-            title: fact.title,
-            body: fact.body,
-            tags: fact.tags,
-            confidence: fact.confidence,
-            source_type: "immutable_document",
-            source_hash: sourceHash,
-            source_ref: sourceRef,
-            created_at: now,
-            updated_at: now,
-            last_accessed_at: null,
-            access_count: 0,
-            deleted_at: null
-          };
-          await this.entryRepo.upsert(wikiFact, tx);
-          insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
+        const titleIndex = /* @__PURE__ */ new Map();
+        const pendingEdges = [];
+        const existingFacts = await this.entryRepo.findRecentByEntityId(entityId, 500, tx);
+        for (const existing of existingFacts) {
+          titleIndex.set(normalizeTitleKey(existing.title), {
+            id: existing.id,
+            okf_type: existing.okf_type ?? null
+          });
+        }
+        let ontologyState = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
+        let { mode, manifest } = ontologyState;
+        for (const { facts, ontology_updates } of orderedChunkFacts) {
+          if (mode === "emergent" && ontology_updates && this.ontologyService) {
+            manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontology_updates, tx);
+            ontologyState = await this.ontologyService.getEffectiveState(entityId, tx);
+            mode = ontologyState.mode;
+          }
+          for (const fact of facts) {
+            const ontologyFact = fact;
+            const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest) ?? { okf_type: null, edges: [] };
+            const id = generateId("fact_");
+            const wikiFact = {
+              id,
+              entity_id: entityId,
+              title: fact.title,
+              body: fact.body,
+              tags: fact.tags,
+              confidence: fact.confidence,
+              source_type: "immutable_document",
+              source_hash: sourceHash,
+              source_ref: sourceRef,
+              created_at: now,
+              updated_at: now,
+              last_accessed_at: null,
+              access_count: 0,
+              deleted_at: null,
+              okf_type: normalized.okf_type
+            };
+            await this.entryRepo.upsert(wikiFact, tx);
+            insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
+            titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
+            if (normalized.edges.length > 0) {
+              pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
+            }
+          }
+        }
+        for (const item of pendingEdges) {
+          await this.ontologyService?.resolveAndPersistEdges(
+            entityId,
+            item.sourceId,
+            item.sourceType,
+            item.edges ?? [],
+            manifest,
+            titleIndex,
+            tx,
+            now
+          );
         }
       });
       await this.searchService.sync(entityId);
@@ -1032,7 +1109,7 @@ function parseEmbedding(blob, text) {
 var FUZZY_THRESHOLD = 0.5;
 var MIN_TOKENS_TO_QUALIFY = 3;
 var MaintenanceService = class {
-  constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService) {
+  constructor(db, prefix, options, entryRepo, taskRepo, eventRepo, metadataRepo, searchService, jobManager, embeddingService, promptService, ontologyService) {
     this.db = db;
     this.prefix = prefix;
     this.options = options;
@@ -1043,6 +1120,7 @@ var MaintenanceService = class {
     this.searchService = searchService;
     this.jobManager = jobManager;
     this.embeddingService = embeddingService;
+    this.ontologyService = ontologyService;
     this.promptService = promptService ?? new PromptService(this.options.config?.prompts);
   }
   async runPrune(entityId, options) {
@@ -1265,21 +1343,36 @@ var MaintenanceService = class {
         tags: typeof rest.tags === "string" ? JSON.parse(rest.tags) : rest.tags
       };
     });
+    const ontologyContext = await this.ontologyService?.buildPromptContext(entityId) ?? null;
     const { systemPrompt, userPrompt } = this.promptService.buildLibrarianPrompt(
       events.reverse(),
       currentFacts,
-      promptOverride
+      promptOverride,
+      ontologyContext
     );
     const responseText = await this.options.llmProvider.generateText({ systemPrompt, userPrompt });
     const result = parseJsonResponse(responseText);
     const facts = Array.isArray(result.facts) ? result.facts : [];
     const tasks = Array.isArray(result.tasks) ? result.tasks : [];
+    const ontologyUpdates = result.ontology_updates;
     const validFacts = facts.map(validateFact).filter((f) => f !== null);
     const validTasks = tasks.map(validateTask).filter((t) => t !== null);
     const now = Date.now();
     const insertedFacts = [];
     await this.db.withTransactionAsync(async (tx) => {
+      let { mode, manifest } = await this.ontologyService?.getEffectiveState(entityId, tx) ?? { mode: "off", manifest: { node_types: [], edge_types: [] } };
+      if (mode === "emergent" && ontologyUpdates && this.ontologyService) {
+        manifest = await this.ontologyService.mergeEmergentUpdates(entityId, ontologyUpdates, tx);
+      }
+      const titleIndex = /* @__PURE__ */ new Map();
+      for (const existing of currentFactsRows) {
+        titleIndex.set(normalizeTitleKey(existing.title), {
+          id: existing.id,
+          okf_type: existing.okf_type ?? null
+        });
+      }
       const factsForDedupe = await this.entryRepo.findRecentByEntityId(entityId, 100, tx);
+      const pendingEdges = [];
       for (const fact of validFacts) {
         const newTokens = titleTokens(fact.title);
         let skip = false;
@@ -1296,6 +1389,8 @@ var MaintenanceService = class {
           }
         }
         if (skip) continue;
+        const ontologyFact = fact;
+        const normalized = this.ontologyService?.validateAndNormalizeFact(ontologyFact, manifest) ?? { okf_type: null, edges: [] };
         const id = generateId("fact_");
         const factObj = {
           id,
@@ -1311,11 +1406,28 @@ var MaintenanceService = class {
           updated_at: now,
           last_accessed_at: null,
           access_count: 0,
-          deleted_at: null
+          deleted_at: null,
+          okf_type: normalized.okf_type
         };
         await this.entryRepo.upsert(factObj, tx);
         insertedFacts.push({ id, entity_id: entityId, title: fact.title, body: fact.body, tags: JSON.stringify(fact.tags) });
         factsForDedupe.push(factObj);
+        titleIndex.set(normalizeTitleKey(fact.title), { id, okf_type: normalized.okf_type });
+        if (normalized.edges.length > 0) {
+          pendingEdges.push({ sourceId: id, sourceType: normalized.okf_type, edges: normalized.edges });
+        }
+      }
+      for (const item of pendingEdges) {
+        await this.ontologyService?.resolveAndPersistEdges(
+          entityId,
+          item.sourceId,
+          item.sourceType,
+          item.edges ?? [],
+          manifest,
+          titleIndex,
+          tx,
+          now
+        );
       }
       for (const task of validTasks) {
         const id = generateId("task_");