@equationalapplications/core-llm-wiki 4.20.0 → 4.22.0

package/dist/index.mjs

@@ -1,5 +1,5 @@
-import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, generateId, BaseRepository, emptyManifest, resolveNodeType, validateInlineEdges, resolveEdgeDefinition, normalizeTitleKey, WikiTransactionError, extractSqliteCode } from './chunk-HKB5FEJG.mjs';
-export { HOOK_TIMEOUT_MARKER, PromptService, PrunePartialFailureError, WikiBusyError, WikiTransactionError, configureRandomSource, parseEmbedding } from './chunk-HKB5FEJG.mjs';
+import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, generateId, BaseRepository, emptyManifest, resolveNodeType, validateInlineEdges, resolveEdgeDefinitions, normalizeTitleKey, WikiTransactionError, extractSqliteCode } from './chunk-LCAU6NYC.mjs';
+export { HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiTransactionError, configureRandomSource, parseEmbedding, validateManifest } from './chunk-LCAU6NYC.mjs';
 import { appendRelatedSection, buildConceptDocument, buildLogMd, buildEntityIndexMd, buildRootIndexMd, isAllowedOkfPath, parseRootIndexMd, parseEntityIndexMd, parseConcept, splitRelatedSection, extractMarkdownLinks, parseLogMd, parseEventIdComment, appendEventIdComment } from '@equationalapplications/core-okf';
 
 // src/db/schema.ts
@@ -22,7 +22,8 @@ async function setupDatabase(db, prefix) {
       deleted_at INTEGER,
       embedding TEXT,
       embedding_blob BLOB,
-      okf_type TEXT
+      okf_type TEXT,
+      ontology_checked_at INTEGER
     );
 
     CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
@@ -248,6 +249,20 @@ var MIGRATIONS = [
         );
       `);
     }
+  },
+  {
+    version: 7,
+    description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
+    run: async (db, prefix) => {
+      const cols = await db.getAllAsync(
+        `PRAGMA table_info(${prefix}entries)`
+      );
+      if (!cols.some((c) => c.name === "ontology_checked_at")) {
+        await db.execAsync(
+          `ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
+        );
+      }
+    }
   }
 ];
 for (let i = 1; i < MIGRATIONS.length; i++) {
@@ -996,6 +1011,79 @@ var EntryRepository = class extends BaseRepository {
     );
     return rows.map(mapRowToFact);
   }
+  /**
+   * Live untyped facts eligible for ontology backfill, oldest first.
+   * Skips facts checked within the recheck cooldown (ontology_checked_at > recheckCutoff).
+   */
+  async findUntypedByEntityId(entityId, limit, recheckCutoff, tx) {
+    const executor = this.getExecutor(tx);
+    const rows = await executor.getAllAsync(
+      `SELECT * FROM ${this.prefix}entries
+       WHERE entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL
+         AND (ontology_checked_at IS NULL OR ontology_checked_at <= ?)
+       ORDER BY updated_at ASC LIMIT ?`,
+      [entityId, recheckCutoff, limit]
+    );
+    return rows.map(mapRowToFact);
+  }
+  /** Counts live untyped facts: eligible (past cooldown) vs deferred (in cooldown). */
+  async countUntypedByEntityId(entityId, recheckCutoff, tx) {
+    const executor = this.getExecutor(tx);
+    const row = await executor.getFirstAsync(
+      `SELECT
+         SUM(CASE WHEN ontology_checked_at IS NULL OR ontology_checked_at <= ? THEN 1 ELSE 0 END) AS eligible,
+         SUM(CASE WHEN ontology_checked_at IS NOT NULL AND ontology_checked_at > ? THEN 1 ELSE 0 END) AS deferred
+       FROM ${this.prefix}entries
+       WHERE entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL`,
+      [recheckCutoff, recheckCutoff, entityId]
+    );
+    return { eligible: Number(row?.eligible ?? 0), deferred: Number(row?.deferred ?? 0) };
+  }
+  /**
+   * Sets okf_type only when currently NULL — additive by construction: a
+   * concurrently-typed fact is never overwritten. Returns changes for the caller
+   * to distinguish applied vs skipped.
+   */
+  async updateOkfType(id, entityId, okfType, tx) {
+    const result = await tx.runAsync(
+      `UPDATE ${this.prefix}entries SET okf_type = ?
+       WHERE id = ? AND entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL`,
+      [okfType, id, entityId]
+    );
+    return { changes: result.changes };
+  }
+  /**
+   * Stamps the backfill recheck cooldown. NEVER touches updated_at — import
+   * merge resolution is last-write-wins on updated_at and a bump here would
+   * make an unchanged local fact beat a genuinely newer remote edit.
+   */
+  async markOntologyChecked(ids, entityId, now, tx) {
+    if (ids.length === 0) return;
+    for (let i = 0; i < ids.length; i += this.chunkSize) {
+      const chunk = ids.slice(i, i + this.chunkSize);
+      const placeholders = chunk.map(() => "?").join(",");
+      await tx.runAsync(
+        `UPDATE ${this.prefix}entries SET ontology_checked_at = ?
+         WHERE id IN (${placeholders}) AND entity_id = ? AND deleted_at IS NULL`,
+        [now, ...chunk, entityId]
+      );
+    }
+  }
+  /**
+   * Lightweight full-breadth title index (id, title, okf_type) over all live
+   * typed facts. Used by ontology backfill edge resolution — a recent-N window
+   * would miss an old fact's contemporaries. Untyped rows are excluded: they
+   * can never pass the edge target-type check, and batch facts typed during
+   * the run are re-added to the in-memory index by the caller.
+   */
+  async findTitleIndexByEntityId(entityId, tx) {
+    const executor = this.getExecutor(tx);
+    const rows = await executor.getAllAsync(
+      `SELECT id, title, okf_type FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL AND okf_type IS NOT NULL`,
+      [entityId]
+    );
+    return rows.map((r) => ({ id: r.id, title: r.title, okf_type: r.okf_type ?? null }));
+  }
 };
 
 // src/repositories/OutboxRepository.ts
@@ -1416,6 +1504,7 @@ var CONFIDENCE_RANK = {
 var EdgeRepository = class extends BaseRepository {
   /**
    * Insert an edge, silently skipping on primary-key or uniqueness conflicts.
+   * Returns true when a row was inserted, false when skipped as a duplicate.
    * Throws when the insert was skipped due to an id collision with a different edge tuple.
    */
   async addIgnoreDuplicate(edge, tx) {
@@ -1425,17 +1514,18 @@ var EdgeRepository = class extends BaseRepository {
        VALUES (?, ?, ?, ?, ?, ?)`,
       [edge.id, edge.entity_id, edge.source_id, edge.target_id, edge.edge_type, edge.created_at]
     );
-    if (result.changes > 0) return;
+    if (result.changes > 0) return true;
     const existing = await executor.getFirstAsync(
       `SELECT entity_id, source_id, target_id, edge_type FROM ${this.prefix}edges WHERE id = ?`,
       [edge.id]
     );
-    if (!existing) return;
+    if (!existing) return false;
     if (String(existing.entity_id) !== edge.entity_id || String(existing.source_id) !== edge.source_id || String(existing.target_id) !== edge.target_id || String(existing.edge_type) !== edge.edge_type) {
       throw new Error(
         `Edge id collision: ${JSON.stringify(edge.id)} already exists with a different (entity_id, source_id, target_id, edge_type) tuple`
       );
     }
+    return false;
   }
   async getByEntityId(entityId, tx) {
     const executor = this.getExecutor(tx);
@@ -1553,7 +1643,8 @@ function mapRowToEdge(row) {
 var FACT_ONTOLOGY_FIELDS = `
 Each fact may optionally include:
 - "okf_type": string \u2014 must be one of the node_types in the manifest
-- "edges": [{ "edge_type": string, "target_title": string }] \u2014 target_title must match another fact's title in this response or existing memory`;
+- "edges": [{ "edge_type": string, "target_title": string }] \u2014 target_title must match another fact's title in this response or existing memory
+- An edge_type may appear in the manifest multiple times with different source_type/target_type; use the row whose types match your fact and target.`;
 var EMERGENT_EXTRA = `
 You may also return "ontology_updates" to propose new types:
 "ontology_updates": {
@@ -1635,14 +1726,18 @@ var OntologyService = class {
     return { okf_type: canonical, edges };
   }
   async resolveAndPersistEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, tx, now) {
-    if (!sourceType || edges.length === 0) return;
+    if (!sourceType || edges.length === 0) return 0;
+    let persisted = 0;
     for (const edge of edges) {
-      const def = resolveEdgeDefinition(edge.edge_type, manifest);
-      if (!def || def.source_type.toLowerCase() !== sourceType.toLowerCase()) continue;
+      const candidates = resolveEdgeDefinitions(edge.edge_type, manifest).filter((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
+      if (candidates.length === 0) continue;
       const targetKey = normalizeTitleKey(edge.target_title);
       const target = titleIndex.get(targetKey);
       if (!target) continue;
-      if (def.target_type.toLowerCase() !== (target.okf_type ?? "").toLowerCase()) continue;
+      const def = candidates.find(
+        (d) => d.target_type.toLowerCase() === (target.okf_type ?? "").toLowerCase()
+      );
+      if (!def) continue;
       const wikiEdge = {
         id: generateId(),
         entity_id: entityId,
@@ -1651,8 +1746,10 @@ var OntologyService = class {
         edge_type: def.type,
         created_at: now
       };
-      await this.edgeRepo.addIgnoreDuplicate(wikiEdge, tx);
+      const inserted = await this.edgeRepo.addIgnoreDuplicate(wikiEdge, tx);
+      if (inserted) persisted++;
     }
+    return persisted;
   }
 };
 
@@ -1892,6 +1989,24 @@ var WikiMemory = class {
   async runHeal(entityId, options) {
     return this.maintenanceService.runHeal(entityId, options);
   }
+  /**
+   * Types already-persisted untyped facts (okf_type IS NULL) in place via one
+   * librarian-style LLM call. Strictly additive: never creates, deletes, or
+   * rewrites facts; never overwrites an existing okf_type. Free when there is
+   * nothing to do (ontology off, or zero eligible untyped facts).
+   *
+   * Hosts own the trigger cadence (e.g. after each sync); loop `while
+   * result.remaining > 0` for convergence. Throws WikiBusyError when another
+   * backfill (or conflicting maintenance op) is running for the entity.
+   *
+   * @param options.promptOverride - Applies only to this call. For persistent
+   * customization set `options.config.prompts.ontologyBackfillSystemPrompt`.
+   * @param options.batchSize - Facts per run (default 25) for providers with
+   * tighter context limits.
+   */
+  async runOntologyBackfill(entityId, options) {
+    return this.maintenanceService.runOntologyBackfill(entityId, options);
+  }
   async runReembed(entityId, opts) {
     return this.maintenanceService.runReembed(entityId, opts);
   }