@equationalapplications/core-llm-wiki 4.19.0 → 4.21.0

package/dist/index.mjs

@@ -1,5 +1,5 @@
-import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, generateId, BaseRepository, emptyManifest, resolveNodeType, validateInlineEdges, resolveEdgeDefinition, normalizeTitleKey } from './chunk-3GMXRLKU.mjs';
-export { HOOK_TIMEOUT_MARKER, PromptService, PrunePartialFailureError, WikiBusyError, configureRandomSource, parseEmbedding } from './chunk-3GMXRLKU.mjs';
+import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, generateId, BaseRepository, emptyManifest, resolveNodeType, validateInlineEdges, resolveEdgeDefinition, normalizeTitleKey, WikiTransactionError, extractSqliteCode } from './chunk-UYLYN5N4.mjs';
+export { HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiTransactionError, configureRandomSource, parseEmbedding } from './chunk-UYLYN5N4.mjs';
 import { appendRelatedSection, buildConceptDocument, buildLogMd, buildEntityIndexMd, buildRootIndexMd, isAllowedOkfPath, parseRootIndexMd, parseEntityIndexMd, parseConcept, splitRelatedSection, extractMarkdownLinks, parseLogMd, parseEventIdComment, appendEventIdComment } from '@equationalapplications/core-okf';
 
 // src/db/schema.ts
@@ -22,7 +22,8 @@ async function setupDatabase(db, prefix) {
       deleted_at INTEGER,
       embedding TEXT,
       embedding_blob BLOB,
-      okf_type TEXT
+      okf_type TEXT,
+      ontology_checked_at INTEGER
     );
 
     CREATE INDEX IF NOT EXISTS ${prefix}entries_entity_idx ON ${prefix}entries(entity_id);
@@ -104,6 +105,50 @@ async function setupDatabase(db, prefix) {
   `);
 }
 
+// src/db/serializedAdapter.ts
+function isDriverError(err) {
+  return extractSqliteCode(err) !== void 0;
+}
+function withOverrides(target, overrides) {
+  return new Proxy(target, {
+    get(obj, prop, receiver) {
+      if (prop in overrides) return Reflect.get(overrides, prop, receiver);
+      const value = Reflect.get(obj, prop, obj);
+      return typeof value === "function" ? value.bind(obj) : value;
+    }
+  });
+}
+function guardReentrancy(tx) {
+  return withOverrides(tx, {
+    withTransactionAsync() {
+      throw new Error(
+        "Nested withTransactionAsync is not supported: you are already inside a transaction. Pass the current `tx` down instead of opening a new transaction."
+      );
+    }
+  });
+}
+var DEADLOCK_WARN_MS = 1e4;
+function withSerializedTransactions(db) {
+  let queue = Promise.resolve();
+  return withOverrides(db, {
+    withTransactionAsync(fn) {
+      const warn = setTimeout(() => {
+        console.warn(
+          "[core-llm-wiki] Transaction queued >10s \u2014 possible deadlock. Inside a transaction callback, use the `tx` parameter, never the outer database handle."
+        );
+      }, DEADLOCK_WARN_MS);
+      const run = queue.catch(() => void 0).then(() => {
+        clearTimeout(warn);
+        return db.withTransactionAsync((tx) => fn(guardReentrancy(tx ?? db)));
+      }).catch((e) => {
+        throw isDriverError(e) ? new WikiTransactionError("Transaction failed", { cause: e }) : e;
+      });
+      queue = run;
+      return run;
+    }
+  });
+}
+
 // src/db/migrations.ts
 var MIGRATIONS = [
   {
@@ -204,6 +249,20 @@ var MIGRATIONS = [
         );
       `);
     }
+  },
+  {
+    version: 7,
+    description: "Add ontology_checked_at to entries for ontology backfill recheck cooldown",
+    run: async (db, prefix) => {
+      const cols = await db.getAllAsync(
+        `PRAGMA table_info(${prefix}entries)`
+      );
+      if (!cols.some((c) => c.name === "ontology_checked_at")) {
+        await db.execAsync(
+          `ALTER TABLE ${prefix}entries ADD COLUMN ontology_checked_at INTEGER`
+        );
+      }
+    }
   }
 ];
 for (let i = 1; i < MIGRATIONS.length; i++) {
@@ -952,6 +1011,79 @@ var EntryRepository = class extends BaseRepository {
     );
     return rows.map(mapRowToFact);
   }
+  /**
+   * Live untyped facts eligible for ontology backfill, oldest first.
+   * Skips facts checked within the recheck cooldown (ontology_checked_at > recheckCutoff).
+   */
+  async findUntypedByEntityId(entityId, limit, recheckCutoff, tx) {
+    const executor = this.getExecutor(tx);
+    const rows = await executor.getAllAsync(
+      `SELECT * FROM ${this.prefix}entries
+       WHERE entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL
+         AND (ontology_checked_at IS NULL OR ontology_checked_at <= ?)
+       ORDER BY updated_at ASC LIMIT ?`,
+      [entityId, recheckCutoff, limit]
+    );
+    return rows.map(mapRowToFact);
+  }
+  /** Counts live untyped facts: eligible (past cooldown) vs deferred (in cooldown). */
+  async countUntypedByEntityId(entityId, recheckCutoff, tx) {
+    const executor = this.getExecutor(tx);
+    const row = await executor.getFirstAsync(
+      `SELECT
+         SUM(CASE WHEN ontology_checked_at IS NULL OR ontology_checked_at <= ? THEN 1 ELSE 0 END) AS eligible,
+         SUM(CASE WHEN ontology_checked_at IS NOT NULL AND ontology_checked_at > ? THEN 1 ELSE 0 END) AS deferred
+       FROM ${this.prefix}entries
+       WHERE entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL`,
+      [recheckCutoff, recheckCutoff, entityId]
+    );
+    return { eligible: Number(row?.eligible ?? 0), deferred: Number(row?.deferred ?? 0) };
+  }
+  /**
+   * Sets okf_type only when currently NULL — additive by construction: a
+   * concurrently-typed fact is never overwritten. Returns changes for the caller
+   * to distinguish applied vs skipped.
+   */
+  async updateOkfType(id, entityId, okfType, tx) {
+    const result = await tx.runAsync(
+      `UPDATE ${this.prefix}entries SET okf_type = ?
+       WHERE id = ? AND entity_id = ? AND okf_type IS NULL AND deleted_at IS NULL`,
+      [okfType, id, entityId]
+    );
+    return { changes: result.changes };
+  }
+  /**
+   * Stamps the backfill recheck cooldown. NEVER touches updated_at — import
+   * merge resolution is last-write-wins on updated_at and a bump here would
+   * make an unchanged local fact beat a genuinely newer remote edit.
+   */
+  async markOntologyChecked(ids, entityId, now, tx) {
+    if (ids.length === 0) return;
+    for (let i = 0; i < ids.length; i += this.chunkSize) {
+      const chunk = ids.slice(i, i + this.chunkSize);
+      const placeholders = chunk.map(() => "?").join(",");
+      await tx.runAsync(
+        `UPDATE ${this.prefix}entries SET ontology_checked_at = ?
+         WHERE id IN (${placeholders}) AND entity_id = ? AND deleted_at IS NULL`,
+        [now, ...chunk, entityId]
+      );
+    }
+  }
+  /**
+   * Lightweight full-breadth title index (id, title, okf_type) over all live
+   * typed facts. Used by ontology backfill edge resolution — a recent-N window
+   * would miss an old fact's contemporaries. Untyped rows are excluded: they
+   * can never pass the edge target-type check, and batch facts typed during
+   * the run are re-added to the in-memory index by the caller.
+   */
+  async findTitleIndexByEntityId(entityId, tx) {
+    const executor = this.getExecutor(tx);
+    const rows = await executor.getAllAsync(
+      `SELECT id, title, okf_type FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL AND okf_type IS NOT NULL`,
+      [entityId]
+    );
+    return rows.map((r) => ({ id: r.id, title: r.title, okf_type: r.okf_type ?? null }));
+  }
 };
 
 // src/repositories/OutboxRepository.ts
@@ -1372,6 +1504,7 @@ var CONFIDENCE_RANK = {
 var EdgeRepository = class extends BaseRepository {
   /**
    * Insert an edge, silently skipping on primary-key or uniqueness conflicts.
+   * Returns true when a row was inserted, false when skipped as a duplicate.
    * Throws when the insert was skipped due to an id collision with a different edge tuple.
    */
   async addIgnoreDuplicate(edge, tx) {
@@ -1381,17 +1514,18 @@ var EdgeRepository = class extends BaseRepository {
        VALUES (?, ?, ?, ?, ?, ?)`,
       [edge.id, edge.entity_id, edge.source_id, edge.target_id, edge.edge_type, edge.created_at]
     );
-    if (result.changes > 0) return;
+    if (result.changes > 0) return true;
     const existing = await executor.getFirstAsync(
       `SELECT entity_id, source_id, target_id, edge_type FROM ${this.prefix}edges WHERE id = ?`,
       [edge.id]
     );
-    if (!existing) return;
+    if (!existing) return false;
     if (String(existing.entity_id) !== edge.entity_id || String(existing.source_id) !== edge.source_id || String(existing.target_id) !== edge.target_id || String(existing.edge_type) !== edge.edge_type) {
       throw new Error(
         `Edge id collision: ${JSON.stringify(edge.id)} already exists with a different (entity_id, source_id, target_id, edge_type) tuple`
       );
     }
+    return false;
   }
   async getByEntityId(entityId, tx) {
     const executor = this.getExecutor(tx);
@@ -1591,7 +1725,8 @@ var OntologyService = class {
     return { okf_type: canonical, edges };
   }
   async resolveAndPersistEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, tx, now) {
-    if (!sourceType || edges.length === 0) return;
+    if (!sourceType || edges.length === 0) return 0;
+    let persisted = 0;
     for (const edge of edges) {
       const def = resolveEdgeDefinition(edge.edge_type, manifest);
       if (!def || def.source_type.toLowerCase() !== sourceType.toLowerCase()) continue;
@@ -1607,8 +1742,10 @@ var OntologyService = class {
         edge_type: def.type,
         created_at: now
       };
-      await this.edgeRepo.addIgnoreDuplicate(wikiEdge, tx);
+      const inserted = await this.edgeRepo.addIgnoreDuplicate(wikiEdge, tx);
+      if (inserted) persisted++;
     }
+    return persisted;
   }
 };
 
@@ -1651,7 +1788,7 @@ var WikiMemory = class {
   constructor(db, options) {
     /** Emits `__testAccess` console warning at most once per instance when NODE_ENV ≠ "test". */
     __privateAdd(this, _testAccessNonTestEnvWarned, false);
-    this.db = db;
+    this.db = withSerializedTransactions(db);
     this.options = options;
     this.prefix = options.config?.tablePrefix ?? "llm_wiki_";
     if (!TABLE_PREFIX_PATTERN.test(this.prefix)) {
@@ -1659,12 +1796,12 @@ var WikiMemory = class {
         `Invalid tablePrefix: ${JSON.stringify(this.prefix)}. Must match ${TABLE_PREFIX_PATTERN} (letter, then alphanumeric/underscore, ending in "_", max 32 chars total).`
       );
     }
-    this.outboxRepo = new OutboxRepository(db, this.prefix, !!options.config?.enableOutbox);
-    this.entryRepo = new EntryRepository(db, this.prefix, this.outboxRepo);
-    this.taskRepo = new TaskRepository(db, this.prefix, this.outboxRepo);
-    this.eventRepo = new EventRepository(db, this.prefix);
-    this.edgeRepo = new EdgeRepository(db, this.prefix);
-    this.metadataRepo = new MetadataRepository(db, this.prefix);
+    this.outboxRepo = new OutboxRepository(this.db, this.prefix, !!options.config?.enableOutbox);
+    this.entryRepo = new EntryRepository(this.db, this.prefix, this.outboxRepo);
+    this.taskRepo = new TaskRepository(this.db, this.prefix, this.outboxRepo);
+    this.eventRepo = new EventRepository(this.db, this.prefix);
+    this.edgeRepo = new EdgeRepository(this.db, this.prefix);
+    this.metadataRepo = new MetadataRepository(this.db, this.prefix);
     this.ontologyService = new OntologyService(
       this.metadataRepo,
       this.edgeRepo,
@@ -1848,6 +1985,24 @@ var WikiMemory = class {
   async runHeal(entityId, options) {
     return this.maintenanceService.runHeal(entityId, options);
   }
+  /**
+   * Types already-persisted untyped facts (okf_type IS NULL) in place via one
+   * librarian-style LLM call. Strictly additive: never creates, deletes, or
+   * rewrites facts; never overwrites an existing okf_type. Free when there is
+   * nothing to do (ontology off, or zero eligible untyped facts).
+   *
+   * Hosts own the trigger cadence (e.g. after each sync); loop `while
+   * result.remaining > 0` for convergence. Throws WikiBusyError when another
+   * backfill (or conflicting maintenance op) is running for the entity.
+   *
+   * @param options.promptOverride - Applies only to this call. For persistent
+   * customization set `options.config.prompts.ontologyBackfillSystemPrompt`.
+   * @param options.batchSize - Facts per run (default 25) for providers with
+   * tighter context limits.
+   */
+  async runOntologyBackfill(entityId, options) {
+    return this.maintenanceService.runOntologyBackfill(entityId, options);
+  }
   async runReembed(entityId, opts) {
     return this.maintenanceService.runReembed(entityId, opts);
   }