@equationalapplications/core-llm-wiki 5.2.1 → 5.3.1

package/dist/index.mjs

@@ -1,5 +1,5 @@
-import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, generateId, BaseRepository, emptyManifest, resolveNodeType, validateInlineEdges, resolveEdgeDefinitions, normalizeTitleKey, WikiTransactionError, extractSqliteCode } from './chunk-6HNIOKT4.mjs';
-export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDuplicateHashError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-6HNIOKT4.mjs';
+import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, WikiSourceRefHashCollision, generateId, BaseRepository, emptyManifest, resolveNodeType, WikiStrictOntologyViolation, validateInlineEdges, resolveEdgeDefinitions, normalizeTitleKey, WikiTransactionError, extractSqliteCode } from './chunk-KIJINNYU.mjs';
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDuplicateHashError, WikiSourceRefHashCollision, WikiStrictOntologyViolation, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-KIJINNYU.mjs';
 import { appendRelatedSection, buildConceptDocument, buildLogMd, buildEntityIndexMd, buildRootIndexMd, isAllowedOkfPath, parseRootIndexMd, parseEntityIndexMd, parseConcept, splitRelatedSection, extractMarkdownLinks, parseLogMd, parseEventIdComment, appendEventIdComment } from '@equationalapplications/core-okf';
 
 // src/db/schema.ts
@@ -736,12 +736,22 @@ var EntryRepository = class extends BaseRepository {
   /**
    * Fetch recent non-deleted entries for an entity (limited), ordered by updated_at DESC.
    * Used by MaintenanceService.doRunLibrarian().
-   */
-  async findRecentByEntityId(entityId, limit, tx) {
-    const executor = this.getExecutor(tx);
+   *
+   * When `excludeSourceRef` is supplied, the WHERE clause filters out that
+   * sourceRef BEFORE applying LIMIT. The caller in IngestionService is
+   * `findRecentByEntityId(entityId, 500, tx)` — it discards same-sourceRef
+   * rows after the read to build a cross-sourceRef title index. If those
+   * rows are filtered out in SQL, the limit applies to usable rows only
+   * and a re-ingest where the caller's sourceRef owns most recent facts
+   * still leaves cross-sourceRef entries in the index.
+   */
+  async findRecentByEntityId(entityId, limit, tx, excludeSourceRef) {
+    const executor = this.getExecutor(tx);
+    const excludeClause = excludeSourceRef ? ` AND (source_ref IS NULL OR source_ref != ?)` : "";
+    const args = excludeSourceRef ? [entityId, excludeSourceRef, limit] : [entityId, limit];
     const rows = await executor.getAllAsync(
-      `SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL ORDER BY updated_at DESC LIMIT ?`,
-      [entityId, limit]
+      `SELECT * FROM ${this.prefix}entries WHERE entity_id = ? AND deleted_at IS NULL${excludeClause} ORDER BY updated_at DESC LIMIT ?`,
+      args
     );
     return rows.map(mapRowToFact);
   }
@@ -1932,6 +1942,34 @@ var EdgeRepository = class extends BaseRepository {
     const executor = this.getExecutor(tx);
     await executor.runAsync(`DELETE FROM ${this.prefix}edges WHERE entity_id = ?`, [entityId]);
   }
+  /**
+   * Hard-delete every edge whose `source_id` is in `sourceFactIds`, scoped to
+   * `entityId`. Returns the number of rows deleted. Edges have no soft-delete
+   * concept; this method is named `softDeleteBySourceFactIds` for symmetry with
+   * `EntryRepository.softDeleteBySource` (which IS a soft delete), but its
+   * semantics are immediate presence/absence removal. Use ONLY inside an open
+   * transaction — the method requires `tx` and performs no autocommit.
+   *
+   * Used by `IngestionService.upsertGraphCore` to retire the prior parse's
+   * edges whose source facts are being superseded by `softDeleteBySource`.
+   */
+  async softDeleteBySourceFactIds(entityId, sourceFactIds, tx) {
+    if (sourceFactIds.length === 0) return 0;
+    const executor = this.getExecutor(tx);
+    const chunkSize = 500;
+    let totalChanges = 0;
+    for (let i = 0; i < sourceFactIds.length; i += chunkSize) {
+      const chunk = sourceFactIds.slice(i, i + chunkSize);
+      const placeholders = chunk.map(() => "?").join(",");
+      const result = await executor.runAsync(
+        `DELETE FROM ${this.prefix}edges
+         WHERE entity_id = ? AND source_id IN (${placeholders})`,
+        [entityId, ...chunk]
+      );
+      totalChanges += result.changes;
+    }
+    return totalChanges;
+  }
   /**
    * Multi-hop traversal from `sourceId` via SQLite `WITH RECURSIVE`. All filtering,
    * dead-ending, cycle-guarding, capping, and ordering happens in this one query.
@@ -2110,16 +2148,27 @@ var OntologyService = class {
     this.invalidateCache(entityId);
     return merged;
   }
-  validateAndNormalizeFact(fact, manifest) {
+  validateAndNormalizeFact(fact, manifest, opts) {
     const rawType = typeof fact.okf_type === "string" ? fact.okf_type : "";
+    const strict = opts?.strict === true;
     const canonical = resolveNodeType(rawType, manifest);
-    if (!canonical) return { okf_type: null, edges: [] };
-    const edges = validateInlineEdges(canonical, null, fact.edges ?? [], manifest);
+    if (!canonical) {
+      if (strict) throw new WikiStrictOntologyViolation(opts?.entityId ?? "", "node", rawType);
+      return { okf_type: null, edges: [] };
+    }
+    const edges = validateInlineEdges(canonical, null, fact.edges ?? [], manifest, opts);
     return { okf_type: canonical, edges };
   }
-  async resolveAndPersistEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, tx, now) {
-    if (!sourceType || edges.length === 0) return 0;
-    let persisted = 0;
+  /**
+   * Pure resolver: given a source type, LLM-supplied edges, the manifest,
+   * and a title index, return a list of concrete `WikiEdge` objects ready
+   * for persistence. Performs no DB writes. Used by callers that batch edge
+   * writes inside their own transaction (e.g. `IngestionService.ingestDocument`
+   * when delegating to `upsertGraphCore`).
+   */
+  resolveEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, now) {
+    if (!sourceType || edges.length === 0) return [];
+    const out = [];
     for (const edge of edges) {
       const candidates = resolveEdgeDefinitions(edge.edge_type, manifest).filter((d) => d.source_type.toLowerCase() === sourceType.toLowerCase());
       if (candidates.length === 0) continue;
@@ -2130,15 +2179,29 @@ var OntologyService = class {
         (d) => d.target_type.toLowerCase() === (target.okf_type ?? "").toLowerCase()
       );
       if (!def) continue;
-      const wikiEdge = {
+      out.push({
         id: generateId(),
         entity_id: entityId,
         source_id: sourceId,
         target_id: target.id,
         edge_type: def.type,
         created_at: now
-      };
-      const inserted = await this.edgeRepo.addIgnoreDuplicate(wikiEdge, tx);
+      });
+    }
+    return out;
+  }
+  /**
+   * Backwards-compatible wrapper: resolves edges via {@link resolveEdges},
+   * then persists each via `edgeRepo.addIgnoreDuplicate` in the supplied
+   * transaction. Returns the number of edges persisted. Used by paths that
+   * want self-contained edge persistence (e.g. MaintenanceService heal /
+   * backfill callers that do not have a separate edge-write step).
+   */
+  async resolveAndPersistEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, tx, now) {
+    const resolved = this.resolveEdges(entityId, sourceId, sourceType, edges, manifest, titleIndex, now);
+    let persisted = 0;
+    for (const edge of resolved) {
+      const inserted = await this.edgeRepo.addIgnoreDuplicate(edge, tx);
       if (inserted) persisted++;
     }
     return persisted;
@@ -2214,6 +2277,8 @@ var WikiMemory = class {
       this.options,
       this.entryRepo,
       this.sourceRefIndexRepo,
+      this.metadataRepo,
+      this.edgeRepo,
       this.searchService,
       this.jobManager,
       this.embeddingService,
@@ -2406,6 +2471,36 @@ var WikiMemory = class {
     const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
     return canonical === null ? [] : [canonical];
   }
+  /**
+   * Returns the set of `entity_id` values with at least one row in this
+   * database, INCLUDING entities whose only remaining rows are soft-deleted
+   * — sorted ascending `entity_id COLLATE BINARY`. Empty array when the
+   * database has no entities.
+   *
+   * The deliberate inclusion of soft-deleted-only entities closes the
+   * decommissioned-scope leak: a scoped namespace whose documents have all
+   * been forgotten or superseded must still appear so host maintenance
+   * sweeps (`runLibrarian`, `runHeal`, `runOntologyBackfill`, `runPrune`)
+   * can visit it and reap the soft-deleted rows.
+   *
+   * Read-only. Never acquires a lock. Never opens a transaction. Propagates
+   * underlying read errors.
+   *
+   * @param options.prefix - Optional string filter applied as
+   *   `id.startsWith(prefix)`. O(n) over distinct ids because the
+   *   `(entity_id, source_ref)` index is not seekable on `entity_id`-only
+   *   prefix. Empty-string prefix matches every id.
+   *
+   * Note: this widens the coverage of `exportDump` (which delegates to
+   * `MetadataRepository.getDistinctEntityIds`); exported dumps now include
+   * orphaned entities. This is a deliberate widening for backup/migration
+   * coverage, not a breaking change to `exportDump`'s return shape.
+   */
+  async listEntityIds(options) {
+    const ids = await this.metadataRepo.getDistinctEntityIds();
+    if (options?.prefix === void 0) return ids;
+    return ids.filter((id) => id.startsWith(options.prefix));
+  }
   async runPrune(entityId, options) {
     return this.maintenanceService.runPrune(entityId, options);
   }
@@ -2502,6 +2597,57 @@ var WikiMemory = class {
   async ingestDocument(entityId, params, opts) {
     return this.ingestionService.ingestDocument(entityId, params, opts);
   }
+  /**
+   * Deterministic write path into the graph. Accepts caller-supplied
+   * `(nodes, edges)` and writes them under `(sourceRef, sourceHash)`
+   * semantics identical to `ingestDocument`, minus the LLM extraction step.
+   *
+   * MUST be called from inside an open `db.withTransactionAsync` callback —
+   * the third argument is the caller's `tx`. The method does not open a
+   * nested transaction, does not acquire any lock, and does not perform
+   * post-commit work (search sync, embedding, cache eviction). Hosts that
+   * want embeddings synced should drive the existing maintenance sweep
+   * (`runLibrarian` / `runHeal` / `runOntologyBackfill` / `runPrune`,
+   * scoped via `listEntityIds`).
+   *
+   * Contract (full pre-flight validation lives inside
+   * `IngestionService.upsertGraphCore`, which this method delegates to):
+   * - C2: no-op when `(entityId, sourceHash)` is already mapped to
+   *   `params.sourceRef`; throws `WikiSourceRefHashCollision` if mapped to
+   *   a different `sourceRef`.
+   * - C3: edges with dangling `targetId` are stored verbatim (no FK, no
+   *   resolution).
+   * - C4: under persisted ontology mode `'strict'`, an out-of-manifest node
+   *   or edge `type` throws `WikiStrictOntologyViolation` (pre-flight,
+   *   all-or-nothing — NONE written on failure).
+   *
+   * @returns Counts: nodesWritten (validated nodes persisted), edgesWritten
+   *   (manifest-valid edges persisted), superseded (prior facts soft-deleted
+   *   plus prior source-ref edges hard-deleted).
+   */
+  async upsertGraph(entityId, params, adapter) {
+    const sourceRef = normalizeSourceRef(params.sourceRef);
+    if (!sourceRef) throw new Error("Invalid sourceRef");
+    const sourceHash = normalizeSourceHash(params.sourceHash);
+    if (!sourceHash) throw new Error("Invalid sourceHash (must be 64-char hex string)");
+    const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash, adapter);
+    if (canonical !== null && canonical === sourceRef) {
+      return { nodesWritten: 0, edgesWritten: 0, superseded: 0 };
+    }
+    if (canonical !== null && canonical !== sourceRef) {
+      throw new WikiSourceRefHashCollision({
+        entityId,
+        sourceHash,
+        existingSourceRef: canonical,
+        attemptedSourceRef: sourceRef
+      });
+    }
+    return this.ingestionService.upsertGraphCore(
+      entityId,
+      { sourceRef, sourceHash, nodes: params.nodes, edges: params.edges },
+      adapter
+    );
+  }
   /**
    * Returns up to `limit` unprocessed outbox events, oldest first.
    * Works regardless of enableOutbox value — allows draining after disabling.