@equationalapplications/core-llm-wiki 5.2.0 → 5.2.1

package/dist/index.mjs

@@ -1,5 +1,5 @@
-import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, generateId, BaseRepository, emptyManifest, resolveNodeType, validateInlineEdges, resolveEdgeDefinitions, normalizeTitleKey, WikiTransactionError, extractSqliteCode } from './chunk-J3N3WRK7.mjs';
-export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDuplicateHashError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-J3N3WRK7.mjs';
+import { __privateAdd, MetadataRepository, EmbeddingService, SearchService, JobManager, PromptService, IngestionService, MaintenanceService, ImportExportService, RetrievalService, WriteService, __privateGet, __privateSet, normalizeSourceRef, normalizeSourceHash, entitySummaryMetaKey, generateId, BaseRepository, emptyManifest, resolveNodeType, validateInlineEdges, resolveEdgeDefinitions, normalizeTitleKey, WikiTransactionError, extractSqliteCode } from './chunk-6HNIOKT4.mjs';
+export { DEFAULT_CHUNK_OVERLAP, DEFAULT_MAX_CHUNK_LENGTH, HEAL_BATCH_SIZE, HEAL_RECHECK_MS, HOOK_TIMEOUT_MARKER, ONTOLOGY_BACKFILL_BATCH_SIZE, ONTOLOGY_BACKFILL_MAX_PROMPT_CHARS, ONTOLOGY_BACKFILL_RECHECK_MS, ONTOLOGY_BACKFILL_SYSTEM_PROMPT, PromptService, PrunePartialFailureError, WikiBusyError, WikiDuplicateHashError, WikiTransactionError, chunkText, configureRandomSource, parseEmbedding, safeSlice, validateManifest } from './chunk-6HNIOKT4.mjs';
 import { appendRelatedSection, buildConceptDocument, buildLogMd, buildEntityIndexMd, buildRootIndexMd, isAllowedOkfPath, parseRootIndexMd, parseEntityIndexMd, parseConcept, splitRelatedSection, extractMarkdownLinks, parseLogMd, parseEventIdComment, appendEventIdComment } from '@equationalapplications/core-okf';
 
 // src/db/schema.ts
@@ -32,6 +32,24 @@ async function setupDatabase(db, prefix) {
     CREATE INDEX IF NOT EXISTS ${prefix}entries_source_hash_idx ON ${prefix}entries(entity_id, source_hash) WHERE source_hash IS NOT NULL;
     CREATE INDEX IF NOT EXISTS ${prefix}entries_updated_idx ON ${prefix}entries(updated_at DESC);
 
+    -- source_ref_index: per-(entity, source_hash) record of the canonical sourceRef
+    -- currently holding that hash. The partial UNIQUE index on (entity_id, source_hash)
+    -- WHERE deleted_at IS NULL enforces the sourceRef-level TOCTOU-race invariant;
+    -- entries-level uniqueness cannot express it because a single ingestDocument call
+    -- writes N facts that all share (entity_id, source_ref, source_hash). See
+    -- docs/superpowers/specs/2026-08-07-dependabot-concurrency-release-hygiene-design.md \xA7B1.
+    CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
+      id TEXT PRIMARY KEY,
+      entity_id TEXT NOT NULL,
+      source_hash TEXT NOT NULL,
+      source_ref TEXT NOT NULL,
+      created_at INTEGER NOT NULL,
+      deleted_at INTEGER
+    );
+    CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
+      ON ${prefix}source_ref_index (entity_id, source_hash)
+      WHERE deleted_at IS NULL;
+
     CREATE TABLE IF NOT EXISTS ${prefix}tasks (
       id TEXT PRIMARY KEY,
       entity_id TEXT NOT NULL,
@@ -278,6 +296,61 @@ var MIGRATIONS = [
         );
       }
     }
+  },
+  {
+    version: 9,
+    description: "add_source_ref_index",
+    run: async (db, prefix) => {
+      const duplicates = await db.getAllAsync(
+        `SELECT entity_id, source_hash, COUNT(DISTINCT source_ref) AS n_refs
+         FROM ${prefix}entries
+         WHERE deleted_at IS NULL AND source_hash IS NOT NULL
+         GROUP BY entity_id, source_hash
+         HAVING COUNT(DISTINCT source_ref) > 1`
+      );
+      if (duplicates.length > 0) {
+        const sample = duplicates.slice(0, 5).map((d) => `(entity_id=${d.entity_id}, source_hash=${d.source_hash.slice(0, 12)}\u2026, n_refs=${d.n_refs})`).join(", ");
+        throw new Error(
+          `Migration v9 (add_source_ref_index) failed: existing live rows have multiple sourceRefs sharing a hash. Found ${duplicates.length} duplicate (entity_id, source_hash) groups. First ${Math.min(5, duplicates.length)}: ${sample}. Resolve each by calling forget({ sourceRef: <loser> }) for the offending sourceRef, then re-run setup.`
+        );
+      }
+      await db.execAsync(
+        `CREATE TABLE IF NOT EXISTS ${prefix}source_ref_index (
+          id TEXT PRIMARY KEY,
+          entity_id TEXT NOT NULL,
+          source_hash TEXT NOT NULL,
+          source_ref TEXT NOT NULL,
+          created_at INTEGER NOT NULL,
+          deleted_at INTEGER
+        );
+        CREATE UNIQUE INDEX IF NOT EXISTS ${prefix}idx_source_ref_hash
+          ON ${prefix}source_ref_index (entity_id, source_hash)
+          WHERE deleted_at IS NULL;`
+      );
+      await db.execAsync(
+        `INSERT OR IGNORE INTO ${prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
+         SELECT
+           'sri:' || entity_id || ':' || source_hash,
+           entity_id,
+           source_hash,
+           source_ref,
+           updated_at,
+           NULL
+         FROM (
+           SELECT
+             entity_id, source_hash, source_ref, updated_at,
+             ROW_NUMBER() OVER (
+               PARTITION BY entity_id, source_hash
+               ORDER BY updated_at ASC, id ASC
+             ) AS rn
+           FROM ${prefix}entries
+           WHERE deleted_at IS NULL
+             AND source_hash IS NOT NULL
+             AND source_ref IS NOT NULL
+         ) ranked
+         WHERE rn = 1;`
+      );
+    }
   }
 ];
 for (let i = 1; i < MIGRATIONS.length; i++) {
@@ -1394,6 +1467,57 @@ var OutboxRepository = class extends BaseRepository {
   }
 };
 
+// src/repositories/SourceRefIndexRepository.ts
+var SourceRefIndexRepository = class extends BaseRepository {
+  /**
+   * Idempotent insert: the partial UNIQUE index catches concurrent inserts for
+   * the same (entity_id, source_hash). The caller (IngestionService) catches
+   * the resulting SQLITE_CONSTRAINT_UNIQUE and translates it to the per-mode
+   * duplicate-hash outcome. Runtime IDs use the `sri_` prefix to avoid
+   * collision with the deterministic `sri:<entity>:<hash>` IDs used by the v9
+   * backfill (the colon separator keeps the two ID spaces disjoint).
+   */
+  async upsert(entityId, sourceHash, sourceRef, tx) {
+    const executor = this.getExecutor(tx);
+    await executor.runAsync(
+      `INSERT INTO ${this.prefix}source_ref_index (id, entity_id, source_hash, source_ref, created_at, deleted_at)
+       VALUES (?, ?, ?, ?, ?, NULL)`,
+      [generateId("sri_"), entityId, sourceHash, sourceRef, Date.now()]
+    );
+  }
+  /**
+   * Idempotent soft-delete of the live row for (entity_id, source_ref).
+   * Called at the start of every ingestDocument to remove the prior run's
+   * index row, so the new upsert doesn't collide with itself. No-op when the
+   * row is already soft-deleted or never existed.
+   */
+  async softDeleteByEntityAndSourceRef(entityId, sourceRef, tx) {
+    const executor = this.getExecutor(tx);
+    const now = Date.now();
+    const result = await executor.runAsync(
+      `UPDATE ${this.prefix}source_ref_index
+       SET deleted_at = ?, created_at = ?
+       WHERE entity_id = ? AND source_ref = ? AND deleted_at IS NULL`,
+      [now, now, entityId, sourceRef]
+    );
+    return result.changes;
+  }
+  /**
+   * Returns the live sourceRef holding the given hash, or null when no live
+   * row exists. Used by the IngestionService pre-check (line 82) and the
+   * catch-and-translate canonical lookup (line 212).
+   */
+  async findActiveByEntityAndHash(entityId, sourceHash, tx) {
+    const executor = this.getExecutor(tx);
+    const row = await executor.getFirstAsync(
+      `SELECT source_ref FROM ${this.prefix}source_ref_index
+       WHERE entity_id = ? AND source_hash = ? AND deleted_at IS NULL`,
+      [entityId, sourceHash]
+    );
+    return row?.source_ref ?? null;
+  }
+};
+
 // src/repositories/TaskRepository.ts
 function mapRowToTask(row) {
   return {
@@ -2070,6 +2194,7 @@ var WikiMemory = class {
     }
     this.outboxRepo = new OutboxRepository(this.db, this.prefix, !!options.config?.enableOutbox);
     this.entryRepo = new EntryRepository(this.db, this.prefix, this.outboxRepo);
+    this.sourceRefIndexRepo = new SourceRefIndexRepository(this.db, this.prefix);
     this.taskRepo = new TaskRepository(this.db, this.prefix, this.outboxRepo);
     this.eventRepo = new EventRepository(this.db, this.prefix);
     this.edgeRepo = new EdgeRepository(this.db, this.prefix);
@@ -2088,6 +2213,7 @@ var WikiMemory = class {
       this.prefix,
       this.options,
       this.entryRepo,
+      this.sourceRefIndexRepo,
       this.searchService,
       this.jobManager,
       this.embeddingService,
@@ -2099,6 +2225,7 @@ var WikiMemory = class {
       this.prefix,
       this.options,
       this.entryRepo,
+      this.sourceRefIndexRepo,
       this.taskRepo,
       this.eventRepo,
       this.metadataRepo,
@@ -2163,6 +2290,7 @@ var WikiMemory = class {
       promptService: this.promptService,
       graphTraversalService: this.graphTraversalService,
       entryRepo: this.entryRepo,
+      sourceRefIndexRepo: this.sourceRefIndexRepo,
       metadataRepo: this.metadataRepo,
       jobManager: this.jobManager
     };
@@ -2239,7 +2367,7 @@ var WikiMemory = class {
     const latestHashes = await this.entryRepo.findLatestSourceHashes(entityId, normalized.map((e) => e.sourceRef));
     const distinctHashes = Array.from(new Set(normalized.map((e) => e.sourceHash)));
     const dupRefs = await Promise.all(
-      distinctHashes.map((h) => this.entryRepo.findSourceRefsByHash(entityId, h))
+      distinctHashes.map((h) => this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, h))
     );
     const dupMap = /* @__PURE__ */ new Map();
     for (let i = 0; i < distinctHashes.length; i++) {
@@ -2248,12 +2376,10 @@ var WikiMemory = class {
     return normalized.map((e) => {
       const stored = latestHashes.get(e.sourceRef);
       const changed = stored === void 0 || stored === null || normalizeSourceHash(stored) !== e.sourceHash;
-      const allRefs = dupMap.get(e.sourceHash) ?? [];
-      const others = allRefs.filter((r) => r !== e.sourceRef);
-      if (others.length === 0) {
+      const canonical = dupMap.get(e.sourceHash) ?? null;
+      if (canonical === null || canonical === e.sourceRef) {
         return { sourceRef: e.rawSourceRef, changed };
       }
-      const canonical = others[0];
       return { sourceRef: e.rawSourceRef, changed, duplicateOf: canonical };
     });
   }
@@ -2268,13 +2394,17 @@ var WikiMemory = class {
     return this.entryRepo.listSourceRefs(entityId);
   }
   /**
-   * Returns the live source_refs for an entity that hold the given source_hash,
-   * sorted `COLLATE BINARY` ascending. The first element is the canonical ref
-   * under the code-unit-minimum rule (no locale dependency). Used by the
-   * ingestDocument guard and by hosts auditing duplicate-content collisions.
+   * Returns the live source_refs for an entity that hold the given source_hash.
+   * With v9, source_ref_index is the source of truth for the sourceRef-level
+   * TOCTOU-race invariant: at most one sourceRef can hold a given
+   * (entity_id, source_hash). The result is either a single-element array
+   * (one canonical ref) or empty (no live ref holds the hash). Returned as
+   * an array to preserve the existing public-API shape used by hosts
+   * auditing duplicate-content collisions.
    */
   async findSourceRefsByHash(entityId, sourceHash) {
-    return this.entryRepo.findSourceRefsByHash(entityId, sourceHash);
+    const canonical = await this.sourceRefIndexRepo.findActiveByEntityAndHash(entityId, sourceHash);
+    return canonical === null ? [] : [canonical];
   }
   async runPrune(entityId, options) {
     return this.maintenanceService.runPrune(entityId, options);