@remnic/core 9.3.748 → 9.3.749

package/src/orchestrator.ts

@@ -114,6 +114,7 @@ import { RecallSearchPipelineCoordinator } from "./orchestration/recall-search-p
 import { TurnIngestionCoordinator } from "./orchestration/turn-ingestion.js";
 import { RecallIntrospectionCoordinator } from "./orchestration/recall-introspection.js";
 import { OrchestratorInitCoordinator } from "./orchestration/orchestrator-init.js";
+import { PersistenceIndexCoordinator } from "./orchestration/persistence-index.js";
 export { hasIdentityRecoveryIntent, resolveEffectiveIdentityInjectionMode } from "./orchestration/recall-result-formatter.js";
 import {
   GraphRecallCoordinator,
@@ -2549,17 +2550,20 @@ export class Orchestrator {
     targetStorage: StorageManager,
     content: string,
   ): Promise<boolean> {
-    const index = await this.contentHashIndexForStorage(targetStorage);
-    return index ? index.has(content) : false;
+    return this.persistenceIndexCoordinator.hasContentHashDedup(
+      targetStorage,
+      content,
+    );
   }
 
   private async addContentHashDedup(
     targetStorage: StorageManager,
     content: string,
   ): Promise<void> {
-    const index = await this.contentHashIndexForStorage(targetStorage);
-    if (!index) return;
-    index.add(content);
+    return this.persistenceIndexCoordinator.addContentHashDedup(
+      targetStorage,
+      content,
+    );
   }
 
   private async removeContentHashForMemory(
@@ -2567,50 +2571,13 @@ export class Orchestrator {
     memory: MemoryFile,
     context: string,
   ): Promise<void> {
-    const index = await this.contentHashIndexForStorage(targetStorage);
-    if (!index) return;
-
-    if (memory.frontmatter.contentHash) {
-      index.removeByHash(memory.frontmatter.contentHash);
-      return;
-    }
-
-    log.warn(
-      `[${context}] removing hash for legacy memory ${memory.frontmatter.id ?? "(unknown)"} via content fallback - no contentHash in frontmatter`,
+    return this.persistenceIndexCoordinator.removeContentHashForMemory(
+      targetStorage,
+      memory,
+      context,
     );
-    index.remove(memory.content);
   }
 
-  /**
-   * Issue #1671 — backfill bi-temporal bounds onto an existing promoted/deduped
-   * copy that was written BEFORE the source fact carried a resolved
-   * `invalid_at`/`observedAt`/`eventTimeSource`.
-   *
-   * On re-extraction/backfill, a fact may now carry a resolved end bound (e.g.
-   * "until June 2025") that the existing copy lacks because it was promoted
-   * before bi-temporal wiring existed. Without this backfill, recall keeps
-   * surfacing an expired fact even though the source copy now expires correctly.
-   *
-   * Finds the active fact in `targetStorage` matching `dedupContent`, then
-   * patches the temporal frontmatter the existing copy is missing. Best-effort
-   * / fail-open — any I/O error is logged and swallowed so the dedup
-   * short-circuit is never blocked by a backfill failure.
-   *
-   * Matching: the stored `frontmatter.contentHash` is compared against
-   * `ContentHashIndex.computeHash(dedupContent)` first (the exact hash the
-   * content-hash index uses), then falls back to stripping citations and
-   * comparing normalized bodies. This handles inline-attribution deployments
-   * where the persisted body carries a citation marker the dedup key does not.
-   *
-   * I/O gate: only triggers when `invalidAt` is present (the end bound that
-   * actually changes recall behavior by expiring the fact). `observedAt` and
-   * `eventTimeSource` alone don'\''t expire a fact, so backfilling them without
-   * an end bound would cause a full readAllMemories scan on every dedup hit
-   * under biTemporal for no recall benefit.
-   *
-   * Only patches fields the existing copy LACKS — never overwrites a bound the
-   * copy already carries.
-   */
   private async backfillTemporalBoundsOnDedupHit(
     targetStorage: StorageManager,
     dedupContent: string,
@@ -2626,130 +2593,17 @@ export class Orchestrator {
     },
     entityRef?: string,
   ): Promise<void> {
-    // I/O gate: scan when there is a recall-relevant bound to backfill —
-    // either an end bound (invalidAt, which expires the fact) or a corrected
-    // EXTRACTED start bound. A start bound only changes recall when it is
-    // extracted (the as-of filter excludes facts whose valid_at is after the
-    // as-of instant); an "assumed" validFrom is just the ingestion anchor
-    // (resolveFactEventTime sets one for every fact), so scanning on it would
-    // run a full readAllMemories on every bi-temporal dedup hit for no benefit
-    // (review cursor PRRT_OvHk / codex PRRT_OvHxVH). observedAt and
-    // eventTimeSource alone never change recall.
-    const hasExtractedStart =
-      bounds.validFrom !== undefined && bounds.eventTimeSource === "extracted";
-    if (!bounds.invalidAt && !hasExtractedStart) return;
-    try {
-      const incomingHash = ContentHashIndex.computeHash(dedupContent);
-      const normalizedIncoming = ContentHashIndex.normalizeContent(dedupContent);
-      // Normalize the entity for same-entity scoping when provided — two
-      // entities can share identical fact text, and patching a different
-      // entity'\''s fact would corrupt its temporal bounds (cursor review).
-      const incomingEntityNorm = entityRef
-        ? normalizeSupersessionKey(entityRef)
-        : undefined;
-      const all = await targetStorage.readAllMemories();
-      const existing = all.find((m) => {
-        if (m.frontmatter.category !== "fact") return false;
-        if ((m.frontmatter.status ?? "active") !== "active") return false;
-        // Same-entity guard: reject only when the stored fact carries a
-        // DIFFERENT entity (two entities can share identical fact text, so
-        // patching the other entity's copy would corrupt its bounds — codex
-        // P2 PRRT_OvB4A). Legacy facts written before entity linkage (no
-        // entityRef) have no entity to conflict with, so they stay eligible
-        // for backfill (cursor PRRT_OvKnV: the guard must NOT silently
-        // no-op for older promoted copies that predate entity linkage).
-        if (
-          incomingEntityNorm &&
-          m.frontmatter.entityRef &&
-          normalizeSupersessionKey(m.frontmatter.entityRef) !== incomingEntityNorm
-        ) {
-          return false;
-        }
-        // Prefer the stored contentHash (what the hash index actually keys
-        // on) — it is computed from contentHashSource (the raw/enriched
-        // body before citation), matching the dedupContent the caller passes.
-        if (m.frontmatter.contentHash) {
-          return m.frontmatter.contentHash === incomingHash;
-        }
-        // Legacy facts without a stored hash: strip citations then compare
-        // normalized bodies so inline-attribution markers don'\''t prevent
-        // a match.
-        return (
-          ContentHashIndex.normalizeContent(
-            stripCitationForTemplate(m.content ?? "", this.config.inlineSourceAttributionFormat),
-          ) === normalizedIncoming
-        );
-      });
-      if (!existing) return;
-      // Build a patch containing ONLY the fields the existing copy lacks.
-      const patch: Partial<MemoryFrontmatter> = {};
-      const fm = existing.frontmatter;
-      if (bounds.invalidAt && (!fm.invalid_at || fm.invalid_at.length === 0)) {
-        patch.invalid_at = bounds.invalidAt;
-      }
-      if (bounds.observedAt && (!fm.observedAt || fm.observedAt.length === 0)) {
-        patch.observedAt = bounds.observedAt;
-      }
-      if (
-        bounds.eventTimeSource &&
-        (!fm.eventTimeSource || fm.eventTimeSource.length === 0)
-      ) {
-        patch.eventTimeSource = bounds.eventTimeSource;
-      }
-      // #1707 thread 2 — per-fact-anchored start bound. A re-extracted
-      // duplicate whose event time resolves a real start bound must carry
-      // that anchor onto the existing copy so as-of recall uses the corrected
-      // valid_at instead of a stale batch-anchored value. Only an EXTRACTED
-      // bound corrects; an "assumed" bound is just the ingestion anchor.
-      //
-      // No-clobber via equality, not provenance inference: exact-content dedup
-      // re-extracts the SAME event-time expression, which #1670 per-fact
-      // anchoring resolves deterministically to the same validFrom — so for
-      // stable content the incoming validFrom EQUALS the copy's valid_at and
-      // we skip the redundant write (the only no-clobber that holds without a
-      // fragile provenance heuristic — review codex PRRT_Ov7LKC). When they
-      // differ (a prior batch/assumed anchor, end-only assumed start, or a
-      // non-deterministic re-resolution), the extracted validFrom is the
-      // authoritative correction and overwrites. The eventTimeSource upgrade
-      // below records that the start is now extracted-anchored.
-      if (
-        bounds.validFrom &&
-        bounds.eventTimeSource === "extracted" &&
-        fm.valid_at !== bounds.validFrom
-      ) {
-        patch.valid_at = bounds.validFrom;
-        // Mark the copy extracted-anchored in the SAME patch so its provenance
-        // reflects the correction (review cursor PRRT_OvHM / codex PRRT_OvHxVD):
-        // without this, a copy upgraded from "assumed" would keep "assumed"
-        // provenance while carrying an extracted start. (The earlier
-        // eventTimeSource block only fills an EMPTY source.)
-        if (fm.eventTimeSource !== "extracted") {
-          patch.eventTimeSource = "extracted";
-        }
-      }
-      if (Object.keys(patch).length === 0) return;
-      const ok = await targetStorage.writeMemoryFrontmatter(existing, patch);
-      if (ok) {
-        log.debug(
-          `bitemporal-backfill: patched ${Object.keys(patch).join(",")} onto existing fact ${fm.id ?? "(unknown)"} in ${targetStorage.dir}`,
-        );
-      }
-    } catch (err) {
-      log.warn(
-        `bitemporal-backfill: failed open for ${targetStorage.dir}: ${err}`,
-      );
-    }
+    return this.persistenceIndexCoordinator.backfillTemporalBoundsOnDedupHit(
+      targetStorage,
+      dedupContent,
+      bounds,
+      entityRef,
+    );
   }
 
   private async saveContentHashIndexes(): Promise<void> {
-    const indexes = new Set<ContentHashIndex>();
-    if (this.contentHashIndex) indexes.add(this.contentHashIndex);
-    for (const index of this.contentHashIndexesByStorageDir.values()) {
-      indexes.add(index);
-    }
-    for (const index of indexes) {
-      await index.save();
-    }
+    return this.persistenceIndexCoordinator.saveContentHashIndexes(
+    );
   }
 
   constructor(config: PluginConfig) {
@@ -5146,6 +5000,31 @@ export class Orchestrator {
     return this._orchestratorInitCoordinator;
   }
 
+  /**
+   * Persistence-index coordinator (issue #1526 seam 23). Owns post-persist
+   * bookkeeping (content-hash dedup, temporal indexes, graph edges,
+   * semantic dedup lookup). Lazy + accessor-wired (late-binding rule).
+   */
+  private _persistenceIndexCoordinator: PersistenceIndexCoordinator | undefined;
+
+  private get persistenceIndexCoordinator(): PersistenceIndexCoordinator {
+    if (!this._persistenceIndexCoordinator) {
+      // eslint-disable-next-line @typescript-eslint/no-this-alias
+      const self = this;
+      this._persistenceIndexCoordinator = new PersistenceIndexCoordinator({
+        get config() { return self.config; },
+        get contentHashIndex() { return self.contentHashIndex; },
+        contentHashIndexForStorage: (targetStorage) => self.contentHashIndexForStorage(targetStorage),
+        get contentHashIndexesByStorageDir() { return self.contentHashIndexesByStorageDir; },
+        get embeddingFallback() { return self.embeddingFallback; },
+        graphIndexFor: (storage) => self.graphIndexFor(storage),
+        readAllMemoriesForNamespaces: (namespaces) => self.readAllMemoriesForNamespaces(namespaces),
+        semanticDedupScopeFor: (targetStorage) => self.semanticDedupScopeFor(targetStorage),
+      });
+    }
+    return this._persistenceIndexCoordinator;
+  }
+
   private async recallInternal(
     prompt: string,
     sessionKey?: string,
@@ -5593,22 +5472,12 @@ export class Orchestrator {
     storage: StorageManager,
     memoryId: string,
   ): Promise<void> {
-    if (!resolveMemoryLifecycleCapabilities(this.config).embeddingFallback) return;
-    if (!(await this.embeddingFallback.isAvailable())) return;
-    const memory = await storage.getMemoryById(memoryId);
-    if (!memory) return;
-    await this.embeddingFallback.indexFile(
+    return this.persistenceIndexCoordinator.indexPersistedMemory(
+      storage,
       memoryId,
-      memory.content,
-      memory.path,
     );
   }
 
-  /**
-   * Build a graph edge for a persisted memory (v8.2).
-   * Shared helper used by both the chunked and non-chunked write paths to avoid duplication.
-   * Fail-open: caller wraps in try/catch.
-   */
   private async buildGraphEdge(
     storage: StorageManager,
     memoryRelPath: string,
@@ -5622,65 +5491,19 @@ export class Orchestrator {
     fallbackCausalPredecessor: string | undefined,
     graphCaps: GraphConstructionCapabilitySet = resolveGraphConstructionCapabilities(this.config),
   ): Promise<void> {
-    // Entity siblings: other memories sharing the same entityRef
-    const entitySiblings: string[] = [];
-    if (entityRef) {
-      try {
-        const allMems = allMemsForGraph ?? [];
-        for (const m of allMems) {
-          if (m.frontmatter.entityRef === entityRef) {
-            const rel = path.relative(storage.dir, m.path);
-            if (rel !== memoryRelPath) entitySiblings.push(rel);
-          }
-        }
-      } catch {
-        /* fail-open */
-      }
-    }
-    // Recent thread memories for time graph
-    const recentInThread: string[] = [];
-    if (threadIdForEdge && threadEpisodeIdsForGraph?.length) {
-      try {
-        recentInThread.push(
-          ...resolveRecentThreadMemoryPaths({
-            threadEpisodeIds: threadEpisodeIdsForGraph,
-            currentMemoryId: memoryId,
-            allMemsForGraph,
-            pathById: memoryPathById,
-            storageDir: storage.dir,
-            maxRecent: 3,
-          }),
-        );
-      } catch {
-        /* fail-open */
-      }
-    }
-    if (
-      recentInThread.length === 0 &&
-      graphCaps.graphWriteSessionAdjacency &&
-      fallbackCausalPredecessor &&
-      fallbackCausalPredecessor !== memoryRelPath
-    ) {
-      recentInThread.push(fallbackCausalPredecessor);
-    }
-    const causalPredecessor =
-      recentInThread[recentInThread.length - 1] ?? fallbackCausalPredecessor;
-    await this.graphIndexFor(storage).onMemoryWritten({
-      memoryPath: memoryRelPath,
+    return this.persistenceIndexCoordinator.buildGraphEdge(
+      storage,
+      memoryRelPath,
       entityRef,
-      content: factContent,
-      created: new Date().toISOString(),
-      threadId: threadIdForEdge,
-      recentInThread,
-      entitySiblings,
-      causalPredecessor,
-      graphCapsOverride: {
-        entityGraph: graphCaps.entityGraph,
-        timeGraph: graphCaps.timeGraph,
-        causalGraph: graphCaps.causalGraph,
-        multiGraphMemory: graphCaps.multiGraphMemory,
-      },
-    });
+      memoryId,
+      factContent,
+      allMemsForGraph,
+      memoryPathById,
+      threadIdForEdge,
+      threadEpisodeIdsForGraph,
+      fallbackCausalPredecessor,
+      graphCaps,
+    );
   }
 
   private graphIndexFor(storage: StorageManager): GraphIndex {
@@ -5692,89 +5515,14 @@ export class Orchestrator {
     return created;
   }
 
-  /**
-   * Batch-update temporal and tag indexes after extraction (v8.1).
-   * Reads each persisted memory's path + frontmatter and adds them to
-   * state/index_time.json and state/index_tags.json.
-   * Fail-open: any error is logged but does not abort extraction.
-   */
   private async updateTemporalTagIndexes(
     storage: StorageManager,
     persistedIds: string[],
   ): Promise<void> {
-    const caps = resolveCapabilities(this.config); // #1566 Cluster C
-    // Build temporal/tag indexes whenever either consumer is enabled:
-    // - queryAwareIndexingEnabled: uses indexes for query-aware prefiltering in recall
-    // - parallelRetrievalEnabled: temporal agent reads index_time.json for date-range lookup
-    // Enabling only parallelRetrievalEnabled without queryAwareIndexingEnabled would silently
-    // produce an empty temporal index, leaving the temporal agent with no data to work from.
-    if (
-      !resolveIndexingCapabilities(this.config).queryAwareIndexing &&
-      !caps.parallelRetrieval
-    )
-      return;
-    // Check for missing indexes BEFORE the early-return so first-time enablement
-    // can bootstrap the full corpus even when this extraction turn persisted nothing.
-    const needsFullRebuild = !indexesExist(this.config.memoryDir);
-    if (!needsFullRebuild && persistedIds.length === 0) return;
-    try {
-      // Read the corpus once to avoid N separate full-corpus scans.
-      // On full rebuild with namespaces enabled, span all configured namespaces so
-      // memories written to other namespaces before the index existed are also captured.
-      const allMemories =
-        needsFullRebuild && resolveNamespaceCapabilities(this.config).namespaces
-          ? await this.readAllMemoriesForNamespaces(
-              Array.from(
-                new Set<string>([
-                  this.config.defaultNamespace,
-                  this.config.sharedNamespace,
-                  ...this.config.namespacePolicies.map((p) => p.name),
-                ]),
-              ),
-            )
-          : await storage.readAllMemories();
-
-      // Bootstrap: index only active (non-archived, non-superseded) memories.
-      // Incremental: index only the newly persisted IDs.
-      const pool = needsFullRebuild
-        ? allMemories.filter((m) => isActiveMemoryStatus(m.frontmatter.status))
-        : (() => {
-            const idSet = new Set(persistedIds);
-            return allMemories.filter((m) => idSet.has(m.frontmatter.id));
-          })();
-
-      const entries: Array<{
-        path: string;
-        createdAt: string;
-        tags: string[];
-      }> = [];
-      for (const mem of pool) {
-        if (mem.path && mem.frontmatter?.created) {
-          entries.push({
-            path: mem.path,
-            createdAt: mem.frontmatter.created,
-            tags: mem.frontmatter.tags ?? [],
-          });
-        }
-      }
-      if (needsFullRebuild) {
-        // Always write empty indexes on full rebuild — even when the active pool
-        // is empty (e.g. store contains only archived/superseded entries).
-        // This marks bootstrap completion so indexesExist() returns true and
-        // subsequent extractions skip the full-corpus scan.
-        clearIndexes(this.config.memoryDir);
-        if (entries.length > 0) {
-          indexMemoriesBatch(this.config.memoryDir, entries);
-        }
-        log.info(
-          `temporal-index: bootstrapped from ${entries.length} active memories`,
-        );
-      } else if (entries.length > 0) {
-        indexMemoriesBatch(this.config.memoryDir, entries);
-      }
-    } catch (err) {
-      log.debug(`temporal-index update failed (non-fatal): ${err}`);
-    }
+    return this.persistenceIndexCoordinator.updateTemporalTagIndexes(
+      storage,
+      persistedIds,
+    );
   }
 
   /** IDs of facts persisted in the last extraction */
@@ -6198,60 +5946,16 @@ export class Orchestrator {
     );
   }
 
-  /**
-   * Issue #373 — nearest-neighbor lookup for the write-time semantic dedup
-   * guard. Returns the top-K embedding hits against the currently indexed
-   * memories, or an empty array when the embedding backend is unavailable.
-   * Intentionally does NOT throw; `decideSemanticDedup` treats both "empty"
-   * and "error" outcomes as fail-open (keep the candidate).
-   *
-   * PR #399 P1 fix: when namespaces are enabled the lookup must be scoped
-   * to the SAME namespace as the fact being written. Otherwise a
-   * high-similarity memory from another namespace can suppress a write in
-   * the target namespace — cross-tenant data loss. Callers pass the target
-   * storage so we can translate its root directory into the correct index
-   * path prefix (and, for the legacy default-namespace layout at
-   * `memoryDir` root, an exclusion list for `namespaces/*`).
-   */
   async semanticDedupLookup(
     content: string,
     limit: number,
     targetStorage: StorageManager,
   ): Promise<SemanticDedupHit[]> {
-    // Round 6 fix (Finding 3): backend-unavailable conditions must THROW so
-    // that `decideSemanticDedup`'s catch block can return
-    // reason="backend_unavailable".  Previously all error/unavailable paths
-    // returned [] — causing decideSemanticDedup to always report
-    // reason="no_candidates" even when the provider was actually down.
-    //
-    // Contract after this fix:
-    //   • embeddingFallbackEnabled=false  → throw (feature not configured;
-    //     caller treats this as backend_unavailable and fails open).
-    //   • isAvailable() returns false     → throw (provider is reachable but
-    //     reports itself unavailable; distinct from empty index).
-    //   • search() throws                 → re-throw (network/provider error).
-    //   • search() returns []             → return [] (empty index, not a
-    //     backend failure; decideSemanticDedup reports no_candidates).
-    if (!resolveMemoryLifecycleCapabilities(this.config).embeddingFallback) {
-      throw new Error("semantic dedup: embedding backend not configured");
-    }
-    if (!(await this.embeddingFallback.isAvailable())) {
-      log.debug("semantic dedup: embedding backend unavailable, skipping");
-      throw new Error("semantic dedup: embedding backend unavailable");
-    }
-    // search() may throw — let it propagate so decideSemanticDedup catches it
-    // and returns reason="backend_unavailable". Pass throwOnTimeout:true so
-    // EmbeddingTimeoutError is re-thrown here (Round 10 fix, Ui1J+Ui1L: the
-    // recall-path caller searchEmbeddingFallback does NOT pass this flag,
-    // keeping its fail-open [] contract on timeout).
-    const scope = this.semanticDedupScopeFor(targetStorage);
-    const hits = await this.embeddingFallback.search(content, limit, { ...scope, throwOnTimeout: true });
-    if (!Array.isArray(hits) || hits.length === 0) return [];
-    return hits.map((hit) => ({
-      id: hit.id,
-      score: hit.score,
-      path: hit.path,
-    }));
+    return this.persistenceIndexCoordinator.semanticDedupLookup(
+      content,
+      limit,
+      targetStorage,
+    );
   }
 
   /**