@remnic/core 10.0.65 → 10.0.67

package/src/orchestration/extraction-persist.ts

@@ -1,4 +1,5 @@
 import { classifyExtractionOrigin, type ExtractionSourceContext } from "./extraction-origin-context.js";
+import { confirmActiveFactDuplicate, persistUnderFactDedupWindow } from "./fact-dedup-window.js";
 import { evaluateInjectionScreen, screenEntityForIndex, screenPersistStrings } from "./extraction-injection-gate.js";
 /**
  * Extraction persistence coordinator (issue #1526, seam 16).
@@ -127,6 +128,16 @@ import {
   normalizeStoredHashSource as normalizeStoredHashSourceContent,
 } from "./extraction-persist-citation.js";
 
+/** One extraction run's persist outcome: per-run values, no shared side channels (#1526). */
+export interface ExtractionPersistResult {
+  persistedIds: string[];
+  memoryPathById: Map<string, string>;
+  /** Facts the judge deferred — drives this run's buffer retention (#562). */
+  deferredCount: number;
+  /** Persisted pending_review ids, excluded from the thread episode set (#1635). */
+  pendingReviewIds: string[];
+}
+
 export class ExtractionPersistCoordinator {
   constructor(
     private readonly deps: ExtractionPersistDeps,
@@ -149,7 +160,7 @@ export class ExtractionPersistCoordinator {
     sourceText?: string,
     graphCaps: GraphConstructionCapabilitySet = resolveGraphConstructionCapabilities(this.deps.config),
     lifecycleCaps: MemoryLifecycleCapabilitySet = resolveMemoryLifecycleCapabilities(this.deps.config),
-  ): Promise<{ persistedIds: string[]; memoryPathById: Map<string, string> }> {
+  ): Promise<ExtractionPersistResult> {
     const citationEnabled = resolvePipelineProcessingCapabilities(this.deps.config).inlineSourceAttribution === true;
     const citationTemplate = this.deps.config.inlineSourceAttributionFormat;
     const factDedupEnabled = resolveRecallAuxiliaryCapabilities(this.deps.config).factDeduplication;
@@ -190,6 +201,10 @@ export class ExtractionPersistCoordinator {
       validAt && validAt.length > 0 ? validAt : new Date().toISOString();
     // #1635: pending_review persisted ids, excluded from the thread episode set below.
     const pendingReviewPersistedIds: string[] = [];
+    // This run's judge-deferred count, returned to the caller (issue #562 PR 2)
+    // instead of shared side-channel state, so concurrent extractions cannot
+    // read each other's retention decision.
+    let deferredCount = 0;
     const persistedIdsByStorage = new Map<string, { storage: StorageManager; ids: string[] }>();
     const trackPersistedId = (
       targetStorage: StorageManager,
@@ -1071,7 +1086,7 @@ export class ExtractionPersistCoordinator {
         "persistExtraction: result or result.facts is invalid, skipping",
         { resultType: typeof result, factsType: typeof result?.facts },
       );
-      return { persistedIds, memoryPathById };
+      return { persistedIds, memoryPathById, deferredCount, pendingReviewIds: pendingReviewPersistedIds };
     }
 
     // Chunking config from plugin settings
@@ -1243,10 +1258,6 @@ export class ExtractionPersistCoordinator {
     // be persisted, not the raw extraction-time category.
     let judgeVerdictsByFactIndex: Map<number, import("../extraction-judge.js").JudgeVerdict> | null = null;
     let judgeGatedCount = 0;
-    // Reset the side-channel defer count at the start of every
-    // persistExtraction call so stale state from a prior call cannot leak
-    // into the caller's buffer-retention decision.
-    this.deps.setLastPersistExtractionDeferredCount(0);
     if (lifecycleCaps.extractionJudge) {
       // #1669 P1 + #1713 Item 1: pre-filter redacted facts from judge candidates
       // so never-store content is not persisted as judge training data.
@@ -1433,7 +1444,7 @@ export class ExtractionPersistCoordinator {
         );
         // Expose defer count to the caller (issue #562 PR 2) so it can decide
         // whether to retain buffer turns for the next extraction pass.
-        this.deps.setLastPersistExtractionDeferredCount(judgeResult.deferred);
+        deferredCount = judgeResult.deferred;
       } catch (err) {
         // Fail-open: if the entire judge pipeline errors, proceed without filtering
         log.warn(
@@ -1653,148 +1664,123 @@ export class ExtractionPersistCoordinator {
         fact.tags,
       );
       let exactDuplicate = false;
-      let needsCorpusConfirm = false;
       try {
         exactDuplicate = await this.deps.hasContentHashDedup(targetStorage, contentHashDedupKey);
-        if (factDedupEnabled && !exactDuplicate) {
-          // Fact dedup disabled → skip the authority check AND (via the guarded
-          // corpus block below) corpus confirmation, so a disabled deployment
-          // never suppresses a write via dedup (PR #2016 thread SD-nH). Enabled:
-          // a MISS is trustworthy only against an AUTHORITATIVE index — re-check.
-          if (await targetStorage.isFactContentHashAuthoritative()) {
-            exactDuplicate = await this.deps.hasContentHashDedup(targetStorage, contentHashDedupKey);
-          } else {
-            needsCorpusConfirm = true;
-          }
-        }
       } catch (err) {
-        // Dedup lookup failed (e.g. the authoritative rebuild could not run). Do
-        // NOT fail-open into a possible duplicate — confirm against the corpus.
-        needsCorpusConfirm = true;
+        // Dedup lookup failed (e.g. the authoritative rebuild could not run).
+        // When dedup is enabled the confirm below still scans the corpus; when
+        // it is disabled, never suppress a write via dedup (PR #2016 SD-nH).
+        exactDuplicate = false;
         log.warn(
           `content-hash dedup lookup unavailable for storage ${targetStorage.dir}; confirming against corpus: ${err}`,
         );
       }
-      // Connector-aware dedup (QOjlB): if the hash says duplicate — or the index
-      // could not authoritatively confirm a miss — verify a same-content,
-      // same-connector active fact exists across the hot+cold tiers. Different
-      // connectors with the same content are NOT duplicates. Fail open (write) on
-      // scan failure so an unverifiable state cannot silently drop content.
-      if (factDedupEnabled && (exactDuplicate || needsCorpusConfirm)) {
-        try {
-          // #2016 cold-tier finding: the authoritative content-hash rebuild
-          // unions the HOT and COLD tiers, so a hash hit can name an active copy
-          // that was demoted to cold/. A hot-only confirmation scan misses it,
-          // flips exactDuplicate back to false, and writes a second active hot
-          // copy of the same content. Scan both tiers so a cold-only active copy
-          // still confirms the duplicate and suppresses the redundant hot write.
-          const [hotMems, coldMems] = await Promise.all([
-            targetStorage.readAllMemories(),
-            targetStorage.readAllColdMemories(),
-          ]);
-          const nc = sourceContext?.sourceConnector?.trim() || undefined;
-          const matchesActiveDuplicate = (m: MemoryFile): boolean => {
-            if (m.frontmatter.category !== writeCategory) return false;
-            if ((m.frontmatter.status ?? "active") !== "active") return false;
-            // Thread 5 (QPDE5): for procedures the hash is keyed on the full
-            // body (title + steps via buildProcedurePersistBody), so compare
-            // against contentHashDedupKey — not canonicalContentForHash which
-            // is the title only. Thread 3 (QO42V): facts with
-            // structuredAttributes get an appended [Attributes: ...] suffix in
-            // the stored body that the hash key lacks; strip it before
-            // comparing so same-connector enriched facts dedup correctly.
-            if (normalizeStoredHashSource(stripAttributesSuffix(m.content ?? "")) !==
-              ContentHashIndex.normalizeContent(contentHashDedupKey)) return false;
-            return (m.frontmatter.sourceConnector?.trim() || undefined) === nc;
-          };
-          exactDuplicate =
-            hotMems.some(matchesActiveDuplicate) || coldMems.some(matchesActiveDuplicate);
-        } catch (err) {
-          log.warn(
-            `connector-aware dedup scan failed for storage ${targetStorage.dir}; writing fail-open: ${err instanceof Error ? err.message : String(err)}`,
-          );
-          exactDuplicate = false;
-        }
+      // Shared confirm options for the persist-time check and the write-window
+      // race recheck below.
+      const raceDedupOpts = (key: string) => ({
+        category: writeCategory,
+        contentHashDedupKey: key,
+        connector: sourceContext?.sourceConnector?.trim() || undefined,
+        citationEnabled,
+        citationTemplate,
+      });
+      // Fact dedup disabled → skip the authority check AND the corpus
+      // confirmation, so a disabled deployment never suppresses a write via
+      // dedup (PR #2016 thread SD-nH).
+      if (factDedupEnabled) {
+        exactDuplicate = await confirmActiveFactDuplicate(
+          { hasContentHashDedup: (storage, key) => this.deps.hasContentHashDedup(storage, key) },
+          targetStorage,
+          raceDedupOpts(contentHashDedupKey),
+        );
       }
-      if (exactDuplicate && injectionScreenStatus !== "pending_review") {
-        // #1671 — before short-circuiting, backfill bi-temporal bounds
-        // onto the existing source-namespace copy if it lacks bounds the
-        // incoming fact now carries (re-extraction with a resolved invalidAt).
-        // Skip when the fact would be rejected/deferred/pending by downstream
-        // gates — a non-durable candidate must not expire an active fact
-        // (chatgpt-codex P1: faithfulness, requireSpans, extraction judge).
-        // #1671 + #1707: backfill bi-temporal bounds onto the existing
-        // source-namespace copy if it lacks bounds the incoming fact now
-        // carries (re-extraction with a resolved bound). #1707 thread 3:
-        // gate on writeCategory === "fact" — the helper only matches facts,
-        // so a non-fact duplicate must not reach the fact-only scan.
-        // #1707 thread 2: also fire when only a corrected start bound
-        // (validFrom) is present, not just an end bound (validUntil). The
-        // downstream-gate skip still applies — a non-durable candidate must
-        // not expire an active fact (chatgpt-codex P1).
+      // PR #2016/#3231: finalize committed writes before a fail-closed
+      // window throw propagates (PRRT_kwDORJXyws6zxG6H).
+      const finalizeOnWindowFailure = async (err: Error): Promise<never> => {
+        await finalizeCommittedWrites(true).catch((finalizeErr) =>
+          log.warn(`committed-write finalization after fact-dedup window failure failed: ${finalizeErr}`),
+        );
+        throw err;
+      };
+      const raceRecheck = (storage: StorageManager, key: string) => () => {
+        // Gates mirror the persist-time suppression gates (PR #2016 SD-nH; #1645 quarantine).
+        if (!factDedupEnabled || injectionScreenStatus === "pending_review") return Promise.resolve(false);
+        return confirmActiveFactDuplicate(
+          { hasContentHashDedup: (s, k) => this.deps.hasContentHashDedup(s, k) },
+          storage,
+          raceDedupOpts(key),
+        );
+      };
+      // #1671/#1707 — backfill bi-temporal bounds onto the surviving copy
+      // when a duplicate short-circuits: persist-time hit OR raced duplicate
+      // suppressed in the window (codex P2, PRRT_kwDORJXyws6zxlor). Facts
+      // only (#1707 t3); a corrected start bound counts (t2). Skipped when a
+      // downstream gate would reject the candidate — a non-durable candidate
+      // must not expire an active fact (faithfulness, requireSpans, judge,
+      // #372 importance).
+      const backfillDuplicateTemporalBounds = async () => {
+        if (!(biTemporal && writeCategory === "fact" && (biTemporal.validUntil || biTemporal.validFrom))) return;
+        const fr = faithfulnessResultsByFactIndex?.get(factLoopIndex);
+        const faithfulnessWouldPending =
+          faithfulnessMode === "enforce" &&
+          fr?.ok === true &&
+          (fr.verdict === "unsupported" || fr.verdict === "contradicted");
+        const requireSpansWouldPending =
+          this.deps.config.provenance?.requireSpans === true &&
+          fact.requireSpansPending === true;
+        const judgeVerdict = judgeVerdictsByFactIndex?.get(factLoopIndex);
+        const judgeWouldGate =
+          !this.deps.config.extractionJudgeShadow &&
+          judgeVerdict !== undefined &&
+          !judgeVerdict.durable;
+        // Importance gate (cursor PRRT_OvKnS): below-threshold duplicates
+        // would never persist (#372) and carry no judge verdict (the
+        // pre-filter skips them), so they must not expire an active fact.
         if (
-          biTemporal &&
-          writeCategory === "fact" &&
-          (biTemporal.validUntil || biTemporal.validFrom)
+          !isAboveImportanceThreshold(importance.level, this.deps.config.extractionMinImportanceLevel) ||
+          faithfulnessWouldPending ||
+          requireSpansWouldPending ||
+          judgeWouldGate
         ) {
-          const fr = faithfulnessResultsByFactIndex?.get(factLoopIndex);
-          const faithfulnessWouldPending =
-            faithfulnessMode === "enforce" &&
-            fr?.ok === true &&
-            (fr.verdict === "unsupported" || fr.verdict === "contradicted");
-          const requireSpansWouldPending =
-            this.deps.config.provenance?.requireSpans === true &&
-            fact.requireSpansPending === true;
-          const judgeVerdict = judgeVerdictsByFactIndex?.get(factLoopIndex);
-          const judgeWouldGate =
-            !this.deps.config.extractionJudgeShadow &&
-            judgeVerdict !== undefined &&
-            !judgeVerdict.durable;
-          // Importance gate (cursor PRRT_OvKnS): below-threshold duplicates
-          // would never persist (#372) and carry no judge verdict (the
-          // pre-filter skips them), so they must not expire an active fact.
-          if (
-            isAboveImportanceThreshold(importance.level, this.deps.config.extractionMinImportanceLevel) &&
-            !faithfulnessWouldPending &&
-            !requireSpansWouldPending &&
-            !judgeWouldGate
-          ) {
-            await this.deps.backfillTemporalBoundsOnDedupHit(
-              targetStorage,
-              contentHashDedupKey,
-              {
-                invalidAt: biTemporal.validUntil,
-                // #1707 thread 2 — carry the corrected start bound too.
-                validFrom: biTemporal.validFrom,
-                observedAt: biTemporal.observedAt,
-                eventTimeSource: biTemporal.eventTimeSource,
-              },
-              fact.entityRef,
-              sourceContext?.sourceConnector,
-            );
-            // #1707 thread 1 — the source branch short-circuits (`continue`
-            // below) before the promotion dedup paths run, so promoted
-            // shared/profile copies written before the source fact carried
-            // resolved bounds stay stale and cross-namespace recall surfaces
-            // an expired fact. Backfill the promotion targets too (fail-open,
-            // backfill-only — never writes a new promoted copy).
-            await backfillTemporalBoundsOnPromotionCopies({
-              sourceStorage: targetStorage,
-              content: fact.content,
-              category: writeCategory,
-              confidence: fact.confidence,
-              entityRef: fact.entityRef,
-              structuredAttributes: fact.structuredAttributes,
-              sourceConnector: sourceContext?.sourceConnector,
-              bounds: {
-                invalidAt: biTemporal.validUntil,
-                validFrom: biTemporal.validFrom,
-                observedAt: biTemporal.observedAt,
-                eventTimeSource: biTemporal.eventTimeSource,
-              },
-            });
-          }
+          return;
         }
+        await this.deps.backfillTemporalBoundsOnDedupHit(
+          targetStorage,
+          contentHashDedupKey,
+          {
+            invalidAt: biTemporal.validUntil,
+            // #1707 thread 2 — carry the corrected start bound too.
+            validFrom: biTemporal.validFrom,
+            observedAt: biTemporal.observedAt,
+            eventTimeSource: biTemporal.eventTimeSource,
+          },
+          fact.entityRef,
+          sourceContext?.sourceConnector,
+        );
+        // #1707 thread 1 — a short-circuited source branch never reaches the
+        // promotion dedup paths, so promoted shared/profile copies written
+        // before the source fact carried resolved bounds stay stale and
+        // cross-namespace recall surfaces an expired fact. Backfill the
+        // promotion targets too (fail-open, backfill-only — never writes a
+        // new promoted copy).
+        await backfillTemporalBoundsOnPromotionCopies({
+          sourceStorage: targetStorage,
+          content: fact.content,
+          category: writeCategory,
+          confidence: fact.confidence,
+          entityRef: fact.entityRef,
+          structuredAttributes: fact.structuredAttributes,
+          sourceConnector: sourceContext?.sourceConnector,
+          bounds: {
+            invalidAt: biTemporal.validUntil,
+            validFrom: biTemporal.validFrom,
+            observedAt: biTemporal.observedAt,
+            eventTimeSource: biTemporal.eventTimeSource,
+          },
+        });
+      };
+      if (exactDuplicate && injectionScreenStatus !== "pending_review") {
+        await backfillDuplicateTemporalBounds();
         log.debug(
           `dedup: skipping duplicate fact "${fact.content.slice(0, 60)}…" in storage ${targetStorage.dir}`,
         );
@@ -2188,7 +2174,11 @@ export class ExtractionPersistCoordinator {
             },
             { source: extractionWriteSource },
 );
-          const parentWrite = await targetStorage.writeSealedMemory(parentWriteEnvelope, {
+          const parentWriteOutcome = await persistUnderFactDedupWindow(
+            targetStorage.dir,
+            raceRecheck(targetStorage, contentHashDedupKey),
+            async () =>
+              targetStorage.writeSealedMemory(parentWriteEnvelope, {
             importance,
             supersedes,
             links: links.length > 0 ? links : undefined,
@@ -2208,7 +2198,16 @@ export class ExtractionPersistCoordinator {
             // durable parent .md; flush the hash now so a throw can't strand it (dup).
             deferHashIndexSave: false,
             ...(factToolScoped ? { toolScoped: true as const } : {}),
-          });
+              }),
+          ).catch(finalizeOnWindowFailure);
+          if (parentWriteOutcome.suppressed) {
+            // Codex P2: backfill bounds onto the winning copy first.
+            await backfillDuplicateTemporalBounds();
+            log.info(`dedup: raced duplicate suppressed in fact-dedup window (chunked parent) for storage ${targetStorage.dir}`);
+            dedupedCount++;
+            continue;
+          }
+          const parentWrite = parentWriteOutcome.result;
           const parentId = parentWrite.id;
           // #1645: surface the tombstone block and gate active post-write paths
           const tombstoneBlocked = parentWrite.tombstoneBlocked;
@@ -2579,7 +2578,11 @@ export class ExtractionPersistCoordinator {
         },
         { source: extractionWriteSource },
 );
-      const factWrite = await targetStorage.writeSealedMemory(factWriteEnvelope, {
+      const factWriteOutcome = await persistUnderFactDedupWindow(
+        targetStorage.dir,
+        raceRecheck(targetStorage, contentHashDedupKey),
+        async () =>
+          targetStorage.writeSealedMemory(factWriteEnvelope, {
         importance,
         supersedes,
         links: links.length > 0 ? links : undefined,
@@ -2597,11 +2600,21 @@ export class ExtractionPersistCoordinator {
         // through to frontmatter so they survive end-to-end.
         ...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
         ...(fact.provenance ? { provenance: fact.provenance } : {}),
-        // #1909: defer the per-fact index flush to the batch save only when
-        // fact dedup is on (the batch saver is a no-op otherwise).
-        deferHashIndexSave: factDedupEnabled,
+        // PR #2016: never defer inside the commit window (chunked-path
+        // rule): a peer loading an authoritative index after the release
+        // must see this hash, or its recheck misses and duplicates it.
+        deferHashIndexSave: false,
         ...(factToolScoped ? { toolScoped: true as const } : {}),
-      });
+          }),
+      ).catch(finalizeOnWindowFailure);
+      if (factWriteOutcome.suppressed) {
+        // Codex P2: backfill bounds onto the window's winning copy first.
+        await backfillDuplicateTemporalBounds();
+        log.info(`dedup: raced duplicate suppressed in fact-dedup window for storage ${targetStorage.dir}`);
+        dedupedCount++;
+        continue;
+      }
+      const factWrite = factWriteOutcome.result;
       const memoryId = factWrite.id;
       // #1645: surface the tombstone block; gate active post-write paths like #1576
       // so a blocked fact creates no active shared copy / supersession / graph entry.
@@ -2978,9 +2991,7 @@ export class ExtractionPersistCoordinator {
       `persisted: ${facts.length - dedupedCount - importanceGatedCount - judgeGatedCount - redactionGatedCount - semanticMergedCount} facts${dedupSuffix}${gatedSuffix}${judgeSuffix}${redactionSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`,
     );
 
-    // #1635: surface pending_review ids so the thread episode set excludes them.
-    this.deps.setLastPersistExtractionPendingReviewIds(pendingReviewPersistedIds);
     // Return the persisted fact IDs for threading
-    return { persistedIds, memoryPathById };
+    return { persistedIds, memoryPathById, deferredCount, pendingReviewIds: pendingReviewPersistedIds };
   }
 }