@remnic/core 10.0.66 → 10.0.67

package/src/orchestration/extraction-run.ts

@@ -125,6 +125,7 @@ import {
 } from "./extraction-run-helpers.js";
 import type { ExtractionResilienceStatus, ExtractionRetryStateEntry } from "./extraction-run-helpers.js";
 import type { ExtractionPersistResult } from "./extraction-persist.js";
+import { ProviderCircuitBreaker, type ProbeOwnerToken } from "./provider-breaker.js";
 
 export {
   capExtractionRetryStateEntries,
@@ -150,12 +151,7 @@ export class ExtractionRunCoordinator {
   // backoff still throttles, so a restart can't resurrect the hot loop.
   private readonly extractionRetryState = new Map<string, Map<string, ExtractionRetryStateEntry>>();
   private readonly hydratedRetryNamespaces = new Set<string>();
-  private providerBreaker: {
-    consecutiveFailures: number;
-    openUntilMs: number;
-    state: "closed" | "open" | "half_open";
-    lastReason: string;
-  } = { consecutiveFailures: 0, openUntilMs: 0, state: "closed", lastReason: "" };
+  private readonly providerBreaker = new ProviderCircuitBreaker();
 
   constructor(private readonly deps: ExtractionRunCoordinatorDeps) {}
 
@@ -282,65 +278,6 @@ export class ExtractionRunCoordinator {
     this.extractionRetryState.set(namespace, nsMap);
   }
 
-  private logBreakerTransition(state: "closed" | "open" | "half_open"): void {
-    // Once per state change (not per attempt) — AGENTS.md observability rule.
-    log.warn("extraction: provider circuit breaker transition", {
-      state,
-      consecutiveFailures: this.providerBreaker.consecutiveFailures,
-      lastReason: this.providerBreaker.lastReason,
-      openUntilMs: this.providerBreaker.openUntilMs,
-    });
-  }
-
-  /** True while the breaker suppresses non-forced attempts. Flips an expired
-   *  `open` breaker to `half_open`, which allows exactly one probe (the next
-   *  non-forced attempt) because the extraction queue is a serial drain. */
-  private isProviderBreakerOpen(now: number): boolean {
-    if (now < this.providerBreaker.openUntilMs) return true;
-    if (this.providerBreaker.state === "open") {
-      this.providerBreaker.state = "half_open";
-      this.logBreakerTransition("half_open");
-    }
-    return false;
-  }
-
-  private onProviderFailure(cls: ExtractionFailureClass, now: number): void {
-    this.providerBreaker.consecutiveFailures += 1;
-    const trip =
-      cls === "auth_config" ||
-      // A failed half-open probe re-opens immediately (standard breaker
-      // semantics). Without this, an auth_config-opened breaker (which trips
-      // below the threshold) would get stuck half_open after a transient
-      // probe failure and stop suppressing (cursor review).
-      this.providerBreaker.state === "half_open" ||
-      this.providerBreaker.consecutiveFailures >= this.config.extractionBreakerFailureThreshold;
-    if (!trip) return;
-    const cooldown =
-      cls === "auth_config"
-        ? this.config.extractionBreakerAuthCooldownMs
-        : this.config.extractionBreakerCooldownMs;
-    this.providerBreaker.openUntilMs = now + Math.max(0, cooldown);
-    this.providerBreaker.lastReason = cls;
-    if (this.providerBreaker.state !== "open") {
-      this.providerBreaker.state = "open";
-      this.logBreakerTransition("open");
-    }
-  }
-
-  private resetProviderBreakerOnSuccess(): void {
-    const wasDisturbed =
-      this.providerBreaker.state !== "closed" ||
-      this.providerBreaker.consecutiveFailures > 0 ||
-      this.providerBreaker.openUntilMs > 0;
-    if (!wasDisturbed) return;
-    const wasOpen = this.providerBreaker.state !== "closed";
-    this.providerBreaker.consecutiveFailures = 0;
-    this.providerBreaker.openUntilMs = 0;
-    this.providerBreaker.lastReason = "";
-    this.providerBreaker.state = "closed";
-    if (wasOpen) this.logBreakerTransition("closed");
-  }
-
   /** Rebuild `meta.extractionRetryState` from the in-memory namespace mirror,
    *  bounded to the newest entries, and prune the mirror to match. */
   private persistRetryStateToMeta(namespace: string, meta: MetaState): void {
@@ -370,10 +307,11 @@ export class ExtractionRunCoordinator {
     cls: ExtractionFailureClass,
     meta: MetaState,
     now: number,
+    probeToken?: ProbeOwnerToken,
   ): Promise<void> {
     try {
       this.hydrateRetryStateFromMeta(namespace, meta);
-      this.applyExtractionFailureToMemory(namespace, fingerprint, cls, now);
+      this.applyExtractionFailureToMemory(namespace, fingerprint, cls, now, probeToken);
       this.persistRetryStateToMeta(namespace, meta);
       // The caller's storage.updateMeta persists this — saving here would write the same file twice.
     } catch (err) {
@@ -389,6 +327,7 @@ export class ExtractionRunCoordinator {
     fingerprint: string,
     cls: ExtractionFailureClass,
     now: number,
+    probeToken?: ProbeOwnerToken,
   ): void {
     const nsMap = this.extractionRetryState.get(namespace) ?? new Map<string, ExtractionRetryStateEntry>();
     const prev = nsMap.get(fingerprint);
@@ -409,7 +348,11 @@ export class ExtractionRunCoordinator {
     }
     nsMap.set(fingerprint, { attempts, nextEligibleAtMs, firstFailedAtMs, lastFailureClass: cls });
     this.extractionRetryState.set(namespace, nsMap);
-    this.onProviderFailure(cls, now);
+    this.providerBreaker.failure(cls, now, {
+      failureThreshold: this.config.extractionBreakerFailureThreshold,
+      cooldownMs: this.config.extractionBreakerCooldownMs,
+      authCooldownMs: this.config.extractionBreakerAuthCooldownMs,
+    }, probeToken);
   }
 
   /** Delete a fingerprint's retry entry from the mirror + meta. Returns whether
@@ -430,12 +373,7 @@ export class ExtractionRunCoordinator {
       backoffFingerprintCount += nsMap.size;
     }
     return {
-      breaker: {
-        state: this.providerBreaker.state,
-        openUntilMs: this.providerBreaker.openUntilMs,
-        consecutiveFailures: this.providerBreaker.consecutiveFailures,
-        lastReason: this.providerBreaker.lastReason,
-      },
+      breaker: this.providerBreaker.snapshot(),
       backoffFingerprintCount,
     };
   }
@@ -834,13 +772,18 @@ export class ExtractionRunCoordinator {
     // Resolve the gate through the shared lifecycle capability plan (issue
     // #1523) rather than reading the raw config flag at each call site.
     const extractionRetryEnabled = resolveMemoryLifecycleCapabilities(this.config).extractionRetry;
+    let extractionProbeOwned = false;
+    let extractionProbeToken: ProbeOwnerToken | undefined;
     if (extractionFingerprint && !forcedExtractionAttempt && extractionRetryEnabled) {
       try {
         meta ??= await runDeadlineAware(() => storage.loadMeta(), "during_retry_gate");
         this.hydrateRetryStateFromMeta(selfNamespace, meta);
         const nowMs = Date.now();
+        const probeAttempt = this.providerBreaker.acquire(nowMs);
+        extractionProbeOwned = probeAttempt.probe;
+        extractionProbeToken = probeAttempt.token;
         let suppressReason: "provider_circuit_open" | "extraction_backoff" | null = null;
-        if (this.isProviderBreakerOpen(nowMs)) {
+        if (probeAttempt.suppressed) {
           suppressReason = "provider_circuit_open";
         } else {
           const st = this.extractionRetryState.get(selfNamespace)?.get(extractionFingerprint);
@@ -865,6 +808,10 @@ export class ExtractionRunCoordinator {
             log.warn("runExtraction: passive correction capture failed on suppressed attempt (non-fatal)", captureErr);
           }
           clearExtractionDeadlineTimer();
+          // Backoff suppression exits inside the gate, before the wrapper
+          // finally: release an acquired probe here (codex follow-up) so a
+          // cooldown-expired fingerprint cannot pin the breaker half_open.
+          if (extractionProbeOwned) this.providerBreaker.releaseProbe(extractionProbeToken);
           return { status: "skipped", reason: suppressReason, persistedCount: 0, durableOutputCount: 0 };
         }
       } catch (err) {
@@ -873,26 +820,28 @@ export class ExtractionRunCoordinator {
         log.warn("runExtraction: extraction retry gate check failed; proceeding (fail-open)", err);
       }
     }
-
     // Pass existing entity names so the LLM can reuse them instead of inventing variants.
-    const existingEntities = await runDeadlineAware(
-      () => storage.listEntityNames(),
-      "during_list_entity_names",
-    );
-    let result: ExtractionResult;
+    // The probe owner must release on ANY pre-persistence exit too (codex P1):
+    // the big try below only covers from the extract call onward.
     try {
-      result = await raceRecallAbort(
-        this.deps.getExtraction().extract(normalizedTurns, existingEntities, extractionAbortSignal),
-        extractionAbortSignal,
-        "extraction aborted (during_extract)",
+      const existingEntities = await runDeadlineAware(
+        () => storage.listEntityNames(),
+        "during_list_entity_names",
       );
-    } catch (error) {
-      clearExtractionDeadlineTimer();
-      if (typeof deadlineMs === "number" && Date.now() >= deadlineMs) {
-        throw new ExtractionDeadlineError("during_extract");
+      let result: ExtractionResult;
+      try {
+        result = await raceRecallAbort(
+          this.deps.getExtraction().extract(normalizedTurns, existingEntities, extractionAbortSignal),
+          extractionAbortSignal,
+          "extraction aborted (during_extract)",
+        );
+      } catch (error) {
+        clearExtractionDeadlineTimer();
+        if (deadlineMs !== undefined && Date.now() >= deadlineMs) {
+          throw new ExtractionDeadlineError("during_extract");
+        }
+        throw error;
       }
-      throw error;
-    }
     try {
       throwIfDeadlineExceeded("before_persist");
       throwIfAborted("before_persist");
@@ -965,7 +914,7 @@ export class ExtractionRunCoordinator {
           await runDeadlineAware(
             () =>
               storage.updateMeta((freshMeta) =>
-                this.recordExtractionFailure(selfNamespace, extractionFingerprint, failureClass, freshMeta, Date.now()),
+                this.recordExtractionFailure(selfNamespace, extractionFingerprint, failureClass, freshMeta, Date.now(), extractionProbeToken),
               ),
             "during_retry_bookkeeping_save",
             false,
@@ -976,14 +925,16 @@ export class ExtractionRunCoordinator {
           // Fail-open, but retain the turns (a cleared buffer with no retry state drops them). Only a
           // LOCK skip reaches here without the mutator having run — a later save must not double-count.
           if (err instanceof Error && err.message.includes("meta.json.lock not acquired"))
-            this.applyExtractionFailureToMemory(selfNamespace, extractionFingerprint, failureClass, Date.now());
+            this.applyExtractionFailureToMemory(selfNamespace, extractionFingerprint, failureClass, Date.now(), extractionProbeToken);
           recordedRetryFailure = true;
           log.warn("runExtraction: retry state kept in memory; meta persist will catch up", err);
         }
       } else if (result.extractionSkippedReason === undefined) {
         // Provider responded without failure → breaker heals; clear any
         // parked backoff for this fingerprint so it proceeds normally.
-        this.resetProviderBreakerOnSuccess();
+        // PRRT_kwDORJXyws6zxG6u: pass the run's probe token so the settlement
+        // only applies to a probe this run owns.
+        this.providerBreaker.success(extractionProbeToken);
         try {
           await runDeadlineAware(
             () =>
@@ -1196,5 +1147,11 @@ export class ExtractionRunCoordinator {
     } finally {
       clearExtractionDeadlineTimer();
     }
+    } finally {
+      // The half-open probe is OWNED, not timed: the WRAPPER finally releases
+      // it on every exit path, including pre-persistence ones (codex P1) that
+      // never reach the inner try above.
+      if (extractionProbeOwned) this.providerBreaker.releaseProbe(extractionProbeToken);
+    }
   }
 }

package/src/orchestration/fact-dedup-window.ts

@@ -0,0 +1,122 @@
+import path from "node:path";
+import { serializeMutations, withHeldFileLock } from "../utils/serialize-mutations.js";
+import { ContentHashIndex, type StorageManager } from "../index.js";
+import type { MemoryFile } from "../types.js";
+import { normalizeStoredHashSource } from "./extraction-persist-citation.js";
+import { stripAttributesSuffix } from "../structured-attributes.js";
+import { log } from "../logger.js";
+
+/**
+ * Fact-dedup commit window (codex P2, #1526): with extractionQueueConcurrency
+ * > 1 (or daemon + external drain workers), two sessions can both pass the
+ * content-hash check at persist time and both write, producing duplicate
+ * active copies — the persist-time checks run well before the write, and
+ * writeMemory mints a fresh id per write. This module serializes the short
+ * "final recheck → write" window per storage dir (in-process chain +
+ * fail-closed cross-process fact-dedup.lock, mirroring the profile/meta lock
+ * shape: bounded, no LLM or corpus-scale work planned inside, throw BEFORE
+ * any read/write when the lock is not acquired). The persist-time check stays
+ * OUTSIDE the window exactly like the consolidation LLM call stays outside
+ * commitConsolidatedProfile.
+ */
+export interface FactDedupProbe {
+  hasContentHashDedup(storage: StorageManager, key: string): Promise<boolean>;
+}
+
+interface FactDedupMatchOptions {
+  readonly category: string;
+  readonly contentHashDedupKey: string;
+  readonly connector?: string;
+  readonly citationEnabled: boolean;
+  readonly citationTemplate: string;
+}
+
+function matchesActiveConnectorDuplicate(memory: MemoryFile, opts: FactDedupMatchOptions): boolean {
+  if (memory.frontmatter.category !== opts.category) return false;
+  if ((memory.frontmatter.status ?? "active") !== "active") return false;
+  // Thread 5 (QPDE5): for procedures the hash is keyed on the full
+  // body (title + steps via buildProcedurePersistBody), so compare
+  // against contentHashDedupKey — not the canonical title-only form.
+  // Thread 3 (QO42V): facts with structuredAttributes get an appended
+  // [Attributes: ...] suffix in the stored body that the hash key lacks;
+  // strip it before comparing so same-connector enriched facts dedup
+  // correctly.
+  if (normalizeStoredHashSource(stripAttributesSuffix(memory.content ?? ""), opts.citationEnabled, opts.citationTemplate) !==
+    ContentHashIndex.normalizeContent(opts.contentHashDedupKey)) return false;
+  return (memory.frontmatter.sourceConnector?.trim() || undefined) === opts.connector;
+}
+
+/**
+ * Connector-aware duplicate confirm: same final semantics as the persist-time
+ * check — an authoritative index miss is a clean NOT-duplicate, a hash hit or
+ * a non-authoritative/failed lookup must be confirmed against the hot+cold
+ * corpus (same category, active, same normalized content, same connector),
+ * and a corpus-scan failure fails OPEN so an unverifiable state cannot
+ * silently drop content.
+ */
+export async function confirmActiveFactDuplicate(
+  probe: FactDedupProbe,
+  storage: StorageManager,
+  opts: FactDedupMatchOptions,
+): Promise<boolean> {
+  let needsCorpusConfirm: boolean;
+  try {
+    if (!(await probe.hasContentHashDedup(storage, opts.contentHashDedupKey))) {
+      if (await storage.isFactContentHashAuthoritative()) return false;
+      needsCorpusConfirm = true;
+    } else {
+      needsCorpusConfirm = true;
+    }
+  } catch {
+    // Dedup lookup failed (e.g. the authoritative rebuild could not run). Do
+    // NOT fail-open into a possible duplicate — confirm against the corpus.
+    needsCorpusConfirm = true;
+  }
+  try {
+    // #2016 cold-tier finding: the authoritative content-hash rebuild unions
+    // the HOT and COLD tiers, so a cold-only active copy must still confirm
+    // the duplicate and suppress the redundant hot write.
+    const [hotMems, coldMems] = await Promise.all([
+      storage.readAllMemories(),
+      storage.readAllColdMemories(),
+    ]);
+    return hotMems.some((m) => matchesActiveConnectorDuplicate(m, opts)) ||
+      coldMems.some((m) => matchesActiveConnectorDuplicate(m, opts));
+  } catch (err) {
+    log.warn(
+      `connector-aware dedup scan failed for storage ${storage.dir}; writing fail-open: ${err instanceof Error ? err.message : String(err)}`,
+    );
+    return false;
+  }
+}
+
+export type FactDedupWindowOutcome<T> = { suppressed: true } | { suppressed: false; result: T };
+
+/**
+ * Run `write` inside the per-dir fact-dedup window: recheck `recheck` (the
+ * SAME confirm used at persist time, now under the lock) and suppress as a
+ * duplicate when a concurrent session committed the same content first;
+ * otherwise write. Lock contention throws (fail closed) — never an unlocked
+ * write; the extraction retry path retains the accepted turns.
+ */
+export async function persistUnderFactDedupWindow<T>(
+  storageDir: string,
+  recheck: () => Promise<boolean>,
+  write: () => Promise<T>,
+): Promise<FactDedupWindowOutcome<T>> {
+  return serializeMutations(`fact-dedup:${storageDir}`, () =>
+    withHeldFileLock(
+      path.join(storageDir, "fact-dedup.lock"),
+      // Bounded: the window holds one corpus confirm + one memory write, not a
+      // scan-length lock (catalog round 5 adoption note).
+      { staleMs: 5_000, maxWaitMs: 2_000, pollMs: 50 },
+      async (acquired) => {
+        if (!acquired) {
+          throw new Error(`fact write skipped: fact-dedup.lock not acquired (${storageDir})`);
+        }
+        if (await recheck()) return { suppressed: true };
+        return { suppressed: false, result: await write() };
+      },
+    ),
+  );
+}

package/src/orchestration/provider-breaker.test.ts

@@ -0,0 +1,89 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { ProviderCircuitBreaker } from "./provider-breaker.js";
+
+const CONFIG = { failureThreshold: 3, cooldownMs: 5_000, authCooldownMs: 60_000 };
+
+test("half-open probe ownership is single-flight under parallel callers", () => {
+  const breaker = new ProviderCircuitBreaker();
+  breaker.failure("provider_retryable", 1_000, CONFIG);
+  breaker.failure("provider_retryable", 1_100, CONFIG);
+  breaker.failure("provider_retryable", 1_200, CONFIG); // threshold → open
+  assert.deepEqual(breaker.acquire(1_300).suppressed, true, "open suppresses during cooldown");
+
+  // Cooldown expires: the FIRST caller takes the probe…
+  const probe = breaker.acquire(6_200);
+  assert.equal(probe.probe, true, "expired breaker hands one probe out");
+  // …siblings stay suppressed for as long as the probe is held — no wall-clock
+  // expiry, however long the provider attempt runs (timeouts + retries).
+  assert.deepEqual(breaker.acquire(600_000).suppressed, true, "a live probe keeps suppressing");
+  assert.deepEqual(breaker.acquire(3_600_000).suppressed, true, "no timer expiry while held");
+
+  // Probe attempt dies WITHOUT a terminal outcome (deadline/abort): explicit
+  // release lets the next attempt re-probe.
+  breaker.releaseProbe(probe.token);
+  const reprobe = breaker.acquire(3_600_001);
+  assert.equal(reprobe.probe, true, "released probe is re-acquired");
+  assert.deepEqual(breaker.acquire(3_600_002).suppressed, true, "re-probe is also single-flight");
+
+  // Terminal outcomes release ownership: the OWNER's failed probe re-opens
+  // immediately (cooldown to 6_500 + 5_000).
+  breaker.failure("provider_retryable", 6_500, CONFIG, reprobe.token);
+  assert.deepEqual(breaker.acquire(7_000).suppressed, true, "failed probe re-opens at once");
+  assert.deepEqual(breaker.acquire(12_000).suppressed, false, "cooldown expiry admits a fresh probe");
+  breaker.success();
+  assert.deepEqual(breaker.acquire(12_001).probe, false, "successful probe closes the breaker");
+});
+
+test("a forced attempt's failure never releases another attempt's live probe", () => {
+  const breaker = new ProviderCircuitBreaker();
+  breaker.failure("provider_retryable", 1_000, CONFIG);
+  breaker.failure("provider_retryable", 1_100, CONFIG);
+  breaker.failure("provider_retryable", 1_200, CONFIG); // threshold → open
+  const probe = breaker.acquire(6_200);
+  assert.equal(probe.probe, true, "ordinary attempt owns the probe");
+
+  // A FORCED attempt bypassed acquire, fails while the probe is in flight:
+  // it must not release the owner or masquerade as the probe result.
+  breaker.failure("provider_retryable", 6_300, CONFIG /* no token */);
+  assert.deepEqual(breaker.acquire(6_400), { suppressed: true, probe: false }, "probe stays single-flight");
+
+  // Only the owner's failure settles it: re-open at once.
+  breaker.failure("provider_retryable", 6_500, CONFIG, probe.token);
+  assert.deepEqual(breaker.acquire(6_600), { suppressed: true, probe: false }, "failed probe re-opens immediately");
+
+  // A released probe is re-acquired only via its own token path.
+  const fresh = breaker.acquire(12_000);
+  assert.equal(fresh.probe, true);
+  breaker.releaseProbe(fresh.token);
+  const reprobe = breaker.acquire(12_001);
+  assert.equal(reprobe.probe, true, "released probe is re-acquired by the next attempt");
+  // A non-owner release with a foreign token is a no-op.
+  breaker.releaseProbe({ __probeOwner: true });
+  assert.deepEqual(breaker.acquire(12_002), { suppressed: true, probe: false });
+});
+
+test("an unowned forced outcome neither trips the half-open threshold nor settles a held probe", () => {
+  const breaker = new ProviderCircuitBreaker();
+  breaker.failure("provider_retryable", 1_000, CONFIG);
+  breaker.failure("provider_retryable", 1_100, CONFIG);
+  breaker.failure("provider_retryable", 1_200, CONFIG); // threshold → open
+  const probe = breaker.acquire(6_200);
+  assert.equal(probe.probe, true, "ordinary attempt owns the probe");
+
+  // Forced failures bypass acquire; even crossing the threshold they must
+  // not reopen over a held probe (PRRT_kwDORJXyws6zxG6V).
+  breaker.failure("provider_retryable", 6_250, CONFIG /* no token */);
+  breaker.failure("provider_retryable", 6_300, CONFIG /* no token */);
+  breaker.failure("provider_retryable", 6_350, CONFIG /* no token */);
+  assert.deepEqual(breaker.acquire(6_400), { suppressed: true, probe: false }, "probe stays single-flight");
+
+  // A forced success without the owner token must not settle the probe or
+  // close the breaker mid-probe (PRRT_kwDORJXyws6zxG6u).
+  breaker.success();
+  assert.deepEqual(breaker.acquire(6_450), { suppressed: true, probe: false }, "probe survives an unowned success");
+
+  // The owner's success settles it for real.
+  breaker.success(probe.token);
+  assert.deepEqual(breaker.acquire(6_500).suppressed, false, "owner success closes the breaker");
+});

package/src/orchestration/provider-breaker.ts

@@ -0,0 +1,158 @@
+import { log } from "../logger.js";
+import type { ExtractionFailureClass } from "../types.js";
+
+export interface ProviderBreakerConfig {
+  readonly failureThreshold: number;
+  readonly cooldownMs: number;
+  readonly authCooldownMs: number;
+}
+
+export interface ProbeOwnerToken {
+  readonly __probeOwner: true;
+}
+
+export interface ProviderBreakerSnapshot {
+  readonly state: "closed" | "open" | "half_open";
+  readonly openUntilMs: number;
+  readonly consecutiveFailures: number;
+  readonly lastReason: string;
+}
+
+export interface ProbeAcquisition {
+  /** True when the attempt is suppressed (breaker open, or the half-open
+   *  probe is owned by another attempt). */
+  readonly suppressed: boolean;
+  /** True when THIS attempt owns the single half-open probe and must
+   *  terminate it via {@link failure}/{@link success} (with its token), or
+   *  release it via {@link releaseProbe} on any early exit (deadline, abort,
+   *  throw). */
+  readonly probe: boolean;
+  /** Opaque ownership token; only the matching token settles or releases the
+   *  probe, so attempts that bypass the gate (forced extractions) can never
+   *  release another attempt's in-flight probe. */
+  readonly token?: ProbeOwnerToken;
+}
+
+/**
+ * Provider circuit breaker with single-flight half-open probe OWNERSHIP
+ * (codex P2 on #3311): under extractionQueueConcurrency > 1 several sessions
+ * can pass an expired `open` breaker in the same window, and a wall-clock
+ * reservation either expires under a live long provider attempt (600s
+ * timeouts) or pins the breaker when an attempt dies without settling. So the
+ * probe is OWNED, not timed: exactly one attempt holds it from the
+ * open→half_open flip until its terminal `failure()`/`success()`, and any
+ * early exit releases it explicitly. A released-but-unfinished probe lets the
+ * next attempt re-probe; a retry backoff still throttles that namespace.
+ */
+export class ProviderCircuitBreaker {
+  private consecutiveFailures = 0;
+  private openUntilMs = 0;
+  private state: ProviderBreakerSnapshot["state"] = "closed";
+  private lastReason = "";
+  private probeHeld = false;
+  private probeToken: ProbeOwnerToken | null = null;
+
+  acquire(now: number): ProbeAcquisition {
+    if (now < this.openUntilMs) return { suppressed: true, probe: false };
+    if (this.state === "open") {
+      this.state = "half_open";
+      this.probeHeld = true;
+      const token: ProbeOwnerToken = { __probeOwner: true };
+      this.probeToken = token;
+      this.logTransition();
+      return { suppressed: false, probe: true, token }; // THIS attempt is the probe.
+    }
+    if (this.state === "half_open") {
+      if (this.probeHeld) return { suppressed: true, probe: false };
+      this.probeHeld = true; // previous owner released without settling; re-probe.
+      const token: ProbeOwnerToken = { __probeOwner: true };
+      this.probeToken = token;
+      return { suppressed: false, probe: true, token };
+    }
+    return { suppressed: false, probe: false };
+  }
+
+  /** Release an owned probe on an attempt that exits without a terminal
+   *  outcome (deadline, abort, throw). Only the matching owner releases;
+   *  idempotent; state stays half_open so the next attempt re-probes. */
+  releaseProbe(token?: ProbeOwnerToken): void {
+    if (token === undefined || this.probeToken !== token) return;
+    this.probeToken = null;
+    this.probeHeld = false;
+  }
+
+  failure(cls: ExtractionFailureClass, now: number, config: ProviderBreakerConfig, probeToken?: ProbeOwnerToken): void {
+    // Only the probe OWNER's failure settles the half-open probe: a failure
+    // from an attempt that bypassed acquire (forced extraction) must not
+    // release another attempt's in-flight probe or masquerade as its result.
+    const owned = probeToken !== undefined && this.probeToken === probeToken;
+    if (owned) {
+      this.probeToken = null;
+      this.probeHeld = false;
+    }
+    this.consecutiveFailures += 1;
+    const trip =
+      // An unowned auth_config failure (forced extraction) must not reopen
+      // over a held half-open probe — same ownership rule as the threshold
+      // term; the auth cooldown still applies once the probe settles.
+      (cls === "auth_config" && (this.state !== "half_open" || owned)) ||
+      // A failed half-open probe re-opens immediately (standard breaker
+      // semantics). Without this, an auth_config-opened breaker (which trips
+      // below the threshold) would get stuck half_open after a transient
+      // probe failure and stop suppressing (cursor review).
+      (owned && this.state === "half_open") ||
+      // PRRT_kwDORJXyws6zxG6V: an unowned (forced-extraction) failure must
+      // not cross the threshold while another attempt owns the half-open
+      // probe — reopening here would strand the owner's token and let the
+      // next cooldown mint a second concurrent probe. The failure count and
+      // retry bookkeeping still accumulate for when the state leaves
+      // half_open.
+      (this.state !== "half_open" && this.consecutiveFailures >= config.failureThreshold);
+    if (!trip) return;
+    const cooldown = cls === "auth_config" ? config.authCooldownMs : config.cooldownMs;
+    this.openUntilMs = now + Math.max(0, cooldown);
+    this.lastReason = cls;
+    if (this.state !== "open") {
+      this.state = "open";
+      this.logTransition();
+    }
+  }
+
+  success(probeToken?: ProbeOwnerToken): void {
+    const wasDisturbed =
+      this.state !== "closed" || this.consecutiveFailures > 0 || this.openUntilMs > 0;
+    if (!wasDisturbed) return;
+    // PRRT_kwDORJXyws6zxG6u: while a half-open probe is held, only its OWNER
+    // may settle it. An unowned (forced-extraction) success must not clear
+    // another run's probe or close the breaker mid-probe — the next ordinary
+    // acquisition could then start a second concurrent probe.
+    if (this.probeHeld && !(probeToken !== undefined && this.probeToken === probeToken)) return;
+    const wasOpen = this.state !== "closed";
+    this.probeToken = null;
+    this.probeHeld = false;
+    this.consecutiveFailures = 0;
+    this.openUntilMs = 0;
+    this.lastReason = "";
+    this.state = "closed";
+    if (wasOpen) this.logTransition();
+  }
+
+  snapshot(): ProviderBreakerSnapshot {
+    return {
+      state: this.state,
+      openUntilMs: this.openUntilMs,
+      consecutiveFailures: this.consecutiveFailures,
+      lastReason: this.lastReason,
+    };
+  }
+
+  private logTransition(): void {
+    // Once per state change (not per attempt) — AGENTS.md observability rule.
+    log.warn("extraction: provider circuit breaker transition", {
+      state: this.state,
+      consecutiveFailures: this.consecutiveFailures,
+      lastReason: this.lastReason,
+      openUntilMs: this.openUntilMs,
+    });
+  }
+}

package/src/orchestration/turn-ingestion.ts

@@ -932,8 +932,8 @@ export class TurnIngestionCoordinator {
     }
     if (settled) return;
     const preemptable = options.preemptable === true && options.preemptRunning !== true;
-    const accepted = this.deps.extractionQueueCoordinator.enqueue(async () => {
-      await this.deps.extractionQueueCoordinator.runPreemptable(preemptable, options.abortSignal, async (abortSignal) => {
+    const accepted = this.deps.extractionQueueCoordinator.enqueue(async (preemptHandle) => {
+      await this.deps.extractionQueueCoordinator.runPreemptable(preemptHandle, preemptable, options.abortSignal, async (abortSignal) => {
       if (settled) return;
       if (
         typeof extractionDeadlineMs === "number" &&
@@ -1082,14 +1082,14 @@ export class TurnIngestionCoordinator {
           deadlineMs: extractionDeadlineMs,
           bufferKey,
           onDurableCommit: options.onDurableCommit,
-          onBeforePersist: () => this.deps.extractionQueueCoordinator.endPreemptable(),
+          onBeforePersist: () => preemptHandle?.disarm(),
           abortSignal,
           failOnExtractionFailure: options.failOnExtractionFailure === true,
           writeNamespaceOverride: options.writeNamespaceOverride,
           principalOverride: options.principalOverride,
           scopeProfileWritePlan: effectiveScopePlan,
           forceExtractionAttempt: options.forceExtractionAttempt === true,
-        }, preemptable);
+        }, preemptable, preemptHandle);
         settleTask(undefined, result);
       } catch (err) {
         const reason = abortSignal.reason;
@@ -1099,7 +1099,7 @@ export class TurnIngestionCoordinator {
         }
       }
       });
-    }, { front: options.preemptRunning === true });
+    }, { front: options.preemptRunning === true, key: bufferKey });
     if (!accepted) {
       settleTask(new Error("extraction queue is stopped"));
       return;