@remnic/core 10.0.66 → 10.0.68

package/src/orchestration/extraction-queue-coordinator.ts

@@ -1,13 +1,15 @@
 /**
  * Extraction queue coordinator — extracted from the orchestrator (issue #1526).
  *
- * Owns the background serial queue that drains extraction tasks one at a
- * time to avoid races between concurrent flush/heartbeat/bulk-import
+ * Owns the background bounded-parallel queue that drains extraction tasks
+ * while avoiding races between concurrent flush/heartbeat/bulk-import
  * triggers:
- *   - queue state (`extractionQueue` + `queueProcessing`)
+ *   - queue state (`queue` + `processing` + per-task preempt slots)
  *   - the scheduling trigger (start the drain when the first task lands and
- *     the processor is idle)
- *   - the serial drain loop itself (`processQueue`)
+ *     capacity is free)
+ *   - the drain loop itself (`processQueue`): up to `concurrency` tasks run
+ *     at once; tasks sharing an entry `key` (a buffer/session) never run
+ *     concurrently, so the same buffer is never extracted twice in parallel
  *   - failure classification (`logExtractionQueueFailure`) — issue #549:
  *     AbortError from session transitions logs at debug, real failures at
  *     error
@@ -19,51 +21,102 @@
  * orchestrator builds each task closure and hands it to `enqueue`; the
  * coordinator owns only the queue mechanics.
  *
- * Behavior-preserving move from orchestrator.ts. No logic changes — the
- * orchestrator keeps a thin delegating `waitForExtractionIdle` and the
- * `queueBufferedExtraction` builder delegates push+scheduling here.
+ * Preemption is per task: each running task receives an
+ * `ExtractionPreemptHandle` at run time and registers its own abort, so two
+ * concurrent automatic extractions cannot clobber each other's registration
+ * (the previous single-slot `runningPreempt` could only hold one).
+ * `preemptRunning` aborts at most ONE armed task — the requested key's own
+ * running work when it is armed, else the most recently armed task — which
+ * is exactly the single-slot semantics generalized from one worker to N.
  */
 
 import { isAbortError, preemptedExtractionError } from "../abort-error.js";
 import { log } from "../logger.js";
 
+/** Preemption registration for ONE running task. Armed work can be aborted
+ *  by a fail-closed flush; disarmed work (past persist) cannot. */
+export interface ExtractionPreemptHandle {
+  /** Register the task's abort (replacing any previous registration). */
+  arm(abort: () => void): void;
+  /** Remove the task's registration (idempotent). */
+  disarm(): void;
+}
+
+/** A queued task; the coordinator passes the task's own preempt handle. */
+export type ExtractionQueueTask = (preempt?: ExtractionPreemptHandle) => Promise<void>;
+
+export interface ExtractionQueueEnqueueOptions {
+  /** Jump the queue and preempt the running automatic work (force flush). */
+  front?: boolean;
+  /** Buffer/session key. Tasks sharing a key never run concurrently. */
+  key?: string;
+}
+
+interface QueueEntry {
+  run: ExtractionQueueTask;
+  key?: string;
+}
+
+interface PreemptSlot {
+  key?: string;
+  armed?: { abort: () => void; seq: number };
+}
+
 /**
- * Coordinates the background serial extraction queue. Owns the in-flight
- * guard + queue array that previously lived as private orchestrator fields.
+ * Coordinates the background bounded-parallel extraction queue. Owns the
+ * queue array, the per-key in-flight guard, and the per-task preempt slots.
  */
 export class ExtractionQueueCoordinator {
+  /** Maximum tasks that run at the same time (config `extractionQueueConcurrency`). */
+  private readonly concurrency: number;
   /**
-   * Background serial queue for extractions (agent_end optimization).
-   * Queue stores tasks that resolve when extraction should run.
+   * Background queue for extractions (agent_end optimization). Queue stores
+   * tasks that resolve when extraction should run.
    */
-  private readonly queue: Array<() => Promise<void>> = [];
-  /** Whether the serial drain loop is currently running. */
+  private readonly queue: QueueEntry[] = [];
+  /** Whether the drain loop is currently running. */
   private processing = false;
   private accepting = true;
-  private runningPreempt: (() => void) | undefined;
+  /** Buffer keys with a task currently running (per-key exclusion). */
+  private readonly activeKeys = new Set<string>();
+  /** Preempt slots of the currently running tasks, keyed by slot identity. */
+  private readonly preemptSlots = new Map<object, PreemptSlot>();
+  private preemptSeq = 0;
+  /** Resolves the processor's park while a new task arrives mid-drain. */
+  private wake?: () => void;
+
+  constructor(concurrency = 1) {
+    this.concurrency = Number.isInteger(concurrency) && concurrency >= 1 ? concurrency : 1;
+  }
 
   /** Current queue depth (idle-wait + test seam). */
   get length(): number {
     return this.queue.length;
   }
 
-  /** Whether the serial drain loop is currently running (idle-wait + test seam). */
+  /** Whether the drain loop is currently running (idle-wait + test seam). */
   get isProcessing(): boolean {
     return this.processing;
   }
 
+  /** How many tasks may run at once (test seam + diagnostics). */
+  get maxConcurrency(): number {
+    return this.concurrency;
+  }
+
   /**
-   * Enqueue a task and start the serial processor if it is idle. This is
+   * Enqueue a task and start the processor when capacity is free. This is
    * the production entry point — the orchestrator's `queueBufferedExtraction`
-   * builds each task closure and hands it here.
+   * builds each task closure and hands it here with its buffer key.
    */
-  enqueue(task: () => Promise<void>, options?: { front?: boolean }): boolean {
+  enqueue(task: ExtractionQueueTask, options?: ExtractionQueueEnqueueOptions): boolean {
     if (!this.accepting) return false;
+    const entry: QueueEntry = { run: task, key: options?.key };
     if (options?.front) {
-      this.preemptRunning();
-      this.queue.unshift(task);
+      this.preemptRunning(options.key);
+      this.queue.unshift(entry);
     } else {
-      this.queue.push(task);
+      this.queue.push(entry);
     }
     if (!this.processing) {
       this.processing = true;
@@ -71,38 +124,43 @@ export class ExtractionQueueCoordinator {
         this.logExtractionQueueFailure(err, "processor");
         this.processing = false;
       });
+    } else {
+      // The processor may be parked waiting for an in-flight task while a
+      // slot is free; wake it so the new task starts instead of waiting
+      // behind work it could overlap with.
+      this.wake?.();
     }
     return true;
   }
 
-  /** Automatic extraction registers this so a fail-closed flush can abort it. */
-  beginPreemptable(abort: () => void): void {
-    this.runningPreempt = abort;
-  }
-
-  endPreemptable(): void {
-    this.runningPreempt = undefined;
-  }
-
-  preemptRunning(): void {
-    const abort = this.runningPreempt;
-    this.runningPreempt = undefined;
+  preemptRunning(preferredKey?: string): void {
+    const armed = Array.from(this.preemptSlots.values()).filter((slot) => slot.armed !== undefined);
+    if (armed.length === 0) return;
+    const preferred = preferredKey !== undefined ? armed.find((slot) => slot.key === preferredKey) : undefined;
+    const chosen =
+      preferred ??
+      armed.reduce((a, b) => ((b.armed?.seq ?? -1) > (a.armed?.seq ?? -1) ? b : a));
+    const abort = chosen.armed?.abort;
+    chosen.armed = undefined;
     abort?.();
   }
+
   runPreemptable<T>(
+    preempt: ExtractionPreemptHandle | undefined,
     preemptable: boolean,
     parent: AbortSignal | undefined,
     run: (signal: AbortSignal) => Promise<T>,
   ): Promise<T> {
-    const preempt = new AbortController();
-    if (preemptable) this.beginPreemptable(() => preempt.abort(preemptedExtractionError()));
-    const signal = parent ? AbortSignal.any([parent, preempt.signal]) : preempt.signal;
+    const preemptController = new AbortController();
+    if (preemptable && preempt) {
+      preempt.arm(() => preemptController.abort(preemptedExtractionError()));
+    }
+    const signal = parent ? AbortSignal.any([parent, preemptController.signal]) : preemptController.signal;
     return Promise.resolve(run(signal)).finally(() => {
-      if (preemptable) this.endPreemptable();
+      if (preemptable && preempt) preempt.disarm();
     });
   }
 
-
   /** Stop new producers from entering the queue. */
   stopAccepting(): void {
     this.accepting = false;
@@ -123,30 +181,82 @@ export class ExtractionQueueCoordinator {
   }
 
   /**
-   * Background serial queue processor.
-   * Processes extractions one at a time to avoid race conditions.
-   * Called automatically when items are queued via `enqueue`; also exposed
-   * for the characterization tests that push raw tasks and assert drain
-   * semantics directly.
+   * Background queue processor. Runs up to `concurrency` tasks at a time,
+   * never two tasks with the same entry key. Called automatically when items
+   * are queued via `enqueue`; also exposed for the characterization tests
+   * that push raw tasks and assert drain semantics directly.
    */
   async processQueue(): Promise<void> {
-    while (this.queue.length > 0) {
-      const task = this.queue.shift();
-      if (task) {
+    const inFlight = new Set<Promise<void>>();
+    try {
+      while (this.queue.length > 0 || inFlight.size > 0) {
+        while (inFlight.size < this.concurrency) {
+          const entry = this.takeRunnable();
+          if (!entry) break;
+          const running = this.runEntry(entry).finally(() => {
+            inFlight.delete(running);
+          });
+          inFlight.add(running);
+        }
+        // Park until a task settles OR a new task is enqueued. A queued task
+        // whose key is still held always implies an in-flight task holding
+        // it, so inFlight is non-empty here whenever the loop must wait.
+        if (inFlight.size === 0) break;
+        const { promise: wakePromise, resolve: wake } = Promise.withResolvers<void>();
+        this.wake = wake;
         try {
-          await task();
-        } catch (err) {
-          this.logExtractionQueueFailure(err, "task");
+          await Promise.race([wakePromise, ...inFlight]);
+        } finally {
+          this.wake = undefined;
         }
       }
+    } finally {
+      this.processing = false;
     }
+  }
+
+  /** Take the next queue entry whose key is not already running (FIFO within a key). */
+  private takeRunnable(): QueueEntry | undefined {
+    for (let index = 0; index < this.queue.length; index += 1) {
+      const entry = this.queue[index];
+      if (entry.key !== undefined && this.activeKeys.has(entry.key)) continue;
+      this.queue.splice(index, 1);
+      if (entry.key !== undefined) this.activeKeys.add(entry.key);
+      return entry;
+    }
+    return undefined;
+  }
+
+  /** Run one entry with its own preempt slot; failures are classified + logged. */
+  private async runEntry(entry: QueueEntry): Promise<void> {
+    const slot: PreemptSlot = {};
+    this.preemptSlots.set(slot, { key: entry.key });
+    try {
+      await entry.run(this.createPreemptHandle(slot));
+    } catch (err) {
+      this.logExtractionQueueFailure(err, "task");
+    } finally {
+      this.preemptSlots.delete(slot);
+      if (entry.key !== undefined) this.activeKeys.delete(entry.key);
+    }
+  }
 
-    this.processing = false;
+  private createPreemptHandle(slot: PreemptSlot): ExtractionPreemptHandle {
+    return {
+      arm: (abort: () => void) => {
+        const state = this.preemptSlots.get(slot);
+        if (state) state.armed = { abort, seq: this.preemptSeq++ };
+      },
+      disarm: () => {
+        const state = this.preemptSlots.get(slot);
+        if (state) state.armed = undefined;
+      },
+    };
   }
 
   /**
    * Classify + log a failure from either the per-task catch inside
-   * `processQueue()` or the outer `processQueue().catch(...)` in
+   * `runEntry` or the outer `processQueue().catch(...)` in
    * `enqueue()`.  Issue #549: `throwIfRecallAborted`
    * (used throughout `runExtraction`) raises an Error whose `name` is
    * `"AbortError"`.  That path fires when `before_reset` aborts a
@@ -211,13 +321,14 @@ export class ExtractionQueueCoordinator {
    * pre-extraction `this.extractionQueue.push(...)` used by the
    * characterization tests that drive the drain manually via `processQueue`.
    */
-  pushRaw(task: () => Promise<void>): void {
-    this.queue.push(task);
+  pushRaw(task: ExtractionQueueTask, options?: { key?: string }): void {
+    this.queue.push({ run: task, key: options?.key });
+    this.wake?.();
   }
 
   /** Remove + return the front task (test seam for the flush deadline tests). */
-  shift(): (() => Promise<void>) | undefined {
-    return this.queue.shift();
+  shift(): ExtractionQueueTask | undefined {
+    return this.queue.shift()?.run;
   }
 
   /**

package/src/orchestration/extraction-run.ts

@@ -125,6 +125,7 @@ import {
 } from "./extraction-run-helpers.js";
 import type { ExtractionResilienceStatus, ExtractionRetryStateEntry } from "./extraction-run-helpers.js";
 import type { ExtractionPersistResult } from "./extraction-persist.js";
+import { ProviderCircuitBreaker, type ProbeOwnerToken } from "./provider-breaker.js";
 
 export {
   capExtractionRetryStateEntries,
@@ -150,12 +151,7 @@ export class ExtractionRunCoordinator {
   // backoff still throttles, so a restart can't resurrect the hot loop.
   private readonly extractionRetryState = new Map<string, Map<string, ExtractionRetryStateEntry>>();
   private readonly hydratedRetryNamespaces = new Set<string>();
-  private providerBreaker: {
-    consecutiveFailures: number;
-    openUntilMs: number;
-    state: "closed" | "open" | "half_open";
-    lastReason: string;
-  } = { consecutiveFailures: 0, openUntilMs: 0, state: "closed", lastReason: "" };
+  private readonly providerBreaker = new ProviderCircuitBreaker();
 
   constructor(private readonly deps: ExtractionRunCoordinatorDeps) {}
 
@@ -282,65 +278,6 @@ export class ExtractionRunCoordinator {
     this.extractionRetryState.set(namespace, nsMap);
   }
 
-  private logBreakerTransition(state: "closed" | "open" | "half_open"): void {
-    // Once per state change (not per attempt) — AGENTS.md observability rule.
-    log.warn("extraction: provider circuit breaker transition", {
-      state,
-      consecutiveFailures: this.providerBreaker.consecutiveFailures,
-      lastReason: this.providerBreaker.lastReason,
-      openUntilMs: this.providerBreaker.openUntilMs,
-    });
-  }
-
-  /** True while the breaker suppresses non-forced attempts. Flips an expired
-   *  `open` breaker to `half_open`, which allows exactly one probe (the next
-   *  non-forced attempt) because the extraction queue is a serial drain. */
-  private isProviderBreakerOpen(now: number): boolean {
-    if (now < this.providerBreaker.openUntilMs) return true;
-    if (this.providerBreaker.state === "open") {
-      this.providerBreaker.state = "half_open";
-      this.logBreakerTransition("half_open");
-    }
-    return false;
-  }
-
-  private onProviderFailure(cls: ExtractionFailureClass, now: number): void {
-    this.providerBreaker.consecutiveFailures += 1;
-    const trip =
-      cls === "auth_config" ||
-      // A failed half-open probe re-opens immediately (standard breaker
-      // semantics). Without this, an auth_config-opened breaker (which trips
-      // below the threshold) would get stuck half_open after a transient
-      // probe failure and stop suppressing (cursor review).
-      this.providerBreaker.state === "half_open" ||
-      this.providerBreaker.consecutiveFailures >= this.config.extractionBreakerFailureThreshold;
-    if (!trip) return;
-    const cooldown =
-      cls === "auth_config"
-        ? this.config.extractionBreakerAuthCooldownMs
-        : this.config.extractionBreakerCooldownMs;
-    this.providerBreaker.openUntilMs = now + Math.max(0, cooldown);
-    this.providerBreaker.lastReason = cls;
-    if (this.providerBreaker.state !== "open") {
-      this.providerBreaker.state = "open";
-      this.logBreakerTransition("open");
-    }
-  }
-
-  private resetProviderBreakerOnSuccess(): void {
-    const wasDisturbed =
-      this.providerBreaker.state !== "closed" ||
-      this.providerBreaker.consecutiveFailures > 0 ||
-      this.providerBreaker.openUntilMs > 0;
-    if (!wasDisturbed) return;
-    const wasOpen = this.providerBreaker.state !== "closed";
-    this.providerBreaker.consecutiveFailures = 0;
-    this.providerBreaker.openUntilMs = 0;
-    this.providerBreaker.lastReason = "";
-    this.providerBreaker.state = "closed";
-    if (wasOpen) this.logBreakerTransition("closed");
-  }
-
   /** Rebuild `meta.extractionRetryState` from the in-memory namespace mirror,
    *  bounded to the newest entries, and prune the mirror to match. */
   private persistRetryStateToMeta(namespace: string, meta: MetaState): void {
@@ -370,10 +307,11 @@ export class ExtractionRunCoordinator {
     cls: ExtractionFailureClass,
     meta: MetaState,
     now: number,
+    probeToken?: ProbeOwnerToken,
   ): Promise<void> {
     try {
       this.hydrateRetryStateFromMeta(namespace, meta);
-      this.applyExtractionFailureToMemory(namespace, fingerprint, cls, now);
+      this.applyExtractionFailureToMemory(namespace, fingerprint, cls, now, probeToken);
       this.persistRetryStateToMeta(namespace, meta);
       // The caller's storage.updateMeta persists this — saving here would write the same file twice.
     } catch (err) {
@@ -389,6 +327,7 @@ export class ExtractionRunCoordinator {
     fingerprint: string,
     cls: ExtractionFailureClass,
     now: number,
+    probeToken?: ProbeOwnerToken,
   ): void {
     const nsMap = this.extractionRetryState.get(namespace) ?? new Map<string, ExtractionRetryStateEntry>();
     const prev = nsMap.get(fingerprint);
@@ -409,7 +348,11 @@ export class ExtractionRunCoordinator {
     }
     nsMap.set(fingerprint, { attempts, nextEligibleAtMs, firstFailedAtMs, lastFailureClass: cls });
     this.extractionRetryState.set(namespace, nsMap);
-    this.onProviderFailure(cls, now);
+    this.providerBreaker.failure(cls, now, {
+      failureThreshold: this.config.extractionBreakerFailureThreshold,
+      cooldownMs: this.config.extractionBreakerCooldownMs,
+      authCooldownMs: this.config.extractionBreakerAuthCooldownMs,
+    }, probeToken);
   }
 
   /** Delete a fingerprint's retry entry from the mirror + meta. Returns whether
@@ -430,12 +373,7 @@ export class ExtractionRunCoordinator {
       backoffFingerprintCount += nsMap.size;
     }
     return {
-      breaker: {
-        state: this.providerBreaker.state,
-        openUntilMs: this.providerBreaker.openUntilMs,
-        consecutiveFailures: this.providerBreaker.consecutiveFailures,
-        lastReason: this.providerBreaker.lastReason,
-      },
+      breaker: this.providerBreaker.snapshot(),
       backoffFingerprintCount,
     };
   }
@@ -834,13 +772,18 @@ export class ExtractionRunCoordinator {
     // Resolve the gate through the shared lifecycle capability plan (issue
     // #1523) rather than reading the raw config flag at each call site.
     const extractionRetryEnabled = resolveMemoryLifecycleCapabilities(this.config).extractionRetry;
+    let extractionProbeOwned = false;
+    let extractionProbeToken: ProbeOwnerToken | undefined;
     if (extractionFingerprint && !forcedExtractionAttempt && extractionRetryEnabled) {
       try {
         meta ??= await runDeadlineAware(() => storage.loadMeta(), "during_retry_gate");
         this.hydrateRetryStateFromMeta(selfNamespace, meta);
         const nowMs = Date.now();
+        const probeAttempt = this.providerBreaker.acquire(nowMs);
+        extractionProbeOwned = probeAttempt.probe;
+        extractionProbeToken = probeAttempt.token;
         let suppressReason: "provider_circuit_open" | "extraction_backoff" | null = null;
-        if (this.isProviderBreakerOpen(nowMs)) {
+        if (probeAttempt.suppressed) {
           suppressReason = "provider_circuit_open";
         } else {
           const st = this.extractionRetryState.get(selfNamespace)?.get(extractionFingerprint);
@@ -865,6 +808,10 @@ export class ExtractionRunCoordinator {
             log.warn("runExtraction: passive correction capture failed on suppressed attempt (non-fatal)", captureErr);
           }
           clearExtractionDeadlineTimer();
+          // Backoff suppression exits inside the gate, before the wrapper
+          // finally: release an acquired probe here (codex follow-up) so a
+          // cooldown-expired fingerprint cannot pin the breaker half_open.
+          if (extractionProbeOwned) this.providerBreaker.releaseProbe(extractionProbeToken);
           return { status: "skipped", reason: suppressReason, persistedCount: 0, durableOutputCount: 0 };
         }
       } catch (err) {
@@ -873,26 +820,28 @@ export class ExtractionRunCoordinator {
         log.warn("runExtraction: extraction retry gate check failed; proceeding (fail-open)", err);
       }
     }
-
     // Pass existing entity names so the LLM can reuse them instead of inventing variants.
-    const existingEntities = await runDeadlineAware(
-      () => storage.listEntityNames(),
-      "during_list_entity_names",
-    );
-    let result: ExtractionResult;
+    // The probe owner must release on ANY pre-persistence exit too (codex P1):
+    // the big try below only covers from the extract call onward.
     try {
-      result = await raceRecallAbort(
-        this.deps.getExtraction().extract(normalizedTurns, existingEntities, extractionAbortSignal),
-        extractionAbortSignal,
-        "extraction aborted (during_extract)",
+      const existingEntities = await runDeadlineAware(
+        () => storage.listEntityNames(),
+        "during_list_entity_names",
       );
-    } catch (error) {
-      clearExtractionDeadlineTimer();
-      if (typeof deadlineMs === "number" && Date.now() >= deadlineMs) {
-        throw new ExtractionDeadlineError("during_extract");
+      let result: ExtractionResult;
+      try {
+        result = await raceRecallAbort(
+          this.deps.getExtraction().extract(normalizedTurns, existingEntities, extractionAbortSignal),
+          extractionAbortSignal,
+          "extraction aborted (during_extract)",
+        );
+      } catch (error) {
+        clearExtractionDeadlineTimer();
+        if (deadlineMs !== undefined && Date.now() >= deadlineMs) {
+          throw new ExtractionDeadlineError("during_extract");
+        }
+        throw error;
       }
-      throw error;
-    }
     try {
       throwIfDeadlineExceeded("before_persist");
       throwIfAborted("before_persist");
@@ -965,7 +914,7 @@ export class ExtractionRunCoordinator {
           await runDeadlineAware(
             () =>
               storage.updateMeta((freshMeta) =>
-                this.recordExtractionFailure(selfNamespace, extractionFingerprint, failureClass, freshMeta, Date.now()),
+                this.recordExtractionFailure(selfNamespace, extractionFingerprint, failureClass, freshMeta, Date.now(), extractionProbeToken),
               ),
             "during_retry_bookkeeping_save",
             false,
@@ -976,14 +925,16 @@ export class ExtractionRunCoordinator {
           // Fail-open, but retain the turns (a cleared buffer with no retry state drops them). Only a
           // LOCK skip reaches here without the mutator having run — a later save must not double-count.
           if (err instanceof Error && err.message.includes("meta.json.lock not acquired"))
-            this.applyExtractionFailureToMemory(selfNamespace, extractionFingerprint, failureClass, Date.now());
+            this.applyExtractionFailureToMemory(selfNamespace, extractionFingerprint, failureClass, Date.now(), extractionProbeToken);
           recordedRetryFailure = true;
           log.warn("runExtraction: retry state kept in memory; meta persist will catch up", err);
         }
       } else if (result.extractionSkippedReason === undefined) {
         // Provider responded without failure → breaker heals; clear any
         // parked backoff for this fingerprint so it proceeds normally.
-        this.resetProviderBreakerOnSuccess();
+        // PRRT_kwDORJXyws6zxG6u: pass the run's probe token so the settlement
+        // only applies to a probe this run owns.
+        this.providerBreaker.success(extractionProbeToken);
         try {
           await runDeadlineAware(
             () =>
@@ -1196,5 +1147,11 @@ export class ExtractionRunCoordinator {
     } finally {
       clearExtractionDeadlineTimer();
     }
+    } finally {
+      // The half-open probe is OWNED, not timed: the WRAPPER finally releases
+      // it on every exit path, including pre-persistence ones (codex P1) that
+      // never reach the inner try above.
+      if (extractionProbeOwned) this.providerBreaker.releaseProbe(extractionProbeToken);
+    }
   }
 }