@remnic/core 10.0.65 → 10.0.67

package/src/orchestration/fact-dedup-window.ts

@@ -0,0 +1,122 @@
+import path from "node:path";
+import { serializeMutations, withHeldFileLock } from "../utils/serialize-mutations.js";
+import { ContentHashIndex, type StorageManager } from "../index.js";
+import type { MemoryFile } from "../types.js";
+import { normalizeStoredHashSource } from "./extraction-persist-citation.js";
+import { stripAttributesSuffix } from "../structured-attributes.js";
+import { log } from "../logger.js";
+
+/**
+ * Fact-dedup commit window (codex P2, #1526): with extractionQueueConcurrency
+ * > 1 (or daemon + external drain workers), two sessions can both pass the
+ * content-hash check at persist time and both write, producing duplicate
+ * active copies — the persist-time checks run well before the write, and
+ * writeMemory mints a fresh id per write. This module serializes the short
+ * "final recheck → write" window per storage dir (in-process chain +
+ * fail-closed cross-process fact-dedup.lock, mirroring the profile/meta lock
+ * shape: bounded, no LLM or corpus-scale work planned inside, throw BEFORE
+ * any read/write when the lock is not acquired). The persist-time check stays
+ * OUTSIDE the window exactly like the consolidation LLM call stays outside
+ * commitConsolidatedProfile.
+ */
+export interface FactDedupProbe {
+  hasContentHashDedup(storage: StorageManager, key: string): Promise<boolean>;
+}
+
+interface FactDedupMatchOptions {
+  readonly category: string;
+  readonly contentHashDedupKey: string;
+  readonly connector?: string;
+  readonly citationEnabled: boolean;
+  readonly citationTemplate: string;
+}
+
+function matchesActiveConnectorDuplicate(memory: MemoryFile, opts: FactDedupMatchOptions): boolean {
+  if (memory.frontmatter.category !== opts.category) return false;
+  if ((memory.frontmatter.status ?? "active") !== "active") return false;
+  // Thread 5 (QPDE5): for procedures the hash is keyed on the full
+  // body (title + steps via buildProcedurePersistBody), so compare
+  // against contentHashDedupKey — not the canonical title-only form.
+  // Thread 3 (QO42V): facts with structuredAttributes get an appended
+  // [Attributes: ...] suffix in the stored body that the hash key lacks;
+  // strip it before comparing so same-connector enriched facts dedup
+  // correctly.
+  if (normalizeStoredHashSource(stripAttributesSuffix(memory.content ?? ""), opts.citationEnabled, opts.citationTemplate) !==
+    ContentHashIndex.normalizeContent(opts.contentHashDedupKey)) return false;
+  return (memory.frontmatter.sourceConnector?.trim() || undefined) === opts.connector;
+}
+
+/**
+ * Connector-aware duplicate confirm: same final semantics as the persist-time
+ * check — an authoritative index miss is a clean NOT-duplicate, a hash hit or
+ * a non-authoritative/failed lookup must be confirmed against the hot+cold
+ * corpus (same category, active, same normalized content, same connector),
+ * and a corpus-scan failure fails OPEN so an unverifiable state cannot
+ * silently drop content.
+ */
+export async function confirmActiveFactDuplicate(
+  probe: FactDedupProbe,
+  storage: StorageManager,
+  opts: FactDedupMatchOptions,
+): Promise<boolean> {
+  let needsCorpusConfirm: boolean;
+  try {
+    if (!(await probe.hasContentHashDedup(storage, opts.contentHashDedupKey))) {
+      if (await storage.isFactContentHashAuthoritative()) return false;
+      needsCorpusConfirm = true;
+    } else {
+      needsCorpusConfirm = true;
+    }
+  } catch {
+    // Dedup lookup failed (e.g. the authoritative rebuild could not run). Do
+    // NOT fail-open into a possible duplicate — confirm against the corpus.
+    needsCorpusConfirm = true;
+  }
+  try {
+    // #2016 cold-tier finding: the authoritative content-hash rebuild unions
+    // the HOT and COLD tiers, so a cold-only active copy must still confirm
+    // the duplicate and suppress the redundant hot write.
+    const [hotMems, coldMems] = await Promise.all([
+      storage.readAllMemories(),
+      storage.readAllColdMemories(),
+    ]);
+    return hotMems.some((m) => matchesActiveConnectorDuplicate(m, opts)) ||
+      coldMems.some((m) => matchesActiveConnectorDuplicate(m, opts));
+  } catch (err) {
+    log.warn(
+      `connector-aware dedup scan failed for storage ${storage.dir}; writing fail-open: ${err instanceof Error ? err.message : String(err)}`,
+    );
+    return false;
+  }
+}
+
+export type FactDedupWindowOutcome<T> = { suppressed: true } | { suppressed: false; result: T };
+
+/**
+ * Run `write` inside the per-dir fact-dedup window: recheck `recheck` (the
+ * SAME confirm used at persist time, now under the lock) and suppress as a
+ * duplicate when a concurrent session committed the same content first;
+ * otherwise write. Lock contention throws (fail closed) — never an unlocked
+ * write; the extraction retry path retains the accepted turns.
+ */
+export async function persistUnderFactDedupWindow<T>(
+  storageDir: string,
+  recheck: () => Promise<boolean>,
+  write: () => Promise<T>,
+): Promise<FactDedupWindowOutcome<T>> {
+  return serializeMutations(`fact-dedup:${storageDir}`, () =>
+    withHeldFileLock(
+      path.join(storageDir, "fact-dedup.lock"),
+      // Bounded: the window holds one corpus confirm + one memory write, not a
+      // scan-length lock (catalog round 5 adoption note).
+      { staleMs: 5_000, maxWaitMs: 2_000, pollMs: 50 },
+      async (acquired) => {
+        if (!acquired) {
+          throw new Error(`fact write skipped: fact-dedup.lock not acquired (${storageDir})`);
+        }
+        if (await recheck()) return { suppressed: true };
+        return { suppressed: false, result: await write() };
+      },
+    ),
+  );
+}

package/src/orchestration/provider-breaker.test.ts

@@ -0,0 +1,89 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { ProviderCircuitBreaker } from "./provider-breaker.js";
+
+const CONFIG = { failureThreshold: 3, cooldownMs: 5_000, authCooldownMs: 60_000 };
+
+test("half-open probe ownership is single-flight under parallel callers", () => {
+  const breaker = new ProviderCircuitBreaker();
+  breaker.failure("provider_retryable", 1_000, CONFIG);
+  breaker.failure("provider_retryable", 1_100, CONFIG);
+  breaker.failure("provider_retryable", 1_200, CONFIG); // threshold → open
+  assert.deepEqual(breaker.acquire(1_300).suppressed, true, "open suppresses during cooldown");
+
+  // Cooldown expires: the FIRST caller takes the probe…
+  const probe = breaker.acquire(6_200);
+  assert.equal(probe.probe, true, "expired breaker hands one probe out");
+  // …siblings stay suppressed for as long as the probe is held — no wall-clock
+  // expiry, however long the provider attempt runs (timeouts + retries).
+  assert.deepEqual(breaker.acquire(600_000).suppressed, true, "a live probe keeps suppressing");
+  assert.deepEqual(breaker.acquire(3_600_000).suppressed, true, "no timer expiry while held");
+
+  // Probe attempt dies WITHOUT a terminal outcome (deadline/abort): explicit
+  // release lets the next attempt re-probe.
+  breaker.releaseProbe(probe.token);
+  const reprobe = breaker.acquire(3_600_001);
+  assert.equal(reprobe.probe, true, "released probe is re-acquired");
+  assert.deepEqual(breaker.acquire(3_600_002).suppressed, true, "re-probe is also single-flight");
+
+  // Terminal outcomes release ownership: the OWNER's failed probe re-opens
+  // immediately (cooldown to 6_500 + 5_000).
+  breaker.failure("provider_retryable", 6_500, CONFIG, reprobe.token);
+  assert.deepEqual(breaker.acquire(7_000).suppressed, true, "failed probe re-opens at once");
+  assert.deepEqual(breaker.acquire(12_000).suppressed, false, "cooldown expiry admits a fresh probe");
+  breaker.success();
+  assert.deepEqual(breaker.acquire(12_001).probe, false, "successful probe closes the breaker");
+});
+
+test("a forced attempt's failure never releases another attempt's live probe", () => {
+  const breaker = new ProviderCircuitBreaker();
+  breaker.failure("provider_retryable", 1_000, CONFIG);
+  breaker.failure("provider_retryable", 1_100, CONFIG);
+  breaker.failure("provider_retryable", 1_200, CONFIG); // threshold → open
+  const probe = breaker.acquire(6_200);
+  assert.equal(probe.probe, true, "ordinary attempt owns the probe");
+
+  // A FORCED attempt bypassed acquire, fails while the probe is in flight:
+  // it must not release the owner or masquerade as the probe result.
+  breaker.failure("provider_retryable", 6_300, CONFIG /* no token */);
+  assert.deepEqual(breaker.acquire(6_400), { suppressed: true, probe: false }, "probe stays single-flight");
+
+  // Only the owner's failure settles it: re-open at once.
+  breaker.failure("provider_retryable", 6_500, CONFIG, probe.token);
+  assert.deepEqual(breaker.acquire(6_600), { suppressed: true, probe: false }, "failed probe re-opens immediately");
+
+  // A released probe is re-acquired only via its own token path.
+  const fresh = breaker.acquire(12_000);
+  assert.equal(fresh.probe, true);
+  breaker.releaseProbe(fresh.token);
+  const reprobe = breaker.acquire(12_001);
+  assert.equal(reprobe.probe, true, "released probe is re-acquired by the next attempt");
+  // A non-owner release with a foreign token is a no-op.
+  breaker.releaseProbe({ __probeOwner: true });
+  assert.deepEqual(breaker.acquire(12_002), { suppressed: true, probe: false });
+});
+
+test("an unowned forced outcome neither trips the half-open threshold nor settles a held probe", () => {
+  const breaker = new ProviderCircuitBreaker();
+  breaker.failure("provider_retryable", 1_000, CONFIG);
+  breaker.failure("provider_retryable", 1_100, CONFIG);
+  breaker.failure("provider_retryable", 1_200, CONFIG); // threshold → open
+  const probe = breaker.acquire(6_200);
+  assert.equal(probe.probe, true, "ordinary attempt owns the probe");
+
+  // Forced failures bypass acquire; even crossing the threshold they must
+  // not reopen over a held probe (PRRT_kwDORJXyws6zxG6V).
+  breaker.failure("provider_retryable", 6_250, CONFIG /* no token */);
+  breaker.failure("provider_retryable", 6_300, CONFIG /* no token */);
+  breaker.failure("provider_retryable", 6_350, CONFIG /* no token */);
+  assert.deepEqual(breaker.acquire(6_400), { suppressed: true, probe: false }, "probe stays single-flight");
+
+  // A forced success without the owner token must not settle the probe or
+  // close the breaker mid-probe (PRRT_kwDORJXyws6zxG6u).
+  breaker.success();
+  assert.deepEqual(breaker.acquire(6_450), { suppressed: true, probe: false }, "probe survives an unowned success");
+
+  // The owner's success settles it for real.
+  breaker.success(probe.token);
+  assert.deepEqual(breaker.acquire(6_500).suppressed, false, "owner success closes the breaker");
+});

package/src/orchestration/provider-breaker.ts

@@ -0,0 +1,158 @@
+import { log } from "../logger.js";
+import type { ExtractionFailureClass } from "../types.js";
+
+export interface ProviderBreakerConfig {
+  readonly failureThreshold: number;
+  readonly cooldownMs: number;
+  readonly authCooldownMs: number;
+}
+
+export interface ProbeOwnerToken {
+  readonly __probeOwner: true;
+}
+
+export interface ProviderBreakerSnapshot {
+  readonly state: "closed" | "open" | "half_open";
+  readonly openUntilMs: number;
+  readonly consecutiveFailures: number;
+  readonly lastReason: string;
+}
+
+export interface ProbeAcquisition {
+  /** True when the attempt is suppressed (breaker open, or the half-open
+   *  probe is owned by another attempt). */
+  readonly suppressed: boolean;
+  /** True when THIS attempt owns the single half-open probe and must
+   *  terminate it via {@link failure}/{@link success} (with its token), or
+   *  release it via {@link releaseProbe} on any early exit (deadline, abort,
+   *  throw). */
+  readonly probe: boolean;
+  /** Opaque ownership token; only the matching token settles or releases the
+   *  probe, so attempts that bypass the gate (forced extractions) can never
+   *  release another attempt's in-flight probe. */
+  readonly token?: ProbeOwnerToken;
+}
+
+/**
+ * Provider circuit breaker with single-flight half-open probe OWNERSHIP
+ * (codex P2 on #3311): under extractionQueueConcurrency > 1 several sessions
+ * can pass an expired `open` breaker in the same window, and a wall-clock
+ * reservation either expires under a live long provider attempt (600s
+ * timeouts) or pins the breaker when an attempt dies without settling. So the
+ * probe is OWNED, not timed: exactly one attempt holds it from the
+ * open→half_open flip until its terminal `failure()`/`success()`, and any
+ * early exit releases it explicitly. A released-but-unfinished probe lets the
+ * next attempt re-probe; a retry backoff still throttles that namespace.
+ */
+export class ProviderCircuitBreaker {
+  private consecutiveFailures = 0;
+  private openUntilMs = 0;
+  private state: ProviderBreakerSnapshot["state"] = "closed";
+  private lastReason = "";
+  private probeHeld = false;
+  private probeToken: ProbeOwnerToken | null = null;
+
+  acquire(now: number): ProbeAcquisition {
+    if (now < this.openUntilMs) return { suppressed: true, probe: false };
+    if (this.state === "open") {
+      this.state = "half_open";
+      this.probeHeld = true;
+      const token: ProbeOwnerToken = { __probeOwner: true };
+      this.probeToken = token;
+      this.logTransition();
+      return { suppressed: false, probe: true, token }; // THIS attempt is the probe.
+    }
+    if (this.state === "half_open") {
+      if (this.probeHeld) return { suppressed: true, probe: false };
+      this.probeHeld = true; // previous owner released without settling; re-probe.
+      const token: ProbeOwnerToken = { __probeOwner: true };
+      this.probeToken = token;
+      return { suppressed: false, probe: true, token };
+    }
+    return { suppressed: false, probe: false };
+  }
+
+  /** Release an owned probe on an attempt that exits without a terminal
+   *  outcome (deadline, abort, throw). Only the matching owner releases;
+   *  idempotent; state stays half_open so the next attempt re-probes. */
+  releaseProbe(token?: ProbeOwnerToken): void {
+    if (token === undefined || this.probeToken !== token) return;
+    this.probeToken = null;
+    this.probeHeld = false;
+  }
+
+  failure(cls: ExtractionFailureClass, now: number, config: ProviderBreakerConfig, probeToken?: ProbeOwnerToken): void {
+    // Only the probe OWNER's failure settles the half-open probe: a failure
+    // from an attempt that bypassed acquire (forced extraction) must not
+    // release another attempt's in-flight probe or masquerade as its result.
+    const owned = probeToken !== undefined && this.probeToken === probeToken;
+    if (owned) {
+      this.probeToken = null;
+      this.probeHeld = false;
+    }
+    this.consecutiveFailures += 1;
+    const trip =
+      // An unowned auth_config failure (forced extraction) must not reopen
+      // over a held half-open probe — same ownership rule as the threshold
+      // term; the auth cooldown still applies once the probe settles.
+      (cls === "auth_config" && (this.state !== "half_open" || owned)) ||
+      // A failed half-open probe re-opens immediately (standard breaker
+      // semantics). Without this, an auth_config-opened breaker (which trips
+      // below the threshold) would get stuck half_open after a transient
+      // probe failure and stop suppressing (cursor review).
+      (owned && this.state === "half_open") ||
+      // PRRT_kwDORJXyws6zxG6V: an unowned (forced-extraction) failure must
+      // not cross the threshold while another attempt owns the half-open
+      // probe — reopening here would strand the owner's token and let the
+      // next cooldown mint a second concurrent probe. The failure count and
+      // retry bookkeeping still accumulate for when the state leaves
+      // half_open.
+      (this.state !== "half_open" && this.consecutiveFailures >= config.failureThreshold);
+    if (!trip) return;
+    const cooldown = cls === "auth_config" ? config.authCooldownMs : config.cooldownMs;
+    this.openUntilMs = now + Math.max(0, cooldown);
+    this.lastReason = cls;
+    if (this.state !== "open") {
+      this.state = "open";
+      this.logTransition();
+    }
+  }
+
+  success(probeToken?: ProbeOwnerToken): void {
+    const wasDisturbed =
+      this.state !== "closed" || this.consecutiveFailures > 0 || this.openUntilMs > 0;
+    if (!wasDisturbed) return;
+    // PRRT_kwDORJXyws6zxG6u: while a half-open probe is held, only its OWNER
+    // may settle it. An unowned (forced-extraction) success must not clear
+    // another run's probe or close the breaker mid-probe — the next ordinary
+    // acquisition could then start a second concurrent probe.
+    if (this.probeHeld && !(probeToken !== undefined && this.probeToken === probeToken)) return;
+    const wasOpen = this.state !== "closed";
+    this.probeToken = null;
+    this.probeHeld = false;
+    this.consecutiveFailures = 0;
+    this.openUntilMs = 0;
+    this.lastReason = "";
+    this.state = "closed";
+    if (wasOpen) this.logTransition();
+  }
+
+  snapshot(): ProviderBreakerSnapshot {
+    return {
+      state: this.state,
+      openUntilMs: this.openUntilMs,
+      consecutiveFailures: this.consecutiveFailures,
+      lastReason: this.lastReason,
+    };
+  }
+
+  private logTransition(): void {
+    // Once per state change (not per attempt) — AGENTS.md observability rule.
+    log.warn("extraction: provider circuit breaker transition", {
+      state: this.state,
+      consecutiveFailures: this.consecutiveFailures,
+      lastReason: this.lastReason,
+      openUntilMs: this.openUntilMs,
+    });
+  }
+}

package/src/orchestration/turn-ingestion.ts

@@ -932,8 +932,8 @@ export class TurnIngestionCoordinator {
     }
     if (settled) return;
     const preemptable = options.preemptable === true && options.preemptRunning !== true;
-    const accepted = this.deps.extractionQueueCoordinator.enqueue(async () => {
-      await this.deps.extractionQueueCoordinator.runPreemptable(preemptable, options.abortSignal, async (abortSignal) => {
+    const accepted = this.deps.extractionQueueCoordinator.enqueue(async (preemptHandle) => {
+      await this.deps.extractionQueueCoordinator.runPreemptable(preemptHandle, preemptable, options.abortSignal, async (abortSignal) => {
       if (settled) return;
       if (
         typeof extractionDeadlineMs === "number" &&
@@ -1082,14 +1082,14 @@ export class TurnIngestionCoordinator {
           deadlineMs: extractionDeadlineMs,
           bufferKey,
           onDurableCommit: options.onDurableCommit,
-          onBeforePersist: () => this.deps.extractionQueueCoordinator.endPreemptable(),
+          onBeforePersist: () => preemptHandle?.disarm(),
           abortSignal,
           failOnExtractionFailure: options.failOnExtractionFailure === true,
           writeNamespaceOverride: options.writeNamespaceOverride,
           principalOverride: options.principalOverride,
           scopeProfileWritePlan: effectiveScopePlan,
           forceExtractionAttempt: options.forceExtractionAttempt === true,
-        }, preemptable);
+        }, preemptable, preemptHandle);
         settleTask(undefined, result);
       } catch (err) {
         const reason = abortSignal.reason;
@@ -1099,7 +1099,7 @@ export class TurnIngestionCoordinator {
         }
       }
       });
-    }, { front: options.preemptRunning === true });
+    }, { front: options.preemptRunning === true, key: bufferKey });
     if (!accepted) {
       settleTask(new Error("extraction queue is stopped"));
       return;

package/src/orchestrator-extraction-queue.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { ExtractionQueueCoordinator } from "./orchestration/extraction-queue-coordinator.js";
+import { ExtractionQueueCoordinator, type ExtractionPreemptHandle, type ExtractionQueueTask } from "./orchestration/extraction-queue-coordinator.js";
 import { initLogger, type LoggerBackend } from "./logger.js";
 import { abortError } from "./abort-error.js";
 
@@ -227,17 +227,17 @@ test("a front enqueue aborts the running automatic task and runs before queued w
   const coordinator = createCoordinator();
   const order: string[] = [];
   const started = Promise.withResolvers<void>();
-  coordinator.enqueue(async () => {
-    const preempt = new AbortController();
-    coordinator.beginPreemptable(() => preempt.abort());
+  coordinator.enqueue(async (preempt) => {
+    const controller = new AbortController();
+    preempt?.arm(() => controller.abort());
     started.resolve();
     try {
-      await new Promise<void>((_resolve, reject) => {
-        preempt.signal.addEventListener("abort", () => reject(abortError("preempted")), { once: true });
-      });
+      const { promise, reject } = Promise.withResolvers<void>();
+      controller.signal.addEventListener("abort", () => reject(abortError("preempted")), { once: true });
+      await promise;
       order.push("automatic-finished");
     } finally {
-      coordinator.endPreemptable();
+      preempt?.disarm();
     }
   });
   await started.promise;
@@ -270,3 +270,174 @@ test("a front enqueue does not abort a task that did not register as preemptable
   assert.deepEqual(order, ["protected", "flush"]);
 });
 
+
+// ── Bounded-parallel queue (extractionQueueConcurrency) ─────────────────────
+
+function createParallelCoordinator(limit: number): ExtractionQueueCoordinator {
+  return new ExtractionQueueCoordinator(limit);
+}
+
+/** A parked queue task with deterministic start/abort/release signals. */
+function makeParkedTask(preemptable: boolean) {
+  const started = Promise.withResolvers<void>();
+  const release = Promise.withResolvers<void>();
+  let didStart = false;
+  let outcome: "parked" | "finished" | "aborted" = "parked";
+  const task: ExtractionQueueTask = async (preempt?: ExtractionPreemptHandle) => {
+    didStart = true;
+    const controller = new AbortController();
+    if (preemptable) {
+      preempt?.arm(() => controller.abort(abortError("preempted")));
+      controller.signal.addEventListener("abort", () => {
+        outcome = "aborted";
+        release.resolve();
+      }, { once: true });
+    }
+    started.resolve();
+    await release.promise;
+    if (outcome === "parked") outcome = "finished";
+    preempt?.disarm();
+  };
+  return {
+    task,
+    startedPromise: started.promise,
+    release: () => release.resolve(),
+    get started() {
+      return didStart;
+    },
+    get outcome() {
+      return outcome;
+    },
+  };
+}
+
+test("independent keys run concurrently up to the configured limit", async () => {
+  const coordinator = createParallelCoordinator(2);
+  const first = makeParkedTask(false);
+  const second = makeParkedTask(false);
+  const third = makeParkedTask(false);
+  coordinator.enqueue(first.task, { key: "session-a" });
+  coordinator.enqueue(second.task, { key: "session-b" });
+  coordinator.enqueue(third.task, { key: "session-c" });
+  await first.startedPromise;
+  await second.startedPromise;
+  assert.equal(third.started, false, "the third task must still be queued, not running: both slots are taken");
+  first.release();
+  await third.startedPromise;
+  second.release();
+  third.release();
+  assert.equal(await coordinator.waitForIdle(5_000), true);
+  assert.equal(first.outcome, "finished");
+  assert.equal(second.outcome, "finished");
+  assert.equal(third.outcome, "finished");
+  assert.equal(coordinator.isProcessing, false);
+});
+
+test("tasks sharing a buffer key never run concurrently even with free capacity", async () => {
+  const coordinator = createParallelCoordinator(2);
+  const busy = makeParkedTask(false);
+  const queuedSameKey = makeParkedTask(false);
+  const otherKey = makeParkedTask(false);
+  coordinator.enqueue(busy.task, { key: "s1" });
+  await busy.startedPromise;
+  coordinator.enqueue(queuedSameKey.task, { key: "s1" });
+  coordinator.enqueue(otherKey.task, { key: "s2" });
+  await otherKey.startedPromise;
+  assert.equal(queuedSameKey.started, false, "the busy session's next task must not have started: its own key is still busy");
+  busy.release();
+  await queuedSameKey.startedPromise;
+  queuedSameKey.release();
+  otherKey.release();
+  assert.equal(await coordinator.waitForIdle(5_000), true);
+  assert.equal(queuedSameKey.outcome, "finished");
+  assert.equal(otherKey.outcome, "finished");
+});
+
+test("the default limit stays serial", async () => {
+  const coordinator = createCoordinator();
+  assert.equal(coordinator.maxConcurrency, 1);
+  const first = makeParkedTask(false);
+  const second = makeParkedTask(false);
+  coordinator.enqueue(first.task, { key: "s1" });
+  coordinator.enqueue(second.task, { key: "s2" });
+  await first.startedPromise;
+  assert.equal(second.started, false, "limit 1 never starts a second task concurrently");
+  first.release();
+  await second.startedPromise;
+  second.release();
+  assert.equal(await coordinator.waitForIdle(5_000), true);
+  assert.equal(second.outcome, "finished");
+});
+
+test("a failing task does not stall the queue", async () => {
+  const { entries } = installCapturingLogger();
+  const coordinator = createParallelCoordinator(2);
+  const parked = makeParkedTask(false);
+  coordinator.enqueue(async () => {
+    throw new Error("extraction exploded");
+  }, { key: "s1" });
+  coordinator.enqueue(parked.task, { key: "s2" });
+  await parked.startedPromise;
+  parked.release();
+  assert.equal(await coordinator.waitForIdle(5_000), true);
+  assert.equal(parked.outcome, "finished");
+  assert.ok(entries.some((entry) => entry.level === "error" && /background extraction task failed/.test(entry.message)));
+});
+
+test("a front enqueue aborts exactly one armed running task when two run", async () => {
+  const coordinator = createParallelCoordinator(2);
+  const first = makeParkedTask(true);
+  const second = makeParkedTask(true);
+  let flushRan = false;
+  coordinator.enqueue(first.task, { key: "s1" });
+  coordinator.enqueue(second.task, { key: "s2" });
+  await first.startedPromise;
+  await second.startedPromise;
+  coordinator.enqueue(async () => {
+    flushRan = true;
+  }, { front: true });
+  assert.equal(second.outcome, "aborted", "the most recently armed task is the one preempted");
+  assert.equal(first.outcome, "parked", "the unrelated running extraction is not cancelled");
+  first.release();
+  assert.equal(await coordinator.waitForIdle(5_000), true);
+  assert.equal(flushRan, true);
+  assert.equal(first.outcome, "finished");
+});
+
+test("a front enqueue prefers the requested key's own running task", async () => {
+  const coordinator = createParallelCoordinator(2);
+  const other = makeParkedTask(true);
+  const mine = makeParkedTask(true);
+  let flushRan = false;
+  coordinator.enqueue(other.task, { key: "s-other" });
+  coordinator.enqueue(mine.task, { key: "s-mine" });
+  await other.startedPromise;
+  await mine.startedPromise;
+  coordinator.enqueue(async () => {
+    flushRan = true;
+  }, { front: true, key: "s-mine" });
+  assert.equal(mine.outcome, "aborted", "the flush preempts its own session's running extraction");
+  assert.equal(other.outcome, "parked", "the unrelated running extraction keeps going");
+  other.release();
+  assert.equal(await coordinator.waitForIdle(5_000), true);
+  assert.equal(flushRan, true);
+  assert.equal(other.outcome, "finished");
+});
+
+test("pauseAndDrain waits for concurrent accepted work and rejects new producers", async () => {
+  const coordinator = createParallelCoordinator(2);
+  const first = makeParkedTask(false);
+  const second = makeParkedTask(false);
+  coordinator.enqueue(first.task, { key: "s1" });
+  coordinator.enqueue(second.task, { key: "s2" });
+  await first.startedPromise;
+  await second.startedPromise;
+  const drain = coordinator.pauseAndDrain(5_000);
+  assert.equal(coordinator.enqueue(async () => {}, { key: "s3" }), false);
+  first.release();
+  second.release();
+  assert.equal(await drain, true);
+  assert.equal(first.outcome, "finished");
+  assert.equal(second.outcome, "finished");
+  assert.equal(coordinator.isProcessing, false);
+});