@remnic/core 9.3.728 → 9.3.729

package/src/orchestration/extraction-queue-coordinator.ts

@@ -0,0 +1,175 @@
+/**
+ * Extraction queue coordinator — extracted from the orchestrator (issue #1526).
+ *
+ * Owns the background serial queue that drains extraction tasks one at a
+ * time to avoid races between concurrent flush/heartbeat/bulk-import
+ * triggers:
+ *   - queue state (`extractionQueue` + `queueProcessing`)
+ *   - the scheduling trigger (start the drain when the first task lands and
+ *     the processor is idle)
+ *   - the serial drain loop itself (`processQueue`)
+ *   - failure classification (`logExtractionQueueFailure`) — issue #549:
+ *     AbortError from session transitions logs at debug, real failures at
+ *     error
+ *   - the idle wait (`waitForIdle`) used by bootstrap and tests
+ *
+ * Does NOT own the WHAT — `runExtraction`, the dedupe fingerprint check,
+ * the deadline timer, and the buffer-clear policy remain on the
+ * orchestrator (they need the full extraction dependency surface). The
+ * orchestrator builds each task closure and hands it to `enqueue`; the
+ * coordinator owns only the queue mechanics.
+ *
+ * Behavior-preserving move from orchestrator.ts. No logic changes — the
+ * orchestrator keeps a thin delegating `waitForExtractionIdle` and the
+ * `queueBufferedExtraction` builder delegates push+scheduling here.
+ */
+
+import { isAbortError } from "../abort-error.js";
+import { log } from "../logger.js";
+
+/**
+ * Coordinates the background serial extraction queue. Owns the in-flight
+ * guard + queue array that previously lived as private orchestrator fields.
+ */
+export class ExtractionQueueCoordinator {
+  /**
+   * Background serial queue for extractions (agent_end optimization).
+   * Queue stores tasks that resolve when extraction should run.
+   */
+  private readonly queue: Array<() => Promise<void>> = [];
+  /** Whether the serial drain loop is currently running. */
+  private processing = false;
+
+  /** Current queue depth (idle-wait + test seam). */
+  get length(): number {
+    return this.queue.length;
+  }
+
+  /** Whether the serial drain loop is currently running (idle-wait + test seam). */
+  get isProcessing(): boolean {
+    return this.processing;
+  }
+
+  /**
+   * Enqueue a task and start the serial processor if it is idle. This is
+   * the production entry point — the orchestrator's `queueBufferedExtraction`
+   * builds each task closure and hands it here.
+   */
+  enqueue(task: () => Promise<void>): void {
+    this.queue.push(task);
+    if (!this.processing) {
+      this.processing = true;
+      this.processQueue().catch((err) => {
+        this.logExtractionQueueFailure(err, "processor");
+        this.processing = false;
+      });
+    }
+  }
+
+  /**
+   * Background serial queue processor.
+   * Processes extractions one at a time to avoid race conditions.
+   * Called automatically when items are queued via `enqueue`; also exposed
+   * for the characterization tests that push raw tasks and assert drain
+   * semantics directly.
+   */
+  async processQueue(): Promise<void> {
+    while (this.queue.length > 0) {
+      const task = this.queue.shift();
+      if (task) {
+        try {
+          await task();
+        } catch (err) {
+          this.logExtractionQueueFailure(err, "task");
+        }
+      }
+    }
+
+    this.processing = false;
+  }
+
+  /**
+   * Classify + log a failure from either the per-task catch inside
+   * `processQueue()` or the outer `processQueue().catch(...)` in
+   * `enqueue()`.  Issue #549: `throwIfRecallAborted`
+   * (used throughout `runExtraction`) raises an Error whose `name` is
+   * `"AbortError"`.  That path fires when `before_reset` aborts a
+   * queued task to avoid duplicate extraction — it is intentional
+   * cancellation, not a failure.  Downgrading the log to debug
+   * prevents spurious `error`-level lines that routinely appear
+   * right next to a successful `persisted: N facts, M entities` log
+   * and that confuse operators into thinking extraction is broken.
+   * Genuine extraction failures (network, parse, I/O) still log at
+   * `error`.
+   *
+   * Source differentiates the two call sites so the log message
+   * names the right layer (`task` vs `processor`).
+   */
+  logExtractionQueueFailure(
+    err: unknown,
+    source: "task" | "processor",
+  ): void {
+    const aborted =
+      source === "task"
+        ? "background extraction task aborted (session transition)"
+        : "background extraction queue processor aborted (session transition)";
+    const failed =
+      source === "task"
+        ? "background extraction task failed"
+        : "background extraction queue processor failed";
+    if (isAbortError(err)) {
+      log.debug(aborted);
+    } else {
+      log.error(failed, err);
+    }
+  }
+
+  /**
+   * Wait until the extraction queue is fully drained (no tasks queued, no
+   * drain in flight) or `timeoutMs` elapses. Used by bootstrap (after a
+   * dry-run import) and by tests that need to assert extraction settled.
+   * Returns false on timeout, true once idle.
+   */
+  async waitForIdle(timeoutMs: number = 60_000): Promise<boolean> {
+    const started = Date.now();
+    while (this.processing || this.queue.length > 0) {
+      if (Date.now() - started > timeoutMs) {
+        log.warn(`waitForExtractionIdle timed out after ${timeoutMs}ms`);
+        return false;
+      }
+      const { promise, resolve } = Promise.withResolvers<void>();
+      setTimeout(resolve, 50);
+      await promise;
+    }
+    return true;
+  }
+
+  // ── Test seams ─────────────────────────────────────────────────────────
+  // Mirror the pre-extraction `orchestrator.extractionQueue` /
+  // `orchestrator.queueProcessing` direct-field access the characterization
+  // and flush deadline tests relied on. Each is a thin, well-named handle
+  // onto the coordinator's private state — no behavior change.
+
+  /**
+   * Push a raw task WITHOUT auto-starting the processor, matching the
+   * pre-extraction `this.extractionQueue.push(...)` used by the
+   * characterization tests that drive the drain manually via `processQueue`.
+   */
+  pushRaw(task: () => Promise<void>): void {
+    this.queue.push(task);
+  }
+
+  /** Remove + return the front task (test seam for the flush deadline tests). */
+  shift(): (() => Promise<void>) | undefined {
+    return this.queue.shift();
+  }
+
+  /**
+   * Force the processor-busy flag so `enqueue` does not auto-start the
+   * drain — used by the flush deadline test to simulate a busy queue and
+   * assert a queued task's deadline expires while it waits.
+   */
+  setProcessingForTest(value: boolean): void {
+    this.processing = value;
+  }
+}

package/src/orchestrator-extraction-queue.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { Orchestrator } from "./orchestrator.js";
+import { ExtractionQueueCoordinator } from "./orchestration/extraction-queue-coordinator.js";
 import { initLogger, type LoggerBackend } from "./logger.js";
 import { abortError } from "./abort-error.js";
 
@@ -31,17 +31,8 @@ function installCapturingLogger(): { entries: LogEntry[] } {
   return { entries };
 }
 
-type QueueOrchestrator = {
-  extractionQueue: Array<() => Promise<void>>;
-  queueProcessing: boolean;
-  processQueue: () => Promise<void>;
-};
-
-function createQueueOrchestrator(): QueueOrchestrator {
-  const orch = Object.create(Orchestrator.prototype) as QueueOrchestrator;
-  orch.extractionQueue = [];
-  orch.queueProcessing = true;
-  return orch;
+function createCoordinator(): ExtractionQueueCoordinator {
+  return new ExtractionQueueCoordinator();
 }
 
 // ── Issue #549 ─────────────────────────────────────────────────────────────
@@ -53,11 +44,11 @@ test("processQueue logs an AbortError task at debug, not error (#549)", async ()
   // That is intentional deduplication, not a failure.  The queue
   // processor must log it at debug.
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator();
-  orch.extractionQueue.push(async () => {
+  const coordinator = createCoordinator();
+  coordinator.pushRaw(async () => {
     throw abortError("extraction aborted (before_extract)");
   });
-  await orch.processQueue();
+  await coordinator.processQueue();
 
   const errorEntries = entries.filter((e) => e.level === "error");
   const debugEntries = entries.filter(
@@ -80,11 +71,11 @@ test("processQueue still logs real task failures at error", async () => {
   // Guard the other half: non-abort errors (network, parse, I/O) must
   // continue to log at error level.
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator();
-  orch.extractionQueue.push(async () => {
+  const coordinator = createCoordinator();
+  coordinator.pushRaw(async () => {
     throw new Error("upstream LLM 500");
   });
-  await orch.processQueue();
+  await coordinator.processQueue();
 
   const errorEntries = entries.filter((e) => e.level === "error");
   assert.equal(errorEntries.length, 1);
@@ -96,14 +87,14 @@ test("processQueue still logs real task failures at error", async () => {
 
 test("processQueue handles a mixed run — abort goes to debug, failure to error", async () => {
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator();
-  orch.extractionQueue.push(async () => {
+  const coordinator = createCoordinator();
+  coordinator.pushRaw(async () => {
     throw abortError("extraction aborted (before_clear_buffer)");
   });
-  orch.extractionQueue.push(async () => {
+  coordinator.pushRaw(async () => {
     throw new Error("I/O failure");
   });
-  await orch.processQueue();
+  await coordinator.processQueue();
 
   const errorCount = entries.filter((e) => e.level === "error").length;
   const abortDebugCount = entries.filter(
@@ -121,30 +112,27 @@ test("processQueue handles a mixed run — abort goes to debug, failure to error
 
 test("processQueue clears queueProcessing on exit regardless of task outcomes", async () => {
   installCapturingLogger();
-  const orch = createQueueOrchestrator();
-  orch.extractionQueue.push(async () => {
+  const coordinator = createCoordinator();
+  coordinator.setProcessingForTest(true);
+  coordinator.pushRaw(async () => {
     throw abortError("extraction aborted");
   });
-  await orch.processQueue();
-  assert.equal(orch.queueProcessing, false);
+  await coordinator.processQueue();
+  assert.equal(coordinator.isProcessing, false);
 });
 
 // ── Outer processQueue().catch() branch (Codex follow-up on #549) ──────────
 
-type QueueProcessorOrchestrator = QueueOrchestrator & {
-  logExtractionQueueFailure: (err: unknown, source: "task" | "processor") => void;
-};
-
 test("logExtractionQueueFailure(processor) classifies AbortError as debug (#549)", async () => {
   // Covers the outer `processQueue().catch(...)` path in
-  // `queueBufferedExtraction`.  A processor-level AbortError can
+  // `enqueue`.  A processor-level AbortError can
   // bubble from e.g. a processQueue rewrite that awaits an external
   // signal, so the handler must fail open to debug — otherwise a
   // session-transition-triggered processor abort would get logged
   // at error next to the successful extraction it just produced.
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator() as QueueProcessorOrchestrator;
-  orch.logExtractionQueueFailure(
+  const coordinator = createCoordinator();
+  coordinator.logExtractionQueueFailure(
     abortError("queue processor aborted"),
     "processor",
   );
@@ -160,8 +148,8 @@ test("logExtractionQueueFailure(processor) classifies AbortError as debug (#549)
 
 test("logExtractionQueueFailure(processor) preserves error-level for real failures", async () => {
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator() as QueueProcessorOrchestrator;
-  orch.logExtractionQueueFailure(
+  const coordinator = createCoordinator();
+  coordinator.logExtractionQueueFailure(
     new Error("unexpected processor crash"),
     "processor",
   );
@@ -180,9 +168,9 @@ test("logExtractionQueueFailure names the right layer (task vs processor)", asyn
   // tells them whether a specific extraction aborted or the queue
   // processor itself did.
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator() as QueueProcessorOrchestrator;
-  orch.logExtractionQueueFailure(abortError("a"), "task");
-  orch.logExtractionQueueFailure(abortError("b"), "processor");
+  const coordinator = createCoordinator();
+  coordinator.logExtractionQueueFailure(abortError("a"), "task");
+  coordinator.logExtractionQueueFailure(abortError("b"), "processor");
   const debugMessages = entries
     .filter((e) => e.level === "debug")
     .map((e) => e.message);

package/src/orchestrator-flush.test.ts

@@ -7,6 +7,7 @@ import {
   BulkImportBatchPartialFailureError,
   Orchestrator,
 } from "./orchestrator.js";
+import { ExtractionQueueCoordinator } from "./orchestration/extraction-queue-coordinator.js";
 import { parseConfig } from "./config.js";
 import { stableHash } from "./coding/git-context.js";
 import type { BufferTurn } from "./types.js";
@@ -117,8 +118,7 @@ test("flushSession waits for queued extraction task completion", async () => {
       return [makeTurn("thread-a", "remember alpha")];
     },
   };
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   orchestrator.runExtraction = async () => {
     extractionStarted = true;
     await new Promise<void>((resolve) => {
@@ -147,8 +147,8 @@ test("flushSession waits for queued extraction task completion", async () => {
 test("ingestBulkImportBatch rejects when the extraction deadline expires in the queue", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = true;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
+  orchestrator.extractionQueueCoordinator.setProcessingForTest(true);
   let runExtractionCalls = 0;
   orchestrator.runExtraction = async () => {
     runExtractionCalls += 1;
@@ -185,7 +185,7 @@ test("ingestBulkImportBatch rejects when the extraction deadline expires in the
   );
   assert.equal(runExtractionCalls, 0);
 
-  const queuedTask = orchestrator.extractionQueue.shift();
+  const queuedTask = orchestrator.extractionQueueCoordinator.shift();
   assert.ok(queuedTask);
   await queuedTask();
   assert.equal(
@@ -198,8 +198,7 @@ test("ingestBulkImportBatch rejects when the extraction deadline expires in the
 test("ingestBulkImportBatch does not report queue wait timeout after extraction starts", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   let runExtractionCalls = 0;
   orchestrator.runExtraction = async () => {
     runExtractionCalls += 1;
@@ -234,8 +233,7 @@ test("ingestBulkImportBatch does not report queue wait timeout after extraction
 test("ingestBulkImportBatch reports post-persist metadata failures separately", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   orchestrator.runExtraction = async () => ({
     status: "completed",
     persistedCount: 1,
@@ -267,8 +265,7 @@ test("ingestBulkImportBatch reports post-persist metadata failures separately",
 test("ingestBulkImportBatch can disable source-valid-at replay context", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   const capturedSlices: BufferTurn[][] = [];
   orchestrator.runExtraction = async (turns: BufferTurn[]) => {
     capturedSlices.push(turns);
@@ -321,8 +318,7 @@ test("ingestBulkImportBatch can disable source-valid-at replay context", async (
 test("ingestBulkImportBatch preserves partial metadata failure before a later slice rejects", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   let runExtractionCalls = 0;
   orchestrator.runExtraction = async () => {
     runExtractionCalls += 1;
@@ -384,8 +380,7 @@ test("ingestBulkImportBatch preserves partial metadata failure before a later sl
 test("ingestBulkImportBatch stops after the first failed source-valid-at slice", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   let runExtractionCalls = 0;
   orchestrator.runExtraction = async () => {
     runExtractionCalls += 1;

package/src/orchestrator.ts

@@ -99,6 +99,7 @@ import {
 } from "./fallback-llm.js";
 import { MaintenanceScheduler } from "./orchestration/maintenance.js";
 import { TierMigrationCoordinator } from "./orchestration/tier-migration-coordinator.js";
+import { ExtractionQueueCoordinator } from "./orchestration/extraction-queue-coordinator.js";
 import {
   runLiveConnectorsOnce,
   type LiveConnectorsRunSummary,
@@ -171,7 +172,6 @@ import {
 import { SessionObserverState } from "./session-observer-state.js";
 import {
   abortError as sharedAbortError,
-  isAbortError,
   throwIfAborted as sharedThrowIfAborted,
 } from "./abort-error.js";
 import { CODEX_THREAD_KEY_PREFIX } from "./thread-key.js";
@@ -2021,10 +2021,14 @@ export class Orchestrator {
     suppressedReasonCounts: Record<string, number>;
   } = { detected: 0, queued: 0, autoApplied: 0, suppressedReasonCounts: {} };
 
-  // Background serial queue for extractions (agent_end optimization)
-  // Queue stores promises that resolve when extraction should run
-  private extractionQueue: Array<() => Promise<void>> = [];
-  private queueProcessing = false;
+  /**
+   * Background serial extraction queue coordinator (issue #1526 — moved
+   * from inline `extractionQueue`/`queueProcessing` fields). Owns queue
+   * state + scheduling + the serial drain + failure classification; the
+   * orchestrator builds each task closure in `queueBufferedExtraction` and
+   * hands it to `enqueue`.
+   */
+  readonly extractionQueueCoordinator: ExtractionQueueCoordinator;
   private heartbeatObserverChains = new Map<string, Promise<void>>();
   private recentExtractionFingerprints = new Map<string, number>();
   private readonly consolidationObservers = new Set<
@@ -2991,6 +2995,8 @@ export class Orchestrator {
       namespaceSearchRouter: this.namespaceSearchRouter,
       namespaceCatalog: this.namespaceCatalog,
     });
+    // Issue #1526: background extraction queue lives on its own coordinator.
+    this.extractionQueueCoordinator = new ExtractionQueueCoordinator();
     const conversationIndexRuntime = createConversationIndexRuntime(config, {
       getQmd: () => this.conversationQmd,
       getFaiss: () => this.conversationFaiss,
@@ -4720,15 +4726,8 @@ export class Orchestrator {
   }
 
   async waitForExtractionIdle(timeoutMs: number = 60_000): Promise<boolean> {
-    const started = Date.now();
-    while (this.queueProcessing || this.extractionQueue.length > 0) {
-      if (Date.now() - started > timeoutMs) {
-        log.warn(`waitForExtractionIdle timed out after ${timeoutMs}ms`);
-        return false;
-      }
-      await new Promise((resolve) => setTimeout(resolve, 50));
-    }
-    return true;
+    // Issue #1526: queue state + idle wait moved to ExtractionQueueCoordinator.
+    return this.extractionQueueCoordinator.waitForIdle(timeoutMs);
   }
 
   async waitForConsolidationIdle(timeoutMs: number = 60_000): Promise<boolean> {
@@ -13165,7 +13164,7 @@ export class Orchestrator {
       }, remainingMs);
     }
 
-    this.extractionQueue.push(async () => {
+    this.extractionQueueCoordinator.enqueue(async () => {
       if (settled) return;
       if (
         typeof extractionDeadlineMs === "number" &&
@@ -13196,13 +13195,6 @@ export class Orchestrator {
       }
     });
 
-    if (!this.queueProcessing) {
-      this.queueProcessing = true;
-      this.processQueue().catch((err) => {
-        this.logExtractionQueueFailure(err, "processor");
-        this.queueProcessing = false;
-      });
-    }
     log.debug(`queued extraction from ${reason}`);
   }
 
@@ -13270,62 +13262,6 @@ export class Orchestrator {
     return true;
   }
 
-  /**
-   * Background serial queue processor.
-   * Processes extractions one at a time to avoid race conditions.
-   * Called automatically when items are queued.
-   */
-  private async processQueue(): Promise<void> {
-    while (this.extractionQueue.length > 0) {
-      const task = this.extractionQueue.shift();
-      if (task) {
-        try {
-          await task();
-        } catch (err) {
-          this.logExtractionQueueFailure(err, "task");
-        }
-      }
-    }
-
-    this.queueProcessing = false;
-  }
-
-  /**
-   * Classify + log a failure from either the per-task catch inside
-   * `processQueue()` or the outer `processQueue().catch(...)` in
-   * `queueBufferedExtraction()`.  Issue #549: `throwIfRecallAborted`
-   * (used throughout `runExtraction`) raises an Error whose `name` is
-   * `"AbortError"`.  That path fires when `before_reset` aborts a
-   * queued task to avoid duplicate extraction — it is intentional
-   * cancellation, not a failure.  Downgrading the log to debug
-   * prevents spurious `error`-level lines that routinely appear
-   * right next to a successful `persisted: N facts, M entities` log
-   * and that confuse operators into thinking extraction is broken.
-   * Genuine extraction failures (network, parse, I/O) still log at
-   * `error`.
-   *
-   * Source differentiates the two call sites so the log message
-   * names the right layer (`task` vs `processor`).
-   */
-  private logExtractionQueueFailure(
-    err: unknown,
-    source: "task" | "processor",
-  ): void {
-    const aborted =
-      source === "task"
-        ? "background extraction task aborted (session transition)"
-        : "background extraction queue processor aborted (session transition)";
-    const failed =
-      source === "task"
-        ? "background extraction task failed"
-        : "background extraction queue processor failed";
-    if (isAbortError(err)) {
-      log.debug(aborted);
-    } else {
-      log.error(failed, err);
-    }
-  }
-
   /**
    * Passive correction capture (issue #1581) — detects corrections expressed
    * passively in conversation turns and routes them to the Correction Contract