@remnic/core 9.3.727 → 9.3.729

package/src/extraction.ts

@@ -38,6 +38,7 @@ import { normalizeProcedureSteps } from "./procedural/procedure-types.js";
 import { normalizeReasoningTrace } from "./reasoning-trace-types.js";
 import { looksLikeMechanicalTelemetryTranscript } from "./telemetry-transcript.js";
 import { buildFactProvenance, type ProvenanceTurnInput } from "./provenance.js";
+import { resolveMemoryLifecycleCapabilities } from "./capabilities.js";
 
 type ExtractionQuestion = ExtractionResult["questions"][number];
 type ExtractedFactResult = ExtractionResult["facts"][number];
@@ -1055,6 +1056,7 @@ export class ExtractionEngine {
   }
 
   async extract(turns: BufferTurn[], existingEntities?: string[]): Promise<ExtractionResult> {
+    const lifecycleCaps = resolveMemoryLifecycleCapabilities(this.config);
 
     // Guard: skip if buffer is empty or all turns are whitespace-only
     const substantiveTurns = turns.filter((t) => t.content.trim().length > 0);
@@ -1083,7 +1085,7 @@ export class ExtractionEngine {
       return { facts: [], profileUpdates: [], entities: [], questions: [] };
     }
     if (
-      this.config.extractionTelemetryPrefilterEnabled &&
+      lifecycleCaps.extractionTelemetryPrefilter &&
       looksLikeMechanicalTelemetryTranscript(conversation)
     ) {
       log.debug("extraction skipped — mechanical action/state telemetry without durable-memory cues");
@@ -1331,6 +1333,7 @@ export class ExtractionEngine {
    * Uses a minimal prompt to fit within local model context limits (typically 4k-8k).
    */
   private async extractWithLocalLlm(conversation: string, existingEntities?: string[]): Promise<ExtractionResult | null> {
+    const lifecycleCaps = resolveMemoryLifecycleCapabilities(this.config);
     log.debug(
       `extractWithLocalLlm: starting extraction, localLlmEnabled=${this.shouldUseLocalLlm}, model=${this.config.localLlmModel}`,
     );
@@ -1406,7 +1409,7 @@ These are durable insights - capture them:
 - CRITICAL: Use canonical hyphenated entity names (e.g., "jane-doe" not "janedoe")
 - CRITICAL: NEVER extract the same fact twice - check for duplicates before adding to facts array
 - CRITICAL: NEVER extract cron job schedules, automation configurations, or system monitoring details (these are operational noise)
-- If uncertain about relevance, prefer NOT extracting${this.config.extractionScopeClassificationEnabled ? `
+- If uncertain about relevance, prefer NOT extracting${lifecycleCaps.extractionScopeClassification ? `
 - For each fact, set "scope" to "global" (cross-project knowledge: framework bugs, library behavior, user preferences, tool configs, general patterns) or "project" (codebase-specific: file paths, env configs, deployment details, project workarounds). When in doubt, prefer "project".` : ""}
 
 === Structured Attributes ===
@@ -1645,6 +1648,7 @@ Omit "eventTime" when the fact has no explicit temporal anchor — do NOT guess
    * Build extraction instructions shared between local and cloud LLM.
    */
   private buildExtractionInstructions(existingEntities?: string[]): string {
+    const lifecycleCaps = resolveMemoryLifecycleCapabilities(this.config);
     return `You are a memory extraction system. Analyze the following conversation and extract durable, reusable memories.
 
 Memory categories:
@@ -1681,7 +1685,7 @@ Rules:
   * Inferred (0.40-0.69): Pattern recognition — reasonable guess from limited evidence
   * Speculative (0.00-0.39): Tentative hypothesis — weak signal, needs future confirmation. Speculative memories auto-expire after 30 days if not confirmed.${this.config.provenance?.enabled ? `
 - Source quotes: For each fact, include a "quote" field containing the EXACT verbatim words from the conversation that support the fact. Copy a contiguous span from a single speaker turn (not a paraphrase, not a summary). Cap at ~300 characters. This grounds every memory in the literal utterance that created it.` : ""}
-- For commitments: include any deadline or timeframe mentioned${this.config.extractionScopeClassificationEnabled ? `
+- For commitments: include any deadline or timeframe mentioned${lifecycleCaps.extractionScopeClassification ? `
 
 Scope classification:
 For each fact, set "scope" to one of:

package/src/maintenance/first-start-migration.ts

@@ -18,6 +18,7 @@
 
 import path from "node:path";
 import { access, mkdir, readFile, unlink, writeFile } from "node:fs/promises";
+import { resolveMemoryLifecycleCapabilities } from "../capabilities.js";
 import type { StorageManager } from "../storage.js";
 import type { PluginConfig } from "../types.js";
 import {
@@ -179,6 +180,7 @@ export async function runFirstStartMigration(
     signal,
   } = options;
   const now = (options.now ?? (() => new Date()))();
+  const lifecycleCaps = resolveMemoryLifecycleCapabilities(config);
   const abortedResult = (candidateCount = 0, demotedCount = 0, failureCount = 0): FirstStartMigrationResult => ({
     skipped: candidateCount === 0 && demotedCount === 0 && failureCount === 0,
     skipReason: candidateCount === 0 && demotedCount === 0 && failureCount === 0 ? "aborted" : undefined,
@@ -193,7 +195,7 @@ export async function runFirstStartMigration(
     return abortedResult();
   }
 
-  if (!config.lifecyclePolicyEnabled) {
+  if (!lifecycleCaps.lifecyclePolicy) {
     return {
       skipped: true,
       skipReason: "lifecyclePolicyEnabled is false",

package/src/orchestration/extraction-queue-coordinator.ts

@@ -0,0 +1,175 @@
+/**
+ * Extraction queue coordinator — extracted from the orchestrator (issue #1526).
+ *
+ * Owns the background serial queue that drains extraction tasks one at a
+ * time to avoid races between concurrent flush/heartbeat/bulk-import
+ * triggers:
+ *   - queue state (`extractionQueue` + `queueProcessing`)
+ *   - the scheduling trigger (start the drain when the first task lands and
+ *     the processor is idle)
+ *   - the serial drain loop itself (`processQueue`)
+ *   - failure classification (`logExtractionQueueFailure`) — issue #549:
+ *     AbortError from session transitions logs at debug, real failures at
+ *     error
+ *   - the idle wait (`waitForIdle`) used by bootstrap and tests
+ *
+ * Does NOT own the WHAT — `runExtraction`, the dedupe fingerprint check,
+ * the deadline timer, and the buffer-clear policy remain on the
+ * orchestrator (they need the full extraction dependency surface). The
+ * orchestrator builds each task closure and hands it to `enqueue`; the
+ * coordinator owns only the queue mechanics.
+ *
+ * Behavior-preserving move from orchestrator.ts. No logic changes — the
+ * orchestrator keeps a thin delegating `waitForExtractionIdle` and the
+ * `queueBufferedExtraction` builder delegates push+scheduling here.
+ */
+
+import { isAbortError } from "../abort-error.js";
+import { log } from "../logger.js";
+
+/**
+ * Coordinates the background serial extraction queue. Owns the in-flight
+ * guard + queue array that previously lived as private orchestrator fields.
+ */
+export class ExtractionQueueCoordinator {
+  /**
+   * Background serial queue for extractions (agent_end optimization).
+   * Queue stores tasks that resolve when extraction should run.
+   */
+  private readonly queue: Array<() => Promise<void>> = [];
+  /** Whether the serial drain loop is currently running. */
+  private processing = false;
+
+  /** Current queue depth (idle-wait + test seam). */
+  get length(): number {
+    return this.queue.length;
+  }
+
+  /** Whether the serial drain loop is currently running (idle-wait + test seam). */
+  get isProcessing(): boolean {
+    return this.processing;
+  }
+
+  /**
+   * Enqueue a task and start the serial processor if it is idle. This is
+   * the production entry point — the orchestrator's `queueBufferedExtraction`
+   * builds each task closure and hands it here.
+   */
+  enqueue(task: () => Promise<void>): void {
+    this.queue.push(task);
+    if (!this.processing) {
+      this.processing = true;
+      this.processQueue().catch((err) => {
+        this.logExtractionQueueFailure(err, "processor");
+        this.processing = false;
+      });
+    }
+  }
+
+  /**
+   * Background serial queue processor.
+   * Processes extractions one at a time to avoid race conditions.
+   * Called automatically when items are queued via `enqueue`; also exposed
+   * for the characterization tests that push raw tasks and assert drain
+   * semantics directly.
+   */
+  async processQueue(): Promise<void> {
+    while (this.queue.length > 0) {
+      const task = this.queue.shift();
+      if (task) {
+        try {
+          await task();
+        } catch (err) {
+          this.logExtractionQueueFailure(err, "task");
+        }
+      }
+    }
+
+    this.processing = false;
+  }
+
+  /**
+   * Classify + log a failure from either the per-task catch inside
+   * `processQueue()` or the outer `processQueue().catch(...)` in
+   * `enqueue()`.  Issue #549: `throwIfRecallAborted`
+   * (used throughout `runExtraction`) raises an Error whose `name` is
+   * `"AbortError"`.  That path fires when `before_reset` aborts a
+   * queued task to avoid duplicate extraction — it is intentional
+   * cancellation, not a failure.  Downgrading the log to debug
+   * prevents spurious `error`-level lines that routinely appear
+   * right next to a successful `persisted: N facts, M entities` log
+   * and that confuse operators into thinking extraction is broken.
+   * Genuine extraction failures (network, parse, I/O) still log at
+   * `error`.
+   *
+   * Source differentiates the two call sites so the log message
+   * names the right layer (`task` vs `processor`).
+   */
+  logExtractionQueueFailure(
+    err: unknown,
+    source: "task" | "processor",
+  ): void {
+    const aborted =
+      source === "task"
+        ? "background extraction task aborted (session transition)"
+        : "background extraction queue processor aborted (session transition)";
+    const failed =
+      source === "task"
+        ? "background extraction task failed"
+        : "background extraction queue processor failed";
+    if (isAbortError(err)) {
+      log.debug(aborted);
+    } else {
+      log.error(failed, err);
+    }
+  }
+
+  /**
+   * Wait until the extraction queue is fully drained (no tasks queued, no
+   * drain in flight) or `timeoutMs` elapses. Used by bootstrap (after a
+   * dry-run import) and by tests that need to assert extraction settled.
+   * Returns false on timeout, true once idle.
+   */
+  async waitForIdle(timeoutMs: number = 60_000): Promise<boolean> {
+    const started = Date.now();
+    while (this.processing || this.queue.length > 0) {
+      if (Date.now() - started > timeoutMs) {
+        log.warn(`waitForExtractionIdle timed out after ${timeoutMs}ms`);
+        return false;
+      }
+      const { promise, resolve } = Promise.withResolvers<void>();
+      setTimeout(resolve, 50);
+      await promise;
+    }
+    return true;
+  }
+
+  // ── Test seams ─────────────────────────────────────────────────────────
+  // Mirror the pre-extraction `orchestrator.extractionQueue` /
+  // `orchestrator.queueProcessing` direct-field access the characterization
+  // and flush deadline tests relied on. Each is a thin, well-named handle
+  // onto the coordinator's private state — no behavior change.
+
+  /**
+   * Push a raw task WITHOUT auto-starting the processor, matching the
+   * pre-extraction `this.extractionQueue.push(...)` used by the
+   * characterization tests that drive the drain manually via `processQueue`.
+   */
+  pushRaw(task: () => Promise<void>): void {
+    this.queue.push(task);
+  }
+
+  /** Remove + return the front task (test seam for the flush deadline tests). */
+  shift(): (() => Promise<void>) | undefined {
+    return this.queue.shift();
+  }
+
+  /**
+   * Force the processor-busy flag so `enqueue` does not auto-start the
+   * drain — used by the flush deadline test to simulate a busy queue and
+   * assert a queued task's deadline expires while it waits.
+   */
+  setProcessingForTest(value: boolean): void {
+    this.processing = value;
+  }
+}

package/src/orchestrator-extraction-queue.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import test from "node:test";
 
-import { Orchestrator } from "./orchestrator.js";
+import { ExtractionQueueCoordinator } from "./orchestration/extraction-queue-coordinator.js";
 import { initLogger, type LoggerBackend } from "./logger.js";
 import { abortError } from "./abort-error.js";
 
@@ -31,17 +31,8 @@ function installCapturingLogger(): { entries: LogEntry[] } {
   return { entries };
 }
 
-type QueueOrchestrator = {
-  extractionQueue: Array<() => Promise<void>>;
-  queueProcessing: boolean;
-  processQueue: () => Promise<void>;
-};
-
-function createQueueOrchestrator(): QueueOrchestrator {
-  const orch = Object.create(Orchestrator.prototype) as QueueOrchestrator;
-  orch.extractionQueue = [];
-  orch.queueProcessing = true;
-  return orch;
+function createCoordinator(): ExtractionQueueCoordinator {
+  return new ExtractionQueueCoordinator();
 }
 
 // ── Issue #549 ─────────────────────────────────────────────────────────────
@@ -53,11 +44,11 @@ test("processQueue logs an AbortError task at debug, not error (#549)", async ()
   // That is intentional deduplication, not a failure.  The queue
   // processor must log it at debug.
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator();
-  orch.extractionQueue.push(async () => {
+  const coordinator = createCoordinator();
+  coordinator.pushRaw(async () => {
     throw abortError("extraction aborted (before_extract)");
   });
-  await orch.processQueue();
+  await coordinator.processQueue();
 
   const errorEntries = entries.filter((e) => e.level === "error");
   const debugEntries = entries.filter(
@@ -80,11 +71,11 @@ test("processQueue still logs real task failures at error", async () => {
   // Guard the other half: non-abort errors (network, parse, I/O) must
   // continue to log at error level.
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator();
-  orch.extractionQueue.push(async () => {
+  const coordinator = createCoordinator();
+  coordinator.pushRaw(async () => {
     throw new Error("upstream LLM 500");
   });
-  await orch.processQueue();
+  await coordinator.processQueue();
 
   const errorEntries = entries.filter((e) => e.level === "error");
   assert.equal(errorEntries.length, 1);
@@ -96,14 +87,14 @@ test("processQueue still logs real task failures at error", async () => {
 
 test("processQueue handles a mixed run — abort goes to debug, failure to error", async () => {
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator();
-  orch.extractionQueue.push(async () => {
+  const coordinator = createCoordinator();
+  coordinator.pushRaw(async () => {
     throw abortError("extraction aborted (before_clear_buffer)");
   });
-  orch.extractionQueue.push(async () => {
+  coordinator.pushRaw(async () => {
     throw new Error("I/O failure");
   });
-  await orch.processQueue();
+  await coordinator.processQueue();
 
   const errorCount = entries.filter((e) => e.level === "error").length;
   const abortDebugCount = entries.filter(
@@ -121,30 +112,27 @@ test("processQueue handles a mixed run — abort goes to debug, failure to error
 
 test("processQueue clears queueProcessing on exit regardless of task outcomes", async () => {
   installCapturingLogger();
-  const orch = createQueueOrchestrator();
-  orch.extractionQueue.push(async () => {
+  const coordinator = createCoordinator();
+  coordinator.setProcessingForTest(true);
+  coordinator.pushRaw(async () => {
     throw abortError("extraction aborted");
   });
-  await orch.processQueue();
-  assert.equal(orch.queueProcessing, false);
+  await coordinator.processQueue();
+  assert.equal(coordinator.isProcessing, false);
 });
 
 // ── Outer processQueue().catch() branch (Codex follow-up on #549) ──────────
 
-type QueueProcessorOrchestrator = QueueOrchestrator & {
-  logExtractionQueueFailure: (err: unknown, source: "task" | "processor") => void;
-};
-
 test("logExtractionQueueFailure(processor) classifies AbortError as debug (#549)", async () => {
   // Covers the outer `processQueue().catch(...)` path in
-  // `queueBufferedExtraction`.  A processor-level AbortError can
+  // `enqueue`.  A processor-level AbortError can
   // bubble from e.g. a processQueue rewrite that awaits an external
   // signal, so the handler must fail open to debug — otherwise a
   // session-transition-triggered processor abort would get logged
   // at error next to the successful extraction it just produced.
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator() as QueueProcessorOrchestrator;
-  orch.logExtractionQueueFailure(
+  const coordinator = createCoordinator();
+  coordinator.logExtractionQueueFailure(
     abortError("queue processor aborted"),
     "processor",
   );
@@ -160,8 +148,8 @@ test("logExtractionQueueFailure(processor) classifies AbortError as debug (#549)
 
 test("logExtractionQueueFailure(processor) preserves error-level for real failures", async () => {
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator() as QueueProcessorOrchestrator;
-  orch.logExtractionQueueFailure(
+  const coordinator = createCoordinator();
+  coordinator.logExtractionQueueFailure(
     new Error("unexpected processor crash"),
     "processor",
   );
@@ -180,9 +168,9 @@ test("logExtractionQueueFailure names the right layer (task vs processor)", asyn
   // tells them whether a specific extraction aborted or the queue
   // processor itself did.
   const { entries } = installCapturingLogger();
-  const orch = createQueueOrchestrator() as QueueProcessorOrchestrator;
-  orch.logExtractionQueueFailure(abortError("a"), "task");
-  orch.logExtractionQueueFailure(abortError("b"), "processor");
+  const coordinator = createCoordinator();
+  coordinator.logExtractionQueueFailure(abortError("a"), "task");
+  coordinator.logExtractionQueueFailure(abortError("b"), "processor");
   const debugMessages = entries
     .filter((e) => e.level === "debug")
     .map((e) => e.message);

package/src/orchestrator-flush.test.ts

@@ -7,6 +7,7 @@ import {
   BulkImportBatchPartialFailureError,
   Orchestrator,
 } from "./orchestrator.js";
+import { ExtractionQueueCoordinator } from "./orchestration/extraction-queue-coordinator.js";
 import { parseConfig } from "./config.js";
 import { stableHash } from "./coding/git-context.js";
 import type { BufferTurn } from "./types.js";
@@ -117,8 +118,7 @@ test("flushSession waits for queued extraction task completion", async () => {
       return [makeTurn("thread-a", "remember alpha")];
     },
   };
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   orchestrator.runExtraction = async () => {
     extractionStarted = true;
     await new Promise<void>((resolve) => {
@@ -147,8 +147,8 @@ test("flushSession waits for queued extraction task completion", async () => {
 test("ingestBulkImportBatch rejects when the extraction deadline expires in the queue", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = true;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
+  orchestrator.extractionQueueCoordinator.setProcessingForTest(true);
   let runExtractionCalls = 0;
   orchestrator.runExtraction = async () => {
     runExtractionCalls += 1;
@@ -185,7 +185,7 @@ test("ingestBulkImportBatch rejects when the extraction deadline expires in the
   );
   assert.equal(runExtractionCalls, 0);
 
-  const queuedTask = orchestrator.extractionQueue.shift();
+  const queuedTask = orchestrator.extractionQueueCoordinator.shift();
   assert.ok(queuedTask);
   await queuedTask();
   assert.equal(
@@ -198,8 +198,7 @@ test("ingestBulkImportBatch rejects when the extraction deadline expires in the
 test("ingestBulkImportBatch does not report queue wait timeout after extraction starts", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   let runExtractionCalls = 0;
   orchestrator.runExtraction = async () => {
     runExtractionCalls += 1;
@@ -234,8 +233,7 @@ test("ingestBulkImportBatch does not report queue wait timeout after extraction
 test("ingestBulkImportBatch reports post-persist metadata failures separately", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   orchestrator.runExtraction = async () => ({
     status: "completed",
     persistedCount: 1,
@@ -267,8 +265,7 @@ test("ingestBulkImportBatch reports post-persist metadata failures separately",
 test("ingestBulkImportBatch can disable source-valid-at replay context", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   const capturedSlices: BufferTurn[][] = [];
   orchestrator.runExtraction = async (turns: BufferTurn[]) => {
     capturedSlices.push(turns);
@@ -321,8 +318,7 @@ test("ingestBulkImportBatch can disable source-valid-at replay context", async (
 test("ingestBulkImportBatch preserves partial metadata failure before a later slice rejects", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   let runExtractionCalls = 0;
   orchestrator.runExtraction = async () => {
     runExtractionCalls += 1;
@@ -384,8 +380,7 @@ test("ingestBulkImportBatch preserves partial metadata failure before a later sl
 test("ingestBulkImportBatch stops after the first failed source-valid-at slice", async () => {
   const orchestrator = Object.create(Orchestrator.prototype) as any;
   orchestrator.config = parseConfig({});
-  orchestrator.extractionQueue = [];
-  orchestrator.queueProcessing = false;
+  orchestrator.extractionQueueCoordinator = new ExtractionQueueCoordinator();
   let runExtractionCalls = 0;
   orchestrator.runExtraction = async () => {
     runExtractionCalls += 1;