@remnic/core 10.0.38 → 10.0.40

package/src/orchestration/extraction-drain.ts

@@ -0,0 +1,261 @@
+/**
+ * Chunked trigger-drain for the extraction pipeline.
+ *
+ * Split out of `turn-ingestion.ts`: that file is at the structural ratchet's
+ * line cap (issue #1995), and the drain is a self-contained unit — it only
+ * needs the host's `runExtraction`, the buffer drop, the storage side-file
+ * copy and the preemption hook.
+ */
+
+import { createHash } from "node:crypto";
+import { isAbortError, preemptedExtractionError } from "../abort-error.js";
+import { describeError } from "../buffer-turn-helpers.js";
+import { chunkExtractionWindow, classifyDrainResult, mergeDrainResults } from "../extraction-window-drain.js";
+import { log } from "../logger.js";
+import type { BufferTurn } from "../types.js";
+import type { ExtractionRunCoordinator, ExtractionRunResult } from "./extraction-run.js";
+
+type RunOptions = Parameters<ExtractionRunCoordinator["runExtraction"]>[1];
+
+/** The `runExtraction` options the drain forwards, with the ones it reads made required. */
+export type DrainRunOptions = RunOptions & {
+  bufferKey: string;
+  clearBufferAfterExtraction: boolean;
+  skipUserTurnThreshold: boolean;
+  failOnExtractionFailure: boolean;
+  forceExtractionAttempt: boolean;
+};
+
+/** What the drain needs from its host (structurally satisfied by `TurnIngestionDeps`). */
+export interface ExtractionDrainDeps {
+  config: {
+    extractionMinUserTurns: number;
+    extractionMinChars: number;
+    extractionMaxTurnChars: number;
+    extractionChunkDropAfterFailures: number;
+  };
+  extractionWindowBudgetChars?(): number;
+  runExtraction(turns: BufferTurn[], options: RunOptions): Promise<ExtractionRunResult>;
+  getStorage(): Promise<{
+    saveDroppedExtractionChunk(bufferKey: string, reason: string, turns: readonly BufferTurn[]): Promise<string>;
+  }>;
+  buffer: {
+    dropExtractionChunk(bufferKey: string, turns: readonly BufferTurn[], options?: { allowNonPrefix?: boolean }): Promise<number>;
+  };
+  extractionQueueCoordinator: { beginPreemptable(abort: () => void): void };
+}
+
+/** Identity of a chunk: its failure count must not follow a buffer key to different content. */
+function chunkFingerprint(chunk: readonly BufferTurn[]): string {
+  const hash = createHash("sha256");
+  for (const turn of chunk) hash.update(`${turn.role}\u0000${turn.timestamp}\u0000${turn.content}\u0001`);
+  return hash.digest("hex");
+}
+
+export class ExtractionDrain {
+  /**
+   * Consecutive failed attempts on the head chunk, per buffer key. In memory on
+   * purpose (a restart re-counts from zero). Bound to the chunk's content: a
+   * count recorded for one chunk never applies to a different one, so a force
+   * flush that clears a failing chunk cannot make unrelated content inherit its
+   * failures.
+   */
+  private readonly failures = new Map<string, { fingerprint: string; count: number }>();
+
+  /**
+   * Buffer key to the first turn of the chunk a qualified window stopped on.
+   * The rest of such a window stays eligible on the next attempt even when it
+   * is assistant-only or short: the window passed the thresholds as a whole,
+   * and re-judging its tail would clear it unextracted. Cleared when the
+   * window drains; one entry per key.
+   */
+  private readonly qualified = new Map<string, string>();
+
+  constructor(private readonly deps: ExtractionDrainDeps) {}
+
+  /** Characters the coordinator would count for this window: user/assistant text, trimmed and capped per turn, context-only turns left out. */
+  private windowChars(turns: readonly BufferTurn[]): number {
+    let total = 0;
+    for (const turn of turns) {
+      if ((turn.role !== "user" && turn.role !== "assistant") || turn.extractionContextOnly === true) continue;
+      total += turn.content.trim().slice(0, this.deps.config.extractionMaxTurnChars).length;
+    }
+    return total;
+  }
+
+  private failureCount(bufferKey: string, chunk: readonly BufferTurn[]): number {
+    const entry = this.failures.get(bufferKey);
+    return entry && entry.fingerprint === chunkFingerprint(chunk) ? entry.count : 0;
+  }
+
+  private setFailureCount(bufferKey: string, chunk: readonly BufferTurn[], count: number): void {
+    if (count <= 0) this.failures.delete(bufferKey);
+    else this.failures.set(bufferKey, { fingerprint: chunkFingerprint(chunk), count });
+  }
+
+  /**
+   * Run `runExtraction` over the trigger-drain window. When the window outgrows
+   * the extraction model's input budget (the stall that keeps a session's buffer
+   * growing forever: every observe re-fires the trigger on a window that can
+   * never extract), split it into budget-fitting chunks and drain them in
+   * order; each successful run clears its chunk, so the buffer shrinks across
+   * observes. A chunk that fails `extractionChunkDropAfterFailures` times in a
+   * row is dropped (side-file copy first). Breaker/backoff retention results
+   * stop the drain without counting as chunk failures.
+   *
+   * Explicit drains (replay/import `clearBufferAfterExtraction: false`, force
+   * flush, fail-closed callers, accepted captures) and windows that fit the
+   * budget (or hosts without the budget accessor) take the single-run path.
+   *
+   * `preemptable`: the run was queued as preemptable automatic work. Each chunk's
+   * persist step ends the host's preemption registration, so the drain registers
+   * it again before every later chunk; a fail-closed flush can then abort the
+   * next chunk's model request instead of waiting behind the whole window.
+   */
+  async run(turns: BufferTurn[], runOptions: DrainRunOptions, preemptable = false): Promise<ExtractionRunResult> {
+    const budgetChars = this.deps.extractionWindowBudgetChars?.() ?? Number.POSITIVE_INFINITY;
+    const chunkable =
+      runOptions.clearBufferAfterExtraction &&
+      !runOptions.forceExtractionAttempt &&
+      !runOptions.failOnExtractionFailure &&
+      !turns.some((turn) => turn.acceptedCapture === true) &&
+      Number.isFinite(budgetChars) &&
+      budgetChars > 0;
+    const chunks = chunkable ? chunkExtractionWindow(turns, budgetChars) : [];
+    const resumed =
+      chunks.length > 0 && this.qualified.get(runOptions.bufferKey) === chunkFingerprint(chunks[0].slice(0, 1));
+    // The tail of a window that already qualified goes through the chunk loop even when it now fits in one
+    // chunk, so its failures still count toward the drop threshold.
+    if (chunks.length <= 1 && !resumed) return await this.deps.runExtraction(turns, runOptions);
+    // Eligibility (minimum user turns and characters) belongs to the whole
+    // window: a window that never qualified keeps the single-run path, which
+    // judges it exactly as before.
+    if (!resumed) {
+      const windowUserTurns = turns.filter((turn) => turn.role === "user").length;
+      const belowChars = !runOptions.skipCharThreshold && this.windowChars(turns) < this.deps.config.extractionMinChars;
+      if ((!runOptions.skipUserTurnThreshold && windowUserTurns < this.deps.config.extractionMinUserTurns) || belowChars) {
+        return await this.deps.runExtraction(turns, runOptions);
+      }
+    }
+    // The window qualified as a whole; its chunks must not be re-judged one by
+    // one (an assistant-only or short chunk would be cleared unextracted).
+    const baseOptions = { ...runOptions, skipCharThreshold: true, skipUserTurnThreshold: true };
+
+    const bufferKey = runOptions.bufferKey;
+    // A native group clears its turns by identity (a pinned capture can sit in front); a drop must find them the same way.
+    const byIdentity = runOptions.clearMatchingTurns === true;
+    const dropAfter = this.deps.config.extractionChunkDropAfterFailures;
+    let aggregate: ExtractionRunResult | null = null;
+    for (const [position, chunk] of chunks.entries()) {
+      this.qualified.set(bufferKey, chunkFingerprint(chunk.slice(0, 1)));
+      // retentionWindow: the chunks processed so far. A chunk's deferred retention replaces theirs (retained turns do not pile
+      // up across chunks) and leaves the retained turns of chunks not yet processed in the buffer.
+      let chunkOptions: DrainRunOptions = { ...baseOptions, retentionWindow: chunks.slice(0, position + 1).flat() };
+      let preempt: AbortController | undefined;
+      if (preemptable && position > 0) {
+        const local = new AbortController();
+        preempt = local;
+        this.deps.extractionQueueCoordinator.beginPreemptable(() => local.abort(preemptedExtractionError()));
+        chunkOptions = {
+          ...chunkOptions,
+          abortSignal: runOptions.abortSignal ? AbortSignal.any([runOptions.abortSignal, local.signal]) : local.signal,
+        };
+      }
+      let failures = this.failureCount(bufferKey, chunk);
+      let result: ExtractionRunResult;
+      try {
+        result = await this.deps.runExtraction(chunk, chunkOptions);
+      } catch (err) {
+        // The chunk-local abort leaves the outer signal untouched, so hand back the
+        // preemption reason the host looks for (as it does for the first chunk).
+        if (isAbortError(err)) throw preempt && preempt.signal.aborted && !runOptions.abortSignal?.aborted ? preempt.signal.reason : err;
+        failures += 1;
+        if (failures < dropAfter) {
+          this.setFailureCount(bufferKey, chunk, failures);
+          throw err;
+        }
+        // A chunk that keeps throwing is dropped like one that keeps failing,
+        // so later chunks are not stuck behind it.
+        const thrown: ExtractionRunResult = {
+          status: "skipped",
+          reason: `thrown: ${describeError(err)}`,
+          persistedCount: 0,
+          durableOutputCount: 0,
+        };
+        if (!(await this.dropFailedChunk(bufferKey, chunk, dropAfter, thrown, byIdentity))) {
+          this.setFailureCount(bufferKey, chunk, failures);
+          throw err;
+        }
+        this.setFailureCount(bufferKey, chunk, 0);
+        continue;
+      }
+      const verdict = classifyDrainResult(result);
+      if (verdict === "drained") {
+        this.setFailureCount(bufferKey, chunk, 0);
+        aggregate = mergeDrainResults(aggregate, result);
+        continue;
+      }
+      // Provider breaker/backoff retention: keep everything for retry.
+      if (verdict === "deferred") return mergeDrainResults(aggregate, result);
+      // An outage or a bad key is not the chunk's fault: keep the chunk and do not count toward dropping it.
+      if (result.failureClass === "provider_retryable" || result.failureClass === "auth_config") {
+        return mergeDrainResults(aggregate, result);
+      }
+      failures += 1;
+      if (failures >= dropAfter) {
+        if (!(await this.dropFailedChunk(bufferKey, chunk, dropAfter, result, byIdentity))) {
+          // No recovery copy: keep the only buffered copy and retry later.
+          this.setFailureCount(bufferKey, chunk, failures);
+          return mergeDrainResults(aggregate, result);
+        }
+        this.setFailureCount(bufferKey, chunk, 0);
+        aggregate = mergeDrainResults(aggregate, result);
+        continue;
+      }
+      this.setFailureCount(bufferKey, chunk, failures);
+      // Retain the remaining window for the next trigger (existing behavior).
+      return mergeDrainResults(aggregate, result);
+    }
+    this.qualified.delete(bufferKey);
+    return aggregate ?? { status: "skipped", persistedCount: 0, durableOutputCount: 0 };
+  }
+
+  /** Side-file copy first (content survives a failed drop), then remove the failing chunk from the buffer. Never throws:
+   *  the drop must not fail the extraction task that already settled its result. */
+  private async dropFailedChunk(
+    bufferKey: string,
+    chunk: readonly BufferTurn[],
+    dropAfter: number,
+    result: ExtractionRunResult,
+    byIdentity: boolean,
+  ): Promise<boolean> {
+    const reason =
+      `dropped after ${dropAfter} repeated extraction failures ` +
+      `(last result: status=${result.status} reason=${result.reason ?? "n/a"})`;
+    try {
+      const storage = await this.deps.getStorage();
+      const path = await storage.saveDroppedExtractionChunk(bufferKey, reason, chunk);
+      log.warn(`buffer[${bufferKey}]: dropping ${chunk.length} turns — ${reason}; copy saved to ${path}`);
+    } catch (err) {
+      // Never destroy the only buffered copy without a recovery file.
+      log.warn(
+        `buffer[${bufferKey}]: could not save the dropped-chunk recovery copy; keeping the chunk in the buffer: ${describeError(err)}`,
+      );
+      return false;
+    }
+    try {
+      const dropped = await this.deps.buffer.dropExtractionChunk(bufferKey, chunk, { allowNonPrefix: byIdentity });
+      if (dropped > 0) {
+        log.warn(`buffer[${bufferKey}]: removed ${dropped} buffered turns — ${reason}`);
+        return true;
+      }
+      // Nothing removed (the live turns changed): the chunk is still buffered, so this is not a drop.
+      log.warn(`buffer[${bufferKey}]: dropped chunk no longer matches the buffer; keeping it and its failure count`);
+    } catch (err) {
+      log.warn(
+        `buffer[${bufferKey}]: failed to remove dropped chunk from buffer (will retry next trigger): ${describeError(err)}`,
+      );
+    }
+    return false;
+  }
+}

package/src/orchestration/extraction-run-helpers.ts

@@ -1,6 +1,7 @@
 import type { BufferTurn, ExtractionFailureClass, ExtractionResult, MetaState } from "../types.js";
 import type { StorageManager } from "../index.js";
 import { log } from "../logger.js";
+import type { ExtractionRunResult } from "./extraction-run.js";
 
 export function combineExtractionAbortSignals(
   primary: AbortSignal | undefined,
@@ -11,6 +12,13 @@ export function combineExtractionAbortSignals(
 }
 
 
+/** Result of a run whose extraction found nothing durable; `retained` marks a recorded failure that left the window in the buffer. */
+export function emptyExtractionResult(retained: boolean, failureClass?: ExtractionFailureClass): ExtractionRunResult {
+  const result: ExtractionRunResult = { status: "skipped", reason: "empty_extraction_result", persistedCount: 0, durableOutputCount: 0 };
+  if (retained) Object.assign(result, { retainedBuffer: true, failureClass: failureClass ?? "provider_retryable" }); // same default as the retry bookkeeping
+  return result;
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null;
 }

package/src/orchestration/extraction-run.test.ts

@@ -88,6 +88,7 @@ const migrationSummary: TierMigrationCycleSummary = {
 
 interface Harness {
   config: PluginConfig;
+  buffer: SmartBuffer;
   storageForNs: (ns: string) => Promise<StorageManager>;
   setStorageForDelay: (delayMs: number) => void;
   newCoordinator: () => ExtractionRunCoordinator;
@@ -105,7 +106,7 @@ interface Harness {
     coord: ExtractionRunCoordinator,
     content: string,
     opts?: { force?: boolean; failClosed?: boolean; writeNamespaceOverride?: string },
-  ) => Promise<{ status: string; reason?: string }>;
+  ) => Promise<{ status: string; reason?: string; failureClass?: string; retainedBuffer?: boolean }>;
   cleanup: () => Promise<void>;
   passiveCapture: () => { principal?: string; namespace?: string } | null;
   setThreadingBlock: (blocked: boolean) => void;
@@ -201,6 +202,7 @@ async function makeHarness(overrides: Record<string, unknown> = {}): Promise<Har
 
   return {
     config,
+    buffer,
     storageForNs,
     setStorageForDelay: (delayMs) => {
       storageForDelayMs = delayMs;
@@ -243,7 +245,7 @@ async function makeHarness(overrides: Record<string, unknown> = {}): Promise<Har
         forceExtractionAttempt: opts.force === true,
         writeNamespaceOverride: opts.writeNamespaceOverride,
       });
-      return { status: result.status, reason: result.reason };
+      return { status: result.status, reason: result.reason, failureClass: result.failureClass, retainedBuffer: result.retainedBuffer };
     },
     cleanup: async () => {
       StorageManager.clearAllStaticCaches();
@@ -1336,3 +1338,82 @@ test("#3140: a successful flush heals the recorded LLM health", async () => {
     await h.cleanup();
   }
 });
+
+// ---------------------------------------------------------------------------
+// Wiring the trigger drain depends on (#3266): what a real run hands back, and what it asks of the buffer
+// ---------------------------------------------------------------------------
+
+test("a retained failure carries the extraction's failure class, so the drain can tell an outage from a bad chunk", async () => {
+  for (const cls of ["provider_retryable", "auth_config", "parse_empty"] as const) {
+    const h = await makeHarness({ extractionRetryScheduleMs: [3_600_000], extractionBreakerFailureThreshold: 100 });
+    try {
+      h.setRespond(() => failureResult(cls));
+      const result = await h.run(h.newCoordinator(), `class-${cls}`);
+      assert.equal(result.retainedBuffer, true, `${cls}: the window is kept for a retry`);
+      assert.equal(result.failureClass, cls, `${cls}: the class reaches the run result`);
+    } finally {
+      await h.cleanup();
+    }
+  }
+});
+
+test("a failure with no class defaults to provider_retryable on the run result, like the retry bookkeeping", async () => {
+  const h = await makeHarness({ extractionRetryScheduleMs: [3_600_000], extractionBreakerFailureThreshold: 100 });
+  try {
+    h.setRespond(() => ({ ...failureResult("parse_empty"), extractionFailureClass: undefined }));
+    const result = await h.run(h.newCoordinator(), "class-default");
+    assert.equal(result.failureClass, "provider_retryable");
+  } finally {
+    await h.cleanup();
+  }
+});
+
+test("a run without a failure reports no failure class", async () => {
+  const h = await makeHarness();
+  try {
+    h.setRespond(() => ({ ...successResult(), facts: [] }));
+    const result = await h.run(h.newCoordinator(), "class-none");
+    assert.equal(result.failureClass, undefined);
+  } finally {
+    await h.cleanup();
+  }
+});
+
+test("the retention window a chunk run is given reaches the buffer as the processed snapshot (#3266)", async () => {
+  const h = await makeHarness();
+  try {
+    const seen: Array<readonly BufferTurn[] | undefined> = [];
+    const original = h.buffer.retainDeferredTurns.bind(h.buffer);
+    h.buffer.retainDeferredTurns = async (key, turns, max, processed) => {
+      seen.push(processed);
+      await original(key, turns, max, processed);
+    };
+    const chunk = makeTurns("chunk-turns");
+    const window = [...makeTurns("earlier-turns"), ...chunk];
+    h.setRespond(() => successResult());
+    await h.newCoordinator().runExtraction(chunk, {
+      skipCharThreshold: true,
+      skipUserTurnThreshold: true,
+      clearBufferAfterExtraction: true,
+      bufferKey: "window",
+      failOnExtractionFailure: false,
+      forceExtractionAttempt: false,
+      retentionWindow: window,
+    });
+    assert.equal(seen.length, 1, "the run asked the buffer to update its retention slot once");
+    assert.deepEqual(seen[0], window, "with the window it was given, not just its own chunk");
+
+    seen.length = 0;
+    await h.newCoordinator().runExtraction(chunk, {
+      skipCharThreshold: true,
+      skipUserTurnThreshold: true,
+      clearBufferAfterExtraction: true,
+      bufferKey: "window",
+      failOnExtractionFailure: false,
+      forceExtractionAttempt: false,
+    });
+    assert.deepEqual(seen[0], chunk, "without a window the run keeps naming its own turns");
+  } finally {
+    await h.cleanup();
+  }
+});

package/src/orchestration/extraction-run.ts

@@ -38,6 +38,7 @@ export interface ExtractionRunResult {
   persistedCount: number;
   durableOutputCount: number;
   postPersistMetadataFailed?: boolean;
+  retainedBuffer?: boolean; failureClass?: ExtractionFailureClass; // retained: the run left the window in the buffer for a retry (a recorded failure) of this class
 }
 
 export class ExtractionDeadlineError extends Error {
@@ -118,7 +119,7 @@ import {
   commitEmptySuccessExtraction,
   computeExtractionRetryNextEligibleMs,
   deriveSourceConnector,
-  deriveTopicsFromExtraction,
+  deriveTopicsFromExtraction, emptyExtractionResult,
   runExtractionPostPersist,
   runExtractionPostPersistBestEffort,
   EXTRACTION_RETRY_STATE_MAX_ENTRIES,
@@ -439,7 +440,7 @@ export class ExtractionRunCoordinator {
     turns: BufferTurn[],
     options: {
       clearBufferAfterExtraction?: boolean;
-      clearMatchingTurns?: boolean;
+      clearMatchingTurns?: boolean; retentionWindow?: readonly BufferTurn[];
       skipCharThreshold?: boolean;
       skipUserTurnThreshold?: boolean;
       deadlineMs?: number;
@@ -1044,7 +1045,7 @@ export class ExtractionRunCoordinator {
       if (!shouldPersistProcessedFingerprint) {
         await commitEmptySuccessExtraction(emptySuccessCommit);
       }
-      return { status: "skipped", reason: "empty_extraction_result", persistedCount: 0, durableOutputCount: 0 };
+      return emptyExtractionResult(recordedRetryFailure, result.extractionFailureClass);
     }
 
     let threadIdForExtraction: string | null = null;
@@ -1108,7 +1109,7 @@ export class ExtractionRunCoordinator {
       normalizedTurns: normalizedTurns as BufferTurn[],
       bufferKey,
       retainDeferredTurns: (key, retainedTurns, max) =>
-        this.deps.getBuffer().retainDeferredTurns(key, retainedTurns, max, turns),
+        this.deps.getBuffer().retainDeferredTurns(key, retainedTurns, max, options.retentionWindow ?? turns),
       clearBuffer,
       failOnExtractionFailure: options.failOnExtractionFailure === true,
     });

package/src/orchestration/qmd-result-resolver.ts

@@ -26,12 +26,13 @@
  * RecallRerankCoordinator / ConversationIndexCoordinator accessor pattern.
  */
 
+import { existsSync } from "node:fs";
 import path from "node:path";
 import { resolveNamespaceCapabilities } from "../capabilities.js";
 import type { StorageManager } from "../index.js";
 import { log } from "../logger.js";
 import { namespaceIdentityFromToken } from "../namespaces/identity.js";
-import { normalizeQmdResultPath } from "../namespaces/search.js";
+import { normalizeQmdResultPath, resolveNamespaceResultPath } from "../namespaces/search.js";
 import { SecureStoreLockedError } from "../secure-store/index.js";
 import { isPathInsideStorageRoot } from "../storage-paths.js";
 import { isSupportPassportPrivateMemory } from "../support-passport/card-projection.js";
@@ -136,6 +137,30 @@ export function qmdResultPathCandidates(storageDir: string, resultPath: string):
   return [...candidates];
 }
 
+/** Canonicalize flat hot-memory identities before merging with native hits (#3194).
+ * This is a path-only map: no candidates, collections or namespace scopes are removed.
+ * Cold/explicit foreign collections keep their existing owner-resolution path.
+ */
+export function normalizeFlatQmdResults(
+  results: QmdSearchResult[],
+  config: PluginConfig,
+  collection = config.qmdCollection,
+): QmdSearchResult[] {
+  if (!collection || resolveNamespaceCapabilities(config).namespaces || collection !== config.qmdCollection) return results;
+  return results.map((result) => ({
+    ...result,
+    path: withHotFactsPath(config.memoryDir, resolveNamespaceResultPath(config.memoryDir, collection, result.path)),
+  }));
+}
+
+/** A hot-facts collection registered at the facts/ subtree returns date-relative paths (#2111); meet the loaded memory's path. */
+function withHotFactsPath(memoryDir: string, resolved: string): string {
+  const relative = path.relative(path.resolve(memoryDir), resolved).split(path.sep).join("/");
+  if (!/^\d{4}-\d{2}-\d{2}\//.test(relative) || existsSync(resolved)) return resolved;
+  const hot = path.join(path.resolve(memoryDir), "facts", relative);
+  return existsSync(hot) ? hot : resolved;
+}
+
 /**
  * Extract a non-empty `dir` from a storage, returning `null` when the
  * property is absent or not a non-empty string at runtime. Proxy-backed

package/src/orchestration/recall-search-pipeline.ts

@@ -27,7 +27,7 @@ import { inferIntentFromText, intentCompatibilityScore } from "../intent.js";
 import { log } from "../logger.js";
 import { NamespaceStorageRouter } from "../namespaces/storage.js";
 import { NegativeExampleStore } from "../negative.js";
-import { qmdCollectionPathParts } from "./qmd-result-resolver.js";
+import { normalizeFlatQmdResults, qmdCollectionPathParts } from "./qmd-result-resolver.js";
 import type { RecallRerankCoordinator, RecallResultPartitionSink } from "./recall-rerank-coordinator.js";
 import type { GraphRecallExpansionOptions, GraphRecallExpansionResult } from "./graph-recall-coordinator.js";
 import { RelevanceStore } from "../relevance.js";
@@ -394,7 +394,7 @@ export class RecallSearchPipelineCoordinator {
         break;
       }
 
-      const primaryResults = options.collection
+      let primaryResults = options.collection
         ? options.abortSignal
           ? await this.deps.qmd.search(
               prompt,
@@ -426,6 +426,7 @@ export class RecallSearchPipelineCoordinator {
               onDegradation: options.onDegradation,
             },
           });
+      primaryResults = normalizeFlatQmdResults(primaryResults, this.deps.config, options.collection);
       lastPrimaryResultCount = primaryResults.length;
       lastHybridResultCount = 0;
       lastHybridTopUpUsed = false;
@@ -452,7 +453,7 @@ export class RecallSearchPipelineCoordinator {
           // uniform across primary + top-up (gotcha #39). Issue #1335 (codex review #1422).
           lastHybridTopUpSkippedReason = "lex_strategy";
         } else {
-          const hybridResults = options.collection
+          let hybridResults = options.collection
             ? await this.deps.qmd.hybridSearch(
                 prompt,
                 options.collection,
@@ -474,6 +475,7 @@ export class RecallSearchPipelineCoordinator {
                   onDegradation: options.onDegradation,
                 },
               });
+          hybridResults = normalizeFlatQmdResults(hybridResults, this.deps.config, options.collection);
           lastHybridResultCount = hybridResults.length;
           lastHybridTopUpUsed = hybridResults.length > 0;
           if (hybridResults.length > 0) {
@@ -740,9 +742,7 @@ export class RecallSearchPipelineCoordinator {
         // limps home later cannot mutate a recorded recall's collector.
         coldQmdObserverActive = false;
         if (longTerm.length > 0) {
-          log.debug(
-            `cold-tier recall source=cold-qmd collection=${coldCollection} hits=${longTerm.length}`,
-          );
+          log.debug(`cold-tier recall source=cold-qmd collection=${coldCollection} hits=${longTerm.length}`);
         }
       }
     }
@@ -940,9 +940,7 @@ export class RecallSearchPipelineCoordinator {
       }
     }
     if (caps.rerank && this.deps.config.rerankProvider === "cloud") {
-      log.debug(
-        "rerankProvider=cloud is reserved/experimental in v2.2.0; skipping rerank",
-      );
+      log.debug("rerankProvider=cloud is reserved/experimental in v2.2.0; skipping rerank");
     }
 
     if (caps.recallTrustScore && results.length > 0) {
@@ -1149,6 +1147,9 @@ export class RecallSearchPipelineCoordinator {
       : null;
     const buildRecallRow = (r: QmdSearchResult, memory: MemoryFile | undefined): QmdSearchResult => ({
       ...r,
+      // Render loaded flat memories, not QMD's front-matter/chunk preview (#3194).
+      snippet: memory && !resolveNamespaceCapabilities(this.deps.config).namespaces ? memory.content : r.snippet,
+      line: memory && !resolveNamespaceCapabilities(this.deps.config).namespaces ? undefined : r.line,
       sourceConnector: memory ? memory.frontmatter.sourceConnector : undefined,
       origin: memory ? memory.frontmatter.origin : undefined,
       // #1952: an active state view carries the chain fields the inject
@@ -1358,9 +1359,7 @@ export class RecallSearchPipelineCoordinator {
       }
     }
     if (connectorPartitionFilteredCount > 0) {
-      log.debug(
-        `connector partition filter removed ${connectorPartitionFilteredCount} tool-scoped candidates from recall`,
-      );
+      log.debug(`connector partition filter removed ${connectorPartitionFilteredCount} tool-scoped candidates from recall`);
     }
     if (supportPassportFilteredCount > 0) {
       log.debug(

package/src/orchestration/turn-ingestion.ts

@@ -42,6 +42,7 @@ import { TranscriptManager } from "../transcript.js";
 import type { BufferTurn, PluginConfig, SourceConnectorProvenance } from "../types.js";
 
 import { groupTurnsByExtractionScope, bufferTurnsEqual } from "../buffer-turn-helpers.js";
+import { ExtractionDrain } from "./extraction-drain.js";
 import { resolveScopeProfilePlan, type ResolvedScopeProfilePlan, hasExplicitNamespacePolicy } from "../namespaces/scope-profiles.js";
 import { resolveCodingNamespaceOverlay, combineNamespaces } from "../coding/coding-namespace.js";
 import { canWriteNamespace, defaultNamespaceForPrincipal } from "../namespaces/principal.js";
@@ -114,6 +115,8 @@ export interface TurnIngestionDeps {
     },
   ): Promise<void>;
   resolveMemoryIdOrHandle(ref: string, sessionKey?: string): string;
+  /** Max conversation chars the extraction model accepts; absent on legacy hosts (single-run behavior). */
+  extractionWindowBudgetChars?(): number;
   runExtraction(
     ...args: Parameters<ExtractionRunCoordinator["runExtraction"]>
   ): Promise<ExtractionRunResult>;
@@ -156,9 +159,10 @@ function isFlushPlanRecoveryTurn(
 }
 
 export class TurnIngestionCoordinator {
-  constructor(
-    private readonly deps: TurnIngestionDeps,
-  ) {}
+  private readonly drain: ExtractionDrain; // chunked trigger-drain (extraction-drain.ts)
+  constructor(private readonly deps: TurnIngestionDeps) {
+    this.drain = new ExtractionDrain(deps);
+  }
 
   async processTurn(
     role: "user" | "assistant",
@@ -1067,7 +1071,7 @@ export class TurnIngestionCoordinator {
             effectiveScopePlan = fresh;
           }
         }
-        const result = await this.deps.runExtraction(effectiveTurns, {
+        const result = await this.drain.run(effectiveTurns, {
           clearBufferAfterExtraction:
             options.clearBufferAfterExtraction ?? true,
           clearMatchingTurns: options.clearMatchingTurns === true,
@@ -1083,7 +1087,7 @@ export class TurnIngestionCoordinator {
           principalOverride: options.principalOverride,
           scopeProfileWritePlan: effectiveScopePlan,
           forceExtractionAttempt: options.forceExtractionAttempt === true,
-        });
+        }, preemptable);
         settleTask(undefined, result);
       } catch (err) {
         const reason = abortSignal.reason;

package/src/orchestrator.ts

@@ -238,6 +238,7 @@ import {
 } from "./maintenance/pattern-reinforcement.js";
 import { refreshConvergedNamespaces } from "./convergence-refresh.js";
 import { ModelRegistry } from "./model-registry.js";
+import { extractionWindowBudgetChars } from "./extraction-window-drain.js";
 import { applyRuntimeRetrievalPolicy, expandQuery } from "./retrieval.js";
 import {
   mergeWithAgentResults,
@@ -3062,6 +3063,26 @@ export class Orchestrator {
     );
   }
 
+  /**
+   * Max conversation characters the extraction model can take as input
+   * (context window the ModelRegistry learned, capped by
+   * `localLlmMaxContext` when set). Exposed for the trigger-drain chunking
+   * in the turn-ingestion coordinator so an oversized buffer window is
+   * drained in fitting chunks instead of failing whole.
+   */
+  private extractionWindowBudgetChars(): number {
+    // Object.create(Orchestrator.prototype) test doubles skip the constructor,
+    // so neither modelRegistry nor config exists there: report "no budget".
+    if (!this.modelRegistry) return Number.POSITIVE_INFINITY;
+    return extractionWindowBudgetChars({
+      registry: this.modelRegistry,
+      localExtractionActive:
+        this.config.modelSource !== "gateway" && resolveLocalLlmCapabilities(this.config).localLlm,
+      model: this.config.localLlmModel,
+      maxContext: this.config.localLlmMaxContext,
+    });
+  }
+
   private normalizeExtractionFingerprintTurns(turns: BufferTurn[]): string[] {
     return this.extractionRunCoordinator.normalizeExtractionFingerprintTurns(turns);
   }

package/src/secure-store/secure-fs.ts

@@ -902,7 +902,9 @@ const ENCRYPTABLE_STATE_SIDECARS = new Set([
 ]);
 
 function isEncryptableStateSidecar(normalized: string): boolean {
-  return ENCRYPTABLE_STATE_SIDECARS.has(normalized);
+  // state/dropped/ holds copies of extraction-buffer turns dropped after
+  // repeated extraction failures — same content sensitivity as buffer.json.
+  return ENCRYPTABLE_STATE_SIDECARS.has(normalized) || normalized.startsWith("state/dropped/");
 }
 
 /**