@remnic/core 10.0.38 → 10.0.39

package/src/extraction-window-drain.ts

@@ -0,0 +1,110 @@
+/**
+ * Trigger-drain helpers (extraction trigger stall fix).
+ *
+ * A session whose buffered window outgrows the extraction model's context
+ * never drains: the trigger fires `extract_batch` on every observe, the
+ * oversized request fails, nothing is cleared, and the window only grows.
+ * These helpers split such a window into budget-fitting chunks and classify
+ * each extraction run so the caller can drain successes, defer on breaker/
+ * backoff retention, and drop repeatedly failing chunks.
+ */
+
+import type { ExtractionRunResult } from "./orchestration/extraction-run.js";
+
+/**
+ * Split `turns` into chunks whose summed content length stays within
+ * `maxChars`. Chunks keep turn order and never split a turn; a single turn
+ * longer than the budget becomes its own chunk (the extraction prompt still
+ * clamps per-turn content). A non-positive or non-finite budget, or an empty
+ * window, yields at most one chunk (no chunking).
+ */
+export function chunkExtractionWindow<T extends { content: string }>(
+  turns: readonly T[],
+  maxChars: number,
+): T[][] {
+  if (turns.length === 0) return [];
+  const budget =
+    Number.isFinite(maxChars) && maxChars > 0 ? Math.floor(maxChars) : Number.POSITIVE_INFINITY;
+  const chunks: T[][] = [];
+  let current: T[] = [];
+  let currentChars = 0;
+  for (const turn of turns) {
+    const turnChars = turn.content.length;
+    if (current.length > 0 && currentChars + turnChars > budget) {
+      chunks.push(current);
+      current = [];
+      currentChars = 0;
+    }
+    current.push(turn);
+    currentChars += turnChars;
+  }
+  if (current.length > 0) chunks.push(current);
+  return chunks;
+}
+
+export type DrainVerdict = "drained" | "deferred" | "failed";
+
+/**
+ * Run results that deliberately RETAIN the buffer (provider breaker open,
+ * per-fingerprint backoff) are deferrals, not chunk failures — counting them
+ * would drop good content during a provider outage.
+ */
+const DRAIN_DEFERRAL_REASONS = {
+  provider_circuit_open: true,
+  extraction_backoff: true,
+};
+
+export function classifyDrainResult(result: ExtractionRunResult): DrainVerdict {
+  if (result.status !== "skipped") return "drained";
+  // An empty result is a failure only when the run kept the window for retry;
+  // otherwise it extracted fine, found nothing durable, and cleared its turns.
+  if (result.reason === "empty_extraction_result") {
+    return result.retainedBuffer === true ? "failed" : "drained";
+  }
+  if (result.reason !== undefined && result.reason in DRAIN_DEFERRAL_REASONS) return "deferred";
+  return "drained";
+}
+
+/**
+ * Combine chunk results into the single `ExtractionRunResult` the queue
+ * contract settles with: counts sum, any completed chunk completes the drain,
+ * and the last non-completed reason wins.
+ */
+export function mergeDrainResults(
+  previous: ExtractionRunResult | null,
+  next: ExtractionRunResult,
+): ExtractionRunResult {
+  if (previous === null) return next;
+  const completed = previous.status === "completed" || next.status === "completed";
+  const merged: ExtractionRunResult = {
+    status: completed ? "completed" : "skipped",
+    persistedCount: previous.persistedCount + next.persistedCount,
+    durableOutputCount: previous.durableOutputCount + next.durableOutputCount,
+  };
+  if (!completed) merged.reason = next.reason ?? previous.reason;
+  if (previous.postPersistMetadataFailed === true || next.postPersistMetadataFailed === true) {
+    merged.postPersistMetadataFailed = true;
+  }
+  return merged;
+}
+
+/** The slice of ModelRegistry the drain budget needs. */
+export interface WindowBudgetRegistry {
+  calculateContextSizes(modelId: string, maxContextOverride?: number): { maxInputChars: number };
+}
+
+/**
+ * Character budget for one extraction request. The learned local-model window
+ * only describes the extractor when the local LLM does the extracting; with a
+ * gateway extractor, a disabled local LLM, or no registry (legacy hosts, test
+ * doubles) there is no budget and the window runs in one request as before.
+ */
+export function extractionWindowBudgetChars(input: {
+  registry: WindowBudgetRegistry | undefined;
+  localExtractionActive: boolean;
+  model: string;
+  maxContext: number | undefined;
+}): number {
+  if (!input.registry || !input.localExtractionActive) return Number.POSITIVE_INFINITY;
+  return input.registry.calculateContextSizes(input.model, input.maxContext).maxInputChars;
+}

package/src/orchestration/extraction-drain.ts

@@ -0,0 +1,261 @@
+/**
+ * Chunked trigger-drain for the extraction pipeline.
+ *
+ * Split out of `turn-ingestion.ts`: that file is at the structural ratchet's
+ * line cap (issue #1995), and the drain is a self-contained unit — it only
+ * needs the host's `runExtraction`, the buffer drop, the storage side-file
+ * copy and the preemption hook.
+ */
+
+import { createHash } from "node:crypto";
+import { isAbortError, preemptedExtractionError } from "../abort-error.js";
+import { describeError } from "../buffer-turn-helpers.js";
+import { chunkExtractionWindow, classifyDrainResult, mergeDrainResults } from "../extraction-window-drain.js";
+import { log } from "../logger.js";
+import type { BufferTurn } from "../types.js";
+import type { ExtractionRunCoordinator, ExtractionRunResult } from "./extraction-run.js";
+
+type RunOptions = Parameters<ExtractionRunCoordinator["runExtraction"]>[1];
+
+/** The `runExtraction` options the drain forwards, with the ones it reads made required. */
+export type DrainRunOptions = RunOptions & {
+  bufferKey: string;
+  clearBufferAfterExtraction: boolean;
+  skipUserTurnThreshold: boolean;
+  failOnExtractionFailure: boolean;
+  forceExtractionAttempt: boolean;
+};
+
+/** What the drain needs from its host (structurally satisfied by `TurnIngestionDeps`). */
+export interface ExtractionDrainDeps {
+  config: {
+    extractionMinUserTurns: number;
+    extractionMinChars: number;
+    extractionMaxTurnChars: number;
+    extractionChunkDropAfterFailures: number;
+  };
+  extractionWindowBudgetChars?(): number;
+  runExtraction(turns: BufferTurn[], options: RunOptions): Promise<ExtractionRunResult>;
+  getStorage(): Promise<{
+    saveDroppedExtractionChunk(bufferKey: string, reason: string, turns: readonly BufferTurn[]): Promise<string>;
+  }>;
+  buffer: {
+    dropExtractionChunk(bufferKey: string, turns: readonly BufferTurn[], options?: { allowNonPrefix?: boolean }): Promise<number>;
+  };
+  extractionQueueCoordinator: { beginPreemptable(abort: () => void): void };
+}
+
+/** Identity of a chunk: its failure count must not follow a buffer key to different content. */
+function chunkFingerprint(chunk: readonly BufferTurn[]): string {
+  const hash = createHash("sha256");
+  for (const turn of chunk) hash.update(`${turn.role}\u0000${turn.timestamp}\u0000${turn.content}\u0001`);
+  return hash.digest("hex");
+}
+
+export class ExtractionDrain {
+  /**
+   * Consecutive failed attempts on the head chunk, per buffer key. In memory on
+   * purpose (a restart re-counts from zero). Bound to the chunk's content: a
+   * count recorded for one chunk never applies to a different one, so a force
+   * flush that clears a failing chunk cannot make unrelated content inherit its
+   * failures.
+   */
+  private readonly failures = new Map<string, { fingerprint: string; count: number }>();
+
+  /**
+   * Buffer key to the first turn of the chunk a qualified window stopped on.
+   * The rest of such a window stays eligible on the next attempt even when it
+   * is assistant-only or short: the window passed the thresholds as a whole,
+   * and re-judging its tail would clear it unextracted. Cleared when the
+   * window drains; one entry per key.
+   */
+  private readonly qualified = new Map<string, string>();
+
+  constructor(private readonly deps: ExtractionDrainDeps) {}
+
+  /** Characters the coordinator would count for this window: user/assistant text, trimmed and capped per turn, context-only turns left out. */
+  private windowChars(turns: readonly BufferTurn[]): number {
+    let total = 0;
+    for (const turn of turns) {
+      if ((turn.role !== "user" && turn.role !== "assistant") || turn.extractionContextOnly === true) continue;
+      total += turn.content.trim().slice(0, this.deps.config.extractionMaxTurnChars).length;
+    }
+    return total;
+  }
+
+  private failureCount(bufferKey: string, chunk: readonly BufferTurn[]): number {
+    const entry = this.failures.get(bufferKey);
+    return entry && entry.fingerprint === chunkFingerprint(chunk) ? entry.count : 0;
+  }
+
+  private setFailureCount(bufferKey: string, chunk: readonly BufferTurn[], count: number): void {
+    if (count <= 0) this.failures.delete(bufferKey);
+    else this.failures.set(bufferKey, { fingerprint: chunkFingerprint(chunk), count });
+  }
+
+  /**
+   * Run `runExtraction` over the trigger-drain window. When the window outgrows
+   * the extraction model's input budget (the stall that keeps a session's buffer
+   * growing forever: every observe re-fires the trigger on a window that can
+   * never extract), split it into budget-fitting chunks and drain them in
+   * order; each successful run clears its chunk, so the buffer shrinks across
+   * observes. A chunk that fails `extractionChunkDropAfterFailures` times in a
+   * row is dropped (side-file copy first). Breaker/backoff retention results
+   * stop the drain without counting as chunk failures.
+   *
+   * Explicit drains (replay/import `clearBufferAfterExtraction: false`, force
+   * flush, fail-closed callers, accepted captures) and windows that fit the
+   * budget (or hosts without the budget accessor) take the single-run path.
+   *
+   * `preemptable`: the run was queued as preemptable automatic work. Each chunk's
+   * persist step ends the host's preemption registration, so the drain registers
+   * it again before every later chunk; a fail-closed flush can then abort the
+   * next chunk's model request instead of waiting behind the whole window.
+   */
+  async run(turns: BufferTurn[], runOptions: DrainRunOptions, preemptable = false): Promise<ExtractionRunResult> {
+    const budgetChars = this.deps.extractionWindowBudgetChars?.() ?? Number.POSITIVE_INFINITY;
+    const chunkable =
+      runOptions.clearBufferAfterExtraction &&
+      !runOptions.forceExtractionAttempt &&
+      !runOptions.failOnExtractionFailure &&
+      !turns.some((turn) => turn.acceptedCapture === true) &&
+      Number.isFinite(budgetChars) &&
+      budgetChars > 0;
+    const chunks = chunkable ? chunkExtractionWindow(turns, budgetChars) : [];
+    const resumed =
+      chunks.length > 0 && this.qualified.get(runOptions.bufferKey) === chunkFingerprint(chunks[0].slice(0, 1));
+    // The tail of a window that already qualified goes through the chunk loop even when it now fits in one
+    // chunk, so its failures still count toward the drop threshold.
+    if (chunks.length <= 1 && !resumed) return await this.deps.runExtraction(turns, runOptions);
+    // Eligibility (minimum user turns and characters) belongs to the whole
+    // window: a window that never qualified keeps the single-run path, which
+    // judges it exactly as before.
+    if (!resumed) {
+      const windowUserTurns = turns.filter((turn) => turn.role === "user").length;
+      const belowChars = !runOptions.skipCharThreshold && this.windowChars(turns) < this.deps.config.extractionMinChars;
+      if ((!runOptions.skipUserTurnThreshold && windowUserTurns < this.deps.config.extractionMinUserTurns) || belowChars) {
+        return await this.deps.runExtraction(turns, runOptions);
+      }
+    }
+    // The window qualified as a whole; its chunks must not be re-judged one by
+    // one (an assistant-only or short chunk would be cleared unextracted).
+    const baseOptions = { ...runOptions, skipCharThreshold: true, skipUserTurnThreshold: true };
+
+    const bufferKey = runOptions.bufferKey;
+    // A native group clears its turns by identity (a pinned capture can sit in front); a drop must find them the same way.
+    const byIdentity = runOptions.clearMatchingTurns === true;
+    const dropAfter = this.deps.config.extractionChunkDropAfterFailures;
+    let aggregate: ExtractionRunResult | null = null;
+    for (const [position, chunk] of chunks.entries()) {
+      this.qualified.set(bufferKey, chunkFingerprint(chunk.slice(0, 1)));
+      // retentionWindow: the chunks processed so far. A chunk's deferred retention replaces theirs (retained turns do not pile
+      // up across chunks) and leaves the retained turns of chunks not yet processed in the buffer.
+      let chunkOptions: DrainRunOptions = { ...baseOptions, retentionWindow: chunks.slice(0, position + 1).flat() };
+      let preempt: AbortController | undefined;
+      if (preemptable && position > 0) {
+        const local = new AbortController();
+        preempt = local;
+        this.deps.extractionQueueCoordinator.beginPreemptable(() => local.abort(preemptedExtractionError()));
+        chunkOptions = {
+          ...chunkOptions,
+          abortSignal: runOptions.abortSignal ? AbortSignal.any([runOptions.abortSignal, local.signal]) : local.signal,
+        };
+      }
+      let failures = this.failureCount(bufferKey, chunk);
+      let result: ExtractionRunResult;
+      try {
+        result = await this.deps.runExtraction(chunk, chunkOptions);
+      } catch (err) {
+        // The chunk-local abort leaves the outer signal untouched, so hand back the
+        // preemption reason the host looks for (as it does for the first chunk).
+        if (isAbortError(err)) throw preempt && preempt.signal.aborted && !runOptions.abortSignal?.aborted ? preempt.signal.reason : err;
+        failures += 1;
+        if (failures < dropAfter) {
+          this.setFailureCount(bufferKey, chunk, failures);
+          throw err;
+        }
+        // A chunk that keeps throwing is dropped like one that keeps failing,
+        // so later chunks are not stuck behind it.
+        const thrown: ExtractionRunResult = {
+          status: "skipped",
+          reason: `thrown: ${describeError(err)}`,
+          persistedCount: 0,
+          durableOutputCount: 0,
+        };
+        if (!(await this.dropFailedChunk(bufferKey, chunk, dropAfter, thrown, byIdentity))) {
+          this.setFailureCount(bufferKey, chunk, failures);
+          throw err;
+        }
+        this.setFailureCount(bufferKey, chunk, 0);
+        continue;
+      }
+      const verdict = classifyDrainResult(result);
+      if (verdict === "drained") {
+        this.setFailureCount(bufferKey, chunk, 0);
+        aggregate = mergeDrainResults(aggregate, result);
+        continue;
+      }
+      // Provider breaker/backoff retention: keep everything for retry.
+      if (verdict === "deferred") return mergeDrainResults(aggregate, result);
+      // An outage or a bad key is not the chunk's fault: keep the chunk and do not count toward dropping it.
+      if (result.failureClass === "provider_retryable" || result.failureClass === "auth_config") {
+        return mergeDrainResults(aggregate, result);
+      }
+      failures += 1;
+      if (failures >= dropAfter) {
+        if (!(await this.dropFailedChunk(bufferKey, chunk, dropAfter, result, byIdentity))) {
+          // No recovery copy: keep the only buffered copy and retry later.
+          this.setFailureCount(bufferKey, chunk, failures);
+          return mergeDrainResults(aggregate, result);
+        }
+        this.setFailureCount(bufferKey, chunk, 0);
+        aggregate = mergeDrainResults(aggregate, result);
+        continue;
+      }
+      this.setFailureCount(bufferKey, chunk, failures);
+      // Retain the remaining window for the next trigger (existing behavior).
+      return mergeDrainResults(aggregate, result);
+    }
+    this.qualified.delete(bufferKey);
+    return aggregate ?? { status: "skipped", persistedCount: 0, durableOutputCount: 0 };
+  }
+
+  /** Side-file copy first (content survives a failed drop), then remove the failing chunk from the buffer. Never throws:
+   *  the drop must not fail the extraction task that already settled its result. */
+  private async dropFailedChunk(
+    bufferKey: string,
+    chunk: readonly BufferTurn[],
+    dropAfter: number,
+    result: ExtractionRunResult,
+    byIdentity: boolean,
+  ): Promise<boolean> {
+    const reason =
+      `dropped after ${dropAfter} repeated extraction failures ` +
+      `(last result: status=${result.status} reason=${result.reason ?? "n/a"})`;
+    try {
+      const storage = await this.deps.getStorage();
+      const path = await storage.saveDroppedExtractionChunk(bufferKey, reason, chunk);
+      log.warn(`buffer[${bufferKey}]: dropping ${chunk.length} turns — ${reason}; copy saved to ${path}`);
+    } catch (err) {
+      // Never destroy the only buffered copy without a recovery file.
+      log.warn(
+        `buffer[${bufferKey}]: could not save the dropped-chunk recovery copy; keeping the chunk in the buffer: ${describeError(err)}`,
+      );
+      return false;
+    }
+    try {
+      const dropped = await this.deps.buffer.dropExtractionChunk(bufferKey, chunk, { allowNonPrefix: byIdentity });
+      if (dropped > 0) {
+        log.warn(`buffer[${bufferKey}]: removed ${dropped} buffered turns — ${reason}`);
+        return true;
+      }
+      // Nothing removed (the live turns changed): the chunk is still buffered, so this is not a drop.
+      log.warn(`buffer[${bufferKey}]: dropped chunk no longer matches the buffer; keeping it and its failure count`);
+    } catch (err) {
+      log.warn(
+        `buffer[${bufferKey}]: failed to remove dropped chunk from buffer (will retry next trigger): ${describeError(err)}`,
+      );
+    }
+    return false;
+  }
+}

package/src/orchestration/extraction-run-helpers.ts

@@ -1,6 +1,7 @@
 import type { BufferTurn, ExtractionFailureClass, ExtractionResult, MetaState } from "../types.js";
 import type { StorageManager } from "../index.js";
 import { log } from "../logger.js";
+import type { ExtractionRunResult } from "./extraction-run.js";
 
 export function combineExtractionAbortSignals(
   primary: AbortSignal | undefined,
@@ -11,6 +12,13 @@ export function combineExtractionAbortSignals(
 }
 
 
+/** Result of a run whose extraction found nothing durable; `retained` marks a recorded failure that left the window in the buffer. */
+export function emptyExtractionResult(retained: boolean, failureClass?: ExtractionFailureClass): ExtractionRunResult {
+  const result: ExtractionRunResult = { status: "skipped", reason: "empty_extraction_result", persistedCount: 0, durableOutputCount: 0 };
+  if (retained) Object.assign(result, { retainedBuffer: true, failureClass: failureClass ?? "provider_retryable" }); // same default as the retry bookkeeping
+  return result;
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null;
 }

package/src/orchestration/extraction-run.ts

@@ -38,6 +38,7 @@ export interface ExtractionRunResult {
   persistedCount: number;
   durableOutputCount: number;
   postPersistMetadataFailed?: boolean;
+  retainedBuffer?: boolean; failureClass?: ExtractionFailureClass; // retained: the run left the window in the buffer for a retry (a recorded failure) of this class
 }
 
 export class ExtractionDeadlineError extends Error {
@@ -118,7 +119,7 @@ import {
   commitEmptySuccessExtraction,
   computeExtractionRetryNextEligibleMs,
   deriveSourceConnector,
-  deriveTopicsFromExtraction,
+  deriveTopicsFromExtraction, emptyExtractionResult,
   runExtractionPostPersist,
   runExtractionPostPersistBestEffort,
   EXTRACTION_RETRY_STATE_MAX_ENTRIES,
@@ -439,7 +440,7 @@ export class ExtractionRunCoordinator {
     turns: BufferTurn[],
     options: {
       clearBufferAfterExtraction?: boolean;
-      clearMatchingTurns?: boolean;
+      clearMatchingTurns?: boolean; retentionWindow?: readonly BufferTurn[];
       skipCharThreshold?: boolean;
       skipUserTurnThreshold?: boolean;
       deadlineMs?: number;
@@ -1044,7 +1045,7 @@ export class ExtractionRunCoordinator {
       if (!shouldPersistProcessedFingerprint) {
         await commitEmptySuccessExtraction(emptySuccessCommit);
       }
-      return { status: "skipped", reason: "empty_extraction_result", persistedCount: 0, durableOutputCount: 0 };
+      return emptyExtractionResult(recordedRetryFailure, result.extractionFailureClass);
     }
 
     let threadIdForExtraction: string | null = null;
@@ -1108,7 +1109,7 @@ export class ExtractionRunCoordinator {
       normalizedTurns: normalizedTurns as BufferTurn[],
       bufferKey,
       retainDeferredTurns: (key, retainedTurns, max) =>
-        this.deps.getBuffer().retainDeferredTurns(key, retainedTurns, max, turns),
+        this.deps.getBuffer().retainDeferredTurns(key, retainedTurns, max, options.retentionWindow ?? turns),
       clearBuffer,
       failOnExtractionFailure: options.failOnExtractionFailure === true,
     });

package/src/orchestration/turn-ingestion.ts

@@ -42,6 +42,7 @@ import { TranscriptManager } from "../transcript.js";
 import type { BufferTurn, PluginConfig, SourceConnectorProvenance } from "../types.js";
 
 import { groupTurnsByExtractionScope, bufferTurnsEqual } from "../buffer-turn-helpers.js";
+import { ExtractionDrain } from "./extraction-drain.js";
 import { resolveScopeProfilePlan, type ResolvedScopeProfilePlan, hasExplicitNamespacePolicy } from "../namespaces/scope-profiles.js";
 import { resolveCodingNamespaceOverlay, combineNamespaces } from "../coding/coding-namespace.js";
 import { canWriteNamespace, defaultNamespaceForPrincipal } from "../namespaces/principal.js";
@@ -114,6 +115,8 @@ export interface TurnIngestionDeps {
     },
   ): Promise<void>;
   resolveMemoryIdOrHandle(ref: string, sessionKey?: string): string;
+  /** Max conversation chars the extraction model accepts; absent on legacy hosts (single-run behavior). */
+  extractionWindowBudgetChars?(): number;
   runExtraction(
     ...args: Parameters<ExtractionRunCoordinator["runExtraction"]>
   ): Promise<ExtractionRunResult>;
@@ -156,9 +159,10 @@ function isFlushPlanRecoveryTurn(
 }
 
 export class TurnIngestionCoordinator {
-  constructor(
-    private readonly deps: TurnIngestionDeps,
-  ) {}
+  private readonly drain: ExtractionDrain; // chunked trigger-drain (extraction-drain.ts)
+  constructor(private readonly deps: TurnIngestionDeps) {
+    this.drain = new ExtractionDrain(deps);
+  }
 
   async processTurn(
     role: "user" | "assistant",
@@ -1067,7 +1071,7 @@ export class TurnIngestionCoordinator {
             effectiveScopePlan = fresh;
           }
         }
-        const result = await this.deps.runExtraction(effectiveTurns, {
+        const result = await this.drain.run(effectiveTurns, {
           clearBufferAfterExtraction:
             options.clearBufferAfterExtraction ?? true,
           clearMatchingTurns: options.clearMatchingTurns === true,
@@ -1083,7 +1087,7 @@ export class TurnIngestionCoordinator {
           principalOverride: options.principalOverride,
           scopeProfileWritePlan: effectiveScopePlan,
           forceExtractionAttempt: options.forceExtractionAttempt === true,
-        });
+        }, preemptable);
         settleTask(undefined, result);
       } catch (err) {
         const reason = abortSignal.reason;

package/src/orchestrator.ts

@@ -238,6 +238,7 @@ import {
 } from "./maintenance/pattern-reinforcement.js";
 import { refreshConvergedNamespaces } from "./convergence-refresh.js";
 import { ModelRegistry } from "./model-registry.js";
+import { extractionWindowBudgetChars } from "./extraction-window-drain.js";
 import { applyRuntimeRetrievalPolicy, expandQuery } from "./retrieval.js";
 import {
   mergeWithAgentResults,
@@ -3062,6 +3063,26 @@ export class Orchestrator {
     );
   }
 
+  /**
+   * Max conversation characters the extraction model can take as input
+   * (context window the ModelRegistry learned, capped by
+   * `localLlmMaxContext` when set). Exposed for the trigger-drain chunking
+   * in the turn-ingestion coordinator so an oversized buffer window is
+   * drained in fitting chunks instead of failing whole.
+   */
+  private extractionWindowBudgetChars(): number {
+    // Object.create(Orchestrator.prototype) test doubles skip the constructor,
+    // so neither modelRegistry nor config exists there: report "no budget".
+    if (!this.modelRegistry) return Number.POSITIVE_INFINITY;
+    return extractionWindowBudgetChars({
+      registry: this.modelRegistry,
+      localExtractionActive:
+        this.config.modelSource !== "gateway" && resolveLocalLlmCapabilities(this.config).localLlm,
+      model: this.config.localLlmModel,
+      maxContext: this.config.localLlmMaxContext,
+    });
+  }
+
   private normalizeExtractionFingerprintTurns(turns: BufferTurn[]): string[] {
     return this.extractionRunCoordinator.normalizeExtractionFingerprintTurns(turns);
   }

package/src/secure-store/secure-fs.ts

@@ -902,7 +902,9 @@ const ENCRYPTABLE_STATE_SIDECARS = new Set([
 ]);
 
 function isEncryptableStateSidecar(normalized: string): boolean {
-  return ENCRYPTABLE_STATE_SIDECARS.has(normalized);
+  // state/dropped/ holds copies of extraction-buffer turns dropped after
+  // repeated extraction failures — same content sensitivity as buffer.json.
+  return ENCRYPTABLE_STATE_SIDECARS.has(normalized) || normalized.startsWith("state/dropped/");
 }
 
 /**

package/src/storage/dropped-extraction-chunk.ts

@@ -0,0 +1,27 @@
+import { randomUUID } from "node:crypto";
+import { mkdir } from "node:fs/promises";
+import path from "node:path";
+import type { BufferTurn } from "../types.js";
+
+/**
+ * Persist a copy of extraction-buffer turns that are being DROPPED after repeated extraction failures (trigger-drain drop
+ * path), so a destructive drop never loses content silently. Written under `<stateDir>/dropped/` through the store's secure
+ * writer, so encrypted stores keep it encrypted (secure-fs encrypts the whole `state/dropped/` prefix). Returns the path.
+ */
+export async function saveDroppedExtractionChunk(
+  stateDir: string,
+  writeSecureFile: (filePath: string, content: string) => Promise<void>,
+  bufferKey: string,
+  reason: string,
+  turns: readonly BufferTurn[],
+): Promise<string> {
+  const droppedDir = path.join(stateDir, "dropped");
+  await mkdir(droppedDir, { recursive: true });
+  const safeKey = bufferKey.replace(/[^a-zA-Z0-9._-]/g, "-").slice(0, 80) || "default";
+  // The random suffix keeps copies written in the same millisecond (or for keys that truncate to the same name) from
+  // overwriting each other.
+  const filePath = path.join(droppedDir, `${Date.now()}-${randomUUID().slice(0, 8)}-${safeKey}.json`);
+  const payload = { bufferKey, reason, droppedAt: new Date().toISOString(), turns };
+  await writeSecureFile(filePath, JSON.stringify(payload, null, 2));
+  return filePath;
+}

package/src/storage-dropped-chunk-filenames.test.ts

@@ -0,0 +1,30 @@
+import assert from "node:assert/strict";
+import { mkdtemp, readdir, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import test from "node:test";
+import { StorageManager } from "./storage.js";
+import type { BufferTurn } from "./types.js";
+
+function makeTurn(content: string): BufferTurn {
+  return { role: "user", content, timestamp: "2026-10-29T00:00:00.000Z", sessionKey: "s1" };
+}
+
+test("dropped-chunk recovery files never overwrite each other, even within one millisecond", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-dropped-chunks-"));
+  const realNow = Date.now;
+  try {
+    Date.now = () => 1_793_000_000_000;
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const first = await storage.saveDroppedExtractionChunk("same-key", "first", [makeTurn("one")]);
+    const second = await storage.saveDroppedExtractionChunk("same-key", "second", [makeTurn("two")]);
+
+    assert.notEqual(first, second);
+    const files = await readdir(path.dirname(first));
+    assert.equal(files.length, 2, "both safety copies exist");
+  } finally {
+    Date.now = realNow;
+    await rm(dir, { recursive: true, force: true });
+  }
+});

package/src/storage.ts

@@ -144,6 +144,7 @@ import {
 import { supersessionKeysForFact } from "./temporal-supersession.js";
 import { BufferSessionFiles } from "./storage/buffer-session-files.js";
 import { runSupersessionSideEffects, type SupersessionAuditOptions } from "./storage/supersession-side-effects.js";
+import { saveDroppedExtractionChunk as saveDroppedChunk } from "./storage/dropped-extraction-chunk.js";
 import type {
   AccessTrackingEntry,
   BufferEntryState,
@@ -4799,7 +4800,6 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
     return results.filter((filePath): filePath is string => filePath !== null);
   }
 
-
   private orderWindowPaths(filePaths: string[]): string[] {
     const correctionPaths: string[] = [];
     const factPaths: string[] = [];
@@ -5520,6 +5520,9 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   async saveBuffer(state: BufferState): Promise<void> { await this.bufferSessionFiles().reconcile(state); }
   async saveBufferSession(key: string, entry: BufferEntryState): Promise<void> { await this.bufferSessionFiles().save(key, entry); }
   async deleteBufferSession(key: string): Promise<void> { await this.bufferSessionFiles().remove(key); }
+  saveDroppedExtractionChunk(bufferKey: string, reason: string, turns: readonly import("./types.js").BufferTurn[]): Promise<string> { // storage/dropped-extraction-chunk.ts
+    return saveDroppedChunk(this.stateDir, (file, content) => this.writeStorageSecureFile(file, content), bufferKey, reason, turns);
+  }
 
   async loadMeta(): Promise<MetaState> {
     const metaPath = path.join(this.stateDir, "meta.json");
@@ -5564,8 +5567,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
 
   async saveMeta(state: MetaState): Promise<void> {
     await this.ensureDirectories();
-    const metaPath = path.join(this.stateDir, "meta.json");
-    await this.writeStorageSecureFile(metaPath, JSON.stringify(state, null, 2));
+    await this.writeStorageSecureFile(path.join(this.stateDir, "meta.json"), JSON.stringify(state, null, 2));
   }
 
   async appendMemoryActionEvents(events: MemoryActionEvent[]): Promise<number> {
@@ -7208,10 +7210,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
   async saveTopics(topics: TopicScore[]): Promise<void> {
     const metaPath = path.join(this.stateDir, "topics.json");
     await mkdir(this.stateDir, { recursive: true });
-    await this.writeStorageSecureFile(
-      metaPath,
-      JSON.stringify({ topics, updatedAt: new Date().toISOString() }, null, 2)
-    );
+    await this.writeStorageSecureFile(metaPath, JSON.stringify({ topics, updatedAt: new Date().toISOString() }, null, 2));
     log.debug(`saved ${topics.length} topic scores`);
   }
 

package/src/types.ts

@@ -1697,6 +1697,13 @@ export interface PluginConfig
   extractionBreakerCooldownMs: number;
   /** Breaker open cooldown (ms) for auth/config failures (longer — provider is misconfigured). */
   extractionBreakerAuthCooldownMs: number;
+  /**
+   * Consecutive failed extraction attempts after which the failing
+   * trigger-drain chunk is dropped (with a side-file copy under
+   * `state/dropped/`) so a session whose window never extracts cannot stall
+   * the buffer forever. Default: `5`.
+   */
+  extractionChunkDropAfterFailures: number;
   // Local LLM fast tier (v9.1) — smaller model for quick ops
   localLlmFastEnabled: boolean;
   localLlmFastModel: string;