@remnic/core 10.0.37 → 10.0.39

package/src/extraction-window-drain.test.ts

@@ -0,0 +1,123 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import {
+  chunkExtractionWindow,
+  classifyDrainResult,
+  extractionWindowBudgetChars,
+  mergeDrainResults,
+} from "./extraction-window-drain.js";
+import type { ExtractionRunResult } from "./orchestration/extraction-run.js";
+import type { BufferTurn } from "./types.js";
+
+function makeTurn(content: string, sessionKey = "s1"): BufferTurn {
+  return { role: "user", content, timestamp: "2026-10-29T00:00:00.000Z", sessionKey };
+}
+
+function skippedResult(reason: string): ExtractionRunResult {
+  return { status: "skipped", reason, persistedCount: 0, durableOutputCount: 0 };
+}
+
+test("chunkExtractionWindow returns a single chunk when the window fits the budget", () => {
+  const turns = [makeTurn("a".repeat(10)), makeTurn("b".repeat(10))];
+  const chunks = chunkExtractionWindow(turns, 100);
+  assert.equal(chunks.length, 1);
+  assert.deepEqual(chunks[0], turns);
+});
+
+test("chunkExtractionWindow splits an oversized window into budget-fitting chunks in order", () => {
+  const turns = [
+    makeTurn("a".repeat(60)),
+    makeTurn("b".repeat(60)),
+    makeTurn("c".repeat(60)),
+    makeTurn("d".repeat(60)),
+  ];
+  const chunks = chunkExtractionWindow(turns, 100);
+  assert.equal(chunks.length, 4);
+  for (const chunk of chunks) {
+    assert.ok(chunk.every((turn) => turn.content.length <= 100));
+  }
+  assert.deepEqual(
+    chunks.flat().map((turn) => turn.content[0]),
+    ["a", "b", "c", "d"],
+  );
+  assert.deepEqual(chunks.flat(), turns);
+});
+
+test("chunkExtractionWindow gives a single oversized turn its own chunk", () => {
+  const turns = [makeTurn("x".repeat(250)), makeTurn("y".repeat(10))];
+  const chunks = chunkExtractionWindow(turns, 100);
+  assert.equal(chunks.length, 2);
+  assert.equal(chunks[0]?.length, 1);
+  assert.equal(chunks[1]?.length, 1);
+});
+
+test("chunkExtractionWindow treats a non-positive budget as no chunking", () => {
+  const turns = [makeTurn("a".repeat(500)), makeTurn("b".repeat(500))];
+  assert.equal(chunkExtractionWindow(turns, 0).length, 1);
+  assert.equal(chunkExtractionWindow(turns, Number.POSITIVE_INFINITY).length, 1);
+});
+
+test("chunkExtractionWindow returns no chunks for an empty window", () => {
+  assert.deepEqual(chunkExtractionWindow([], 100), []);
+});
+
+test("classifyDrainResult treats completed runs and benign skips as drained", () => {
+  assert.equal(classifyDrainResult({ status: "completed", persistedCount: 1, durableOutputCount: 2 }), "drained");
+  assert.equal(classifyDrainResult(skippedResult("below_threshold")), "drained");
+  assert.equal(classifyDrainResult(skippedResult("cron_session")), "drained");
+  assert.equal(classifyDrainResult(skippedResult("empty_normalized_turns")), "drained");
+  assert.equal(classifyDrainResult(skippedResult("processed_fingerprint")), "drained");
+});
+
+test("classifyDrainResult reports breaker/backoff skips as deferred", () => {
+  assert.equal(classifyDrainResult(skippedResult("provider_circuit_open")), "deferred");
+  assert.equal(classifyDrainResult(skippedResult("extraction_backoff")), "deferred");
+});
+
+test("classifyDrainResult reports empty extraction results as failed", () => {
+  assert.equal(
+    classifyDrainResult({ ...skippedResult("empty_extraction_result"), retainedBuffer: true }),
+    "failed",
+    "an empty result that kept the window for retry is a chunk failure",
+  );
+  assert.equal(
+    classifyDrainResult(skippedResult("empty_extraction_result")),
+    "drained",
+    "an empty result that cleared its turns extracted fine and found nothing durable",
+  );
+});
+
+test("mergeDrainResults sums counts, completes when any chunk completes, keeps the last skip reason", () => {
+  const first = skippedResult("empty_extraction_result");
+  const second: ExtractionRunResult = { status: "completed", persistedCount: 3, durableOutputCount: 4 };
+  // A completed chunk means the drain made durable progress.
+  const merged = mergeDrainResults(mergeDrainResults(null, first), second);
+  assert.equal(merged.status, "completed");
+  assert.equal(merged.persistedCount, 3);
+  assert.equal(merged.durableOutputCount, 4);
+  // All chunks skipped: the aggregate stays skipped with the last reason.
+  const mergedFailed = mergeDrainResults(
+    mergeDrainResults(null, skippedResult("provider_circuit_open")),
+    first,
+  );
+  assert.equal(mergedFailed.status, "skipped");
+  assert.equal(mergedFailed.reason, "empty_extraction_result");
+  assert.equal(mergedFailed.persistedCount, 0);
+});
+
+test("extractionWindowBudgetChars applies the learned local window only when the local model extracts", () => {
+  const registry = { calculateContextSizes: () => ({ maxInputChars: 12_000 }) };
+  const base = { registry, model: "local-model", maxContext: undefined };
+
+  assert.equal(extractionWindowBudgetChars({ ...base, localExtractionActive: true }), 12_000);
+  assert.equal(
+    extractionWindowBudgetChars({ ...base, localExtractionActive: false }),
+    Number.POSITIVE_INFINITY,
+    "gateway or disabled local LLM: the local window does not describe the extractor",
+  );
+  assert.equal(
+    extractionWindowBudgetChars({ ...base, registry: undefined, localExtractionActive: true }),
+    Number.POSITIVE_INFINITY,
+    "no registry (legacy host or test double): no budget",
+  );
+});

package/src/extraction-window-drain.ts

@@ -0,0 +1,110 @@
+/**
+ * Trigger-drain helpers (extraction trigger stall fix).
+ *
+ * A session whose buffered window outgrows the extraction model's context
+ * never drains: the trigger fires `extract_batch` on every observe, the
+ * oversized request fails, nothing is cleared, and the window only grows.
+ * These helpers split such a window into budget-fitting chunks and classify
+ * each extraction run so the caller can drain successes, defer on breaker/
+ * backoff retention, and drop repeatedly failing chunks.
+ */
+
+import type { ExtractionRunResult } from "./orchestration/extraction-run.js";
+
+/**
+ * Split `turns` into chunks whose summed content length stays within
+ * `maxChars`. Chunks keep turn order and never split a turn; a single turn
+ * longer than the budget becomes its own chunk (the extraction prompt still
+ * clamps per-turn content). A non-positive or non-finite budget, or an empty
+ * window, yields at most one chunk (no chunking).
+ */
+export function chunkExtractionWindow<T extends { content: string }>(
+  turns: readonly T[],
+  maxChars: number,
+): T[][] {
+  if (turns.length === 0) return [];
+  const budget =
+    Number.isFinite(maxChars) && maxChars > 0 ? Math.floor(maxChars) : Number.POSITIVE_INFINITY;
+  const chunks: T[][] = [];
+  let current: T[] = [];
+  let currentChars = 0;
+  for (const turn of turns) {
+    const turnChars = turn.content.length;
+    if (current.length > 0 && currentChars + turnChars > budget) {
+      chunks.push(current);
+      current = [];
+      currentChars = 0;
+    }
+    current.push(turn);
+    currentChars += turnChars;
+  }
+  if (current.length > 0) chunks.push(current);
+  return chunks;
+}
+
+export type DrainVerdict = "drained" | "deferred" | "failed";
+
+/**
+ * Run results that deliberately RETAIN the buffer (provider breaker open,
+ * per-fingerprint backoff) are deferrals, not chunk failures — counting them
+ * would drop good content during a provider outage.
+ */
+const DRAIN_DEFERRAL_REASONS = {
+  provider_circuit_open: true,
+  extraction_backoff: true,
+};
+
+export function classifyDrainResult(result: ExtractionRunResult): DrainVerdict {
+  if (result.status !== "skipped") return "drained";
+  // An empty result is a failure only when the run kept the window for retry;
+  // otherwise it extracted fine, found nothing durable, and cleared its turns.
+  if (result.reason === "empty_extraction_result") {
+    return result.retainedBuffer === true ? "failed" : "drained";
+  }
+  if (result.reason !== undefined && result.reason in DRAIN_DEFERRAL_REASONS) return "deferred";
+  return "drained";
+}
+
+/**
+ * Combine chunk results into the single `ExtractionRunResult` the queue
+ * contract settles with: counts sum, any completed chunk completes the drain,
+ * and the last non-completed reason wins.
+ */
+export function mergeDrainResults(
+  previous: ExtractionRunResult | null,
+  next: ExtractionRunResult,
+): ExtractionRunResult {
+  if (previous === null) return next;
+  const completed = previous.status === "completed" || next.status === "completed";
+  const merged: ExtractionRunResult = {
+    status: completed ? "completed" : "skipped",
+    persistedCount: previous.persistedCount + next.persistedCount,
+    durableOutputCount: previous.durableOutputCount + next.durableOutputCount,
+  };
+  if (!completed) merged.reason = next.reason ?? previous.reason;
+  if (previous.postPersistMetadataFailed === true || next.postPersistMetadataFailed === true) {
+    merged.postPersistMetadataFailed = true;
+  }
+  return merged;
+}
+
+/** The slice of ModelRegistry the drain budget needs. */
+export interface WindowBudgetRegistry {
+  calculateContextSizes(modelId: string, maxContextOverride?: number): { maxInputChars: number };
+}
+
+/**
+ * Character budget for one extraction request. The learned local-model window
+ * only describes the extractor when the local LLM does the extracting; with a
+ * gateway extractor, a disabled local LLM, or no registry (legacy hosts, test
+ * doubles) there is no budget and the window runs in one request as before.
+ */
+export function extractionWindowBudgetChars(input: {
+  registry: WindowBudgetRegistry | undefined;
+  localExtractionActive: boolean;
+  model: string;
+  maxContext: number | undefined;
+}): number {
+  if (!input.registry || !input.localExtractionActive) return Number.POSITIVE_INFINITY;
+  return input.registry.calculateContextSizes(input.model, input.maxContext).maxInputChars;
+}

package/src/fallback-llm.test.ts

@@ -1453,6 +1453,57 @@ test("fallback llm uses the Responses API for openai-responses transports", { co
   }
 });
 
+test("fallback llm adds reasoningHeadroomTokens to max_output_tokens on the Responses API", { concurrency: false }, async () => {
+  clearModelsJsonCache();
+  clearSecretCache();
+
+  const llm = new FallbackLlmClient({
+    agents: { defaults: { model: { primary: "responses-provider/demo-model" } } },
+    models: {
+      providers: {
+        "responses-provider": {
+          baseUrl: "https://responses.example/v1",
+          api: "openai-responses",
+          apiKey: "responses-key",
+          models: [],
+        },
+      },
+    },
+  });
+
+  const originalFetch = globalThis.fetch;
+  const bodies: Array<{ max_output_tokens?: number }> = [];
+  // SAFETY: the stub returns the Response shape the client reads and nothing else.
+  globalThis.fetch = (async (_url, init) => {
+    bodies.push(JSON.parse(String(init?.body ?? "{}")));
+    return new Response(JSON.stringify({ output_text: "ok" }), {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+    });
+  }) as typeof fetch;
+
+  try {
+    const messages = [{ role: "user" as const, content: "Say OK" }];
+    // Non-finite headroom is ignored, like a negative one.
+    await llm.chatCompletion(messages, { temperature: 0, maxTokens: 8, reasoningHeadroomTokens: Number.NaN });
+    await llm.chatCompletion(messages, { temperature: 0, maxTokens: 64, reasoningHeadroomTokens: 1024 });
+    await llm.chatCompletion(messages, { temperature: 0, maxTokens: 64 });
+    await llm.chatCompletion(messages, { temperature: 0, maxTokens: 64, reasoningHeadroomTokens: -5 });
+    await llm.chatCompletion(messages, { temperature: 0, reasoningHeadroomTokens: 1024 });
+    await llm.chatCompletion(messages, { temperature: 0, maxTokens: 64, reasoningHeadroomTokens: Number.POSITIVE_INFINITY });
+    assert.equal(bodies[5].max_output_tokens, 64, "an infinite headroom adds nothing (it would serialize as null)");
+    assert.equal(bodies[0].max_output_tokens, 8, "a non-finite headroom adds nothing");
+    assert.equal(bodies[1].max_output_tokens, 64 + 1024, "headroom is added on top of the compact budget");
+    assert.equal(bodies[2].max_output_tokens, 64, "no headroom option: exactly maxTokens, as before");
+    assert.equal(bodies[3].max_output_tokens, 64, "a negative headroom adds nothing");
+    assert.equal(bodies[4].max_output_tokens, 4096, "no maxTokens: the default budget is not inflated by headroom");
+  } finally {
+    globalThis.fetch = originalFetch;
+    clearModelsJsonCache();
+    clearSecretCache();
+  }
+});
+
 test("fallback llm ignores echoed input_text blocks in responses output extraction", { concurrency: false }, async () => {
   clearModelsJsonCache();
   clearSecretCache();
@@ -2122,3 +2173,46 @@ test("timeout records TimeoutError on failureDiag (issue #2891)", { concurrency:
     clearSecretCache();
   }
 });
+
+test("fallback llm adds reasoningHeadroomTokens to the Chat Completions token limit too", { concurrency: false }, async () => {
+  clearModelsJsonCache();
+  clearSecretCache();
+
+  const llm = new FallbackLlmClient({
+    agents: { defaults: { model: { primary: "chat-provider/demo-model" } } },
+    models: {
+      providers: {
+        "chat-provider": {
+          baseUrl: "https://chat.example/v1",
+          api: "openai-completions",
+          apiKey: "chat-key",
+          models: [],
+        },
+      },
+    },
+  });
+
+  const originalFetch = globalThis.fetch;
+  const bodies: Array<{ max_tokens?: number; max_completion_tokens?: number }> = [];
+  // SAFETY: the stub returns the Chat Completions shape the client reads and nothing else.
+  globalThis.fetch = (async (_url, init) => {
+    bodies.push(JSON.parse(String(init?.body ?? "{}")));
+    return new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+    });
+  }) as typeof fetch;
+
+  try {
+    const messages = [{ role: "user" as const, content: "Say OK" }];
+    await llm.chatCompletion(messages, { temperature: 0, maxTokens: 64, reasoningHeadroomTokens: 1024 });
+    await llm.chatCompletion(messages, { temperature: 0, maxTokens: 64 });
+    const limit = (b: (typeof bodies)[number]) => b.max_completion_tokens ?? b.max_tokens;
+    assert.equal(limit(bodies[0]), 64 + 1024, "headroom is added on top of the compact budget");
+    assert.equal(limit(bodies[1]), 64, "no headroom option: exactly maxTokens, as before");
+  } finally {
+    globalThis.fetch = originalFetch;
+    clearModelsJsonCache();
+    clearSecretCache();
+  }
+});

package/src/fallback-llm.ts

@@ -46,6 +46,7 @@ export {
 export interface FallbackLlmOptions {
   temperature?: number;
   maxTokens?: number;
+  reasoningHeadroomTokens?: number; // Responses API only: extra max_output_tokens for reasoning (they count against it); unset adds none
   timeoutMs?: number;
   signal?: AbortSignal;
   /** Set the Responses API storage policy. Other transports ignore this value. */
@@ -764,17 +765,16 @@ export class FallbackLlmClient {
       return await this.callOllamaChat(effectiveConfig, model.modelId, messages, options);
     }
 
+    const wanted = Math.floor(options.reasoningHeadroomTokens ?? 0);
+    const headroom = Number.isFinite(wanted) ? Math.max(0, wanted) : 0; // NaN and Infinity add nothing
+    const capped = options.maxTokens === undefined ? options : { ...options, maxTokens: options.maxTokens + headroom };
+
     if (
       model.providerConfig.api === "openai-responses" ||
       model.providerConfig.api === "openai-codex-responses" ||
       model.providerConfig.api === "azure-openai-responses"
     ) {
-      return await this.callOpenAIResponses(
-        effectiveConfig,
-        model.modelId,
-        messages,
-        options,
-      );
+      return await this.callOpenAIResponses(effectiveConfig, model.modelId, messages, capped);
     }
 
     // For OpenAI-compatible chat-completions APIs (openai-completions,
@@ -785,7 +785,7 @@ export class FallbackLlmClient {
       effectiveConfig,
       model.modelId,
       messages,
-      options,
+      capped,
       shouldAssumeOpenAiChatCompletions(effectiveConfig.baseUrl),
     );
   }

package/src/orchestration/extraction-drain.ts

@@ -0,0 +1,261 @@
+/**
+ * Chunked trigger-drain for the extraction pipeline.
+ *
+ * Split out of `turn-ingestion.ts`: that file is at the structural ratchet's
+ * line cap (issue #1995), and the drain is a self-contained unit — it only
+ * needs the host's `runExtraction`, the buffer drop, the storage side-file
+ * copy and the preemption hook.
+ */
+
+import { createHash } from "node:crypto";
+import { isAbortError, preemptedExtractionError } from "../abort-error.js";
+import { describeError } from "../buffer-turn-helpers.js";
+import { chunkExtractionWindow, classifyDrainResult, mergeDrainResults } from "../extraction-window-drain.js";
+import { log } from "../logger.js";
+import type { BufferTurn } from "../types.js";
+import type { ExtractionRunCoordinator, ExtractionRunResult } from "./extraction-run.js";
+
+type RunOptions = Parameters<ExtractionRunCoordinator["runExtraction"]>[1];
+
+/** The `runExtraction` options the drain forwards, with the ones it reads made required. */
+export type DrainRunOptions = RunOptions & {
+  bufferKey: string;
+  clearBufferAfterExtraction: boolean;
+  skipUserTurnThreshold: boolean;
+  failOnExtractionFailure: boolean;
+  forceExtractionAttempt: boolean;
+};
+
+/** What the drain needs from its host (structurally satisfied by `TurnIngestionDeps`). */
+export interface ExtractionDrainDeps {
+  config: {
+    extractionMinUserTurns: number;
+    extractionMinChars: number;
+    extractionMaxTurnChars: number;
+    extractionChunkDropAfterFailures: number;
+  };
+  extractionWindowBudgetChars?(): number;
+  runExtraction(turns: BufferTurn[], options: RunOptions): Promise<ExtractionRunResult>;
+  getStorage(): Promise<{
+    saveDroppedExtractionChunk(bufferKey: string, reason: string, turns: readonly BufferTurn[]): Promise<string>;
+  }>;
+  buffer: {
+    dropExtractionChunk(bufferKey: string, turns: readonly BufferTurn[], options?: { allowNonPrefix?: boolean }): Promise<number>;
+  };
+  extractionQueueCoordinator: { beginPreemptable(abort: () => void): void };
+}
+
+/** Identity of a chunk: its failure count must not follow a buffer key to different content. */
+function chunkFingerprint(chunk: readonly BufferTurn[]): string {
+  const hash = createHash("sha256");
+  for (const turn of chunk) hash.update(`${turn.role}\u0000${turn.timestamp}\u0000${turn.content}\u0001`);
+  return hash.digest("hex");
+}
+
+export class ExtractionDrain {
+  /**
+   * Consecutive failed attempts on the head chunk, per buffer key. In memory on
+   * purpose (a restart re-counts from zero). Bound to the chunk's content: a
+   * count recorded for one chunk never applies to a different one, so a force
+   * flush that clears a failing chunk cannot make unrelated content inherit its
+   * failures.
+   */
+  private readonly failures = new Map<string, { fingerprint: string; count: number }>();
+
+  /**
+   * Buffer key to the first turn of the chunk a qualified window stopped on.
+   * The rest of such a window stays eligible on the next attempt even when it
+   * is assistant-only or short: the window passed the thresholds as a whole,
+   * and re-judging its tail would clear it unextracted. Cleared when the
+   * window drains; one entry per key.
+   */
+  private readonly qualified = new Map<string, string>();
+
+  constructor(private readonly deps: ExtractionDrainDeps) {}
+
+  /** Characters the coordinator would count for this window: user/assistant text, trimmed and capped per turn, context-only turns left out. */
+  private windowChars(turns: readonly BufferTurn[]): number {
+    let total = 0;
+    for (const turn of turns) {
+      if ((turn.role !== "user" && turn.role !== "assistant") || turn.extractionContextOnly === true) continue;
+      total += turn.content.trim().slice(0, this.deps.config.extractionMaxTurnChars).length;
+    }
+    return total;
+  }
+
+  private failureCount(bufferKey: string, chunk: readonly BufferTurn[]): number {
+    const entry = this.failures.get(bufferKey);
+    return entry && entry.fingerprint === chunkFingerprint(chunk) ? entry.count : 0;
+  }
+
+  private setFailureCount(bufferKey: string, chunk: readonly BufferTurn[], count: number): void {
+    if (count <= 0) this.failures.delete(bufferKey);
+    else this.failures.set(bufferKey, { fingerprint: chunkFingerprint(chunk), count });
+  }
+
+  /**
+   * Run `runExtraction` over the trigger-drain window. When the window outgrows
+   * the extraction model's input budget (the stall that keeps a session's buffer
+   * growing forever: every observe re-fires the trigger on a window that can
+   * never extract), split it into budget-fitting chunks and drain them in
+   * order; each successful run clears its chunk, so the buffer shrinks across
+   * observes. A chunk that fails `extractionChunkDropAfterFailures` times in a
+   * row is dropped (side-file copy first). Breaker/backoff retention results
+   * stop the drain without counting as chunk failures.
+   *
+   * Explicit drains (replay/import `clearBufferAfterExtraction: false`, force
+   * flush, fail-closed callers, accepted captures) and windows that fit the
+   * budget (or hosts without the budget accessor) take the single-run path.
+   *
+   * `preemptable`: the run was queued as preemptable automatic work. Each chunk's
+   * persist step ends the host's preemption registration, so the drain registers
+   * it again before every later chunk; a fail-closed flush can then abort the
+   * next chunk's model request instead of waiting behind the whole window.
+   */
+  async run(turns: BufferTurn[], runOptions: DrainRunOptions, preemptable = false): Promise<ExtractionRunResult> {
+    const budgetChars = this.deps.extractionWindowBudgetChars?.() ?? Number.POSITIVE_INFINITY;
+    const chunkable =
+      runOptions.clearBufferAfterExtraction &&
+      !runOptions.forceExtractionAttempt &&
+      !runOptions.failOnExtractionFailure &&
+      !turns.some((turn) => turn.acceptedCapture === true) &&
+      Number.isFinite(budgetChars) &&
+      budgetChars > 0;
+    const chunks = chunkable ? chunkExtractionWindow(turns, budgetChars) : [];
+    const resumed =
+      chunks.length > 0 && this.qualified.get(runOptions.bufferKey) === chunkFingerprint(chunks[0].slice(0, 1));
+    // The tail of a window that already qualified goes through the chunk loop even when it now fits in one
+    // chunk, so its failures still count toward the drop threshold.
+    if (chunks.length <= 1 && !resumed) return await this.deps.runExtraction(turns, runOptions);
+    // Eligibility (minimum user turns and characters) belongs to the whole
+    // window: a window that never qualified keeps the single-run path, which
+    // judges it exactly as before.
+    if (!resumed) {
+      const windowUserTurns = turns.filter((turn) => turn.role === "user").length;
+      const belowChars = !runOptions.skipCharThreshold && this.windowChars(turns) < this.deps.config.extractionMinChars;
+      if ((!runOptions.skipUserTurnThreshold && windowUserTurns < this.deps.config.extractionMinUserTurns) || belowChars) {
+        return await this.deps.runExtraction(turns, runOptions);
+      }
+    }
+    // The window qualified as a whole; its chunks must not be re-judged one by
+    // one (an assistant-only or short chunk would be cleared unextracted).
+    const baseOptions = { ...runOptions, skipCharThreshold: true, skipUserTurnThreshold: true };
+
+    const bufferKey = runOptions.bufferKey;
+    // A native group clears its turns by identity (a pinned capture can sit in front); a drop must find them the same way.
+    const byIdentity = runOptions.clearMatchingTurns === true;
+    const dropAfter = this.deps.config.extractionChunkDropAfterFailures;
+    let aggregate: ExtractionRunResult | null = null;
+    for (const [position, chunk] of chunks.entries()) {
+      this.qualified.set(bufferKey, chunkFingerprint(chunk.slice(0, 1)));
+      // retentionWindow: the chunks processed so far. A chunk's deferred retention replaces theirs (retained turns do not pile
+      // up across chunks) and leaves the retained turns of chunks not yet processed in the buffer.
+      let chunkOptions: DrainRunOptions = { ...baseOptions, retentionWindow: chunks.slice(0, position + 1).flat() };
+      let preempt: AbortController | undefined;
+      if (preemptable && position > 0) {
+        const local = new AbortController();
+        preempt = local;
+        this.deps.extractionQueueCoordinator.beginPreemptable(() => local.abort(preemptedExtractionError()));
+        chunkOptions = {
+          ...chunkOptions,
+          abortSignal: runOptions.abortSignal ? AbortSignal.any([runOptions.abortSignal, local.signal]) : local.signal,
+        };
+      }
+      let failures = this.failureCount(bufferKey, chunk);
+      let result: ExtractionRunResult;
+      try {
+        result = await this.deps.runExtraction(chunk, chunkOptions);
+      } catch (err) {
+        // The chunk-local abort leaves the outer signal untouched, so hand back the
+        // preemption reason the host looks for (as it does for the first chunk).
+        if (isAbortError(err)) throw preempt && preempt.signal.aborted && !runOptions.abortSignal?.aborted ? preempt.signal.reason : err;
+        failures += 1;
+        if (failures < dropAfter) {
+          this.setFailureCount(bufferKey, chunk, failures);
+          throw err;
+        }
+        // A chunk that keeps throwing is dropped like one that keeps failing,
+        // so later chunks are not stuck behind it.
+        const thrown: ExtractionRunResult = {
+          status: "skipped",
+          reason: `thrown: ${describeError(err)}`,
+          persistedCount: 0,
+          durableOutputCount: 0,
+        };
+        if (!(await this.dropFailedChunk(bufferKey, chunk, dropAfter, thrown, byIdentity))) {
+          this.setFailureCount(bufferKey, chunk, failures);
+          throw err;
+        }
+        this.setFailureCount(bufferKey, chunk, 0);
+        continue;
+      }
+      const verdict = classifyDrainResult(result);
+      if (verdict === "drained") {
+        this.setFailureCount(bufferKey, chunk, 0);
+        aggregate = mergeDrainResults(aggregate, result);
+        continue;
+      }
+      // Provider breaker/backoff retention: keep everything for retry.
+      if (verdict === "deferred") return mergeDrainResults(aggregate, result);
+      // An outage or a bad key is not the chunk's fault: keep the chunk and do not count toward dropping it.
+      if (result.failureClass === "provider_retryable" || result.failureClass === "auth_config") {
+        return mergeDrainResults(aggregate, result);
+      }
+      failures += 1;
+      if (failures >= dropAfter) {
+        if (!(await this.dropFailedChunk(bufferKey, chunk, dropAfter, result, byIdentity))) {
+          // No recovery copy: keep the only buffered copy and retry later.
+          this.setFailureCount(bufferKey, chunk, failures);
+          return mergeDrainResults(aggregate, result);
+        }
+        this.setFailureCount(bufferKey, chunk, 0);
+        aggregate = mergeDrainResults(aggregate, result);
+        continue;
+      }
+      this.setFailureCount(bufferKey, chunk, failures);
+      // Retain the remaining window for the next trigger (existing behavior).
+      return mergeDrainResults(aggregate, result);
+    }
+    this.qualified.delete(bufferKey);
+    return aggregate ?? { status: "skipped", persistedCount: 0, durableOutputCount: 0 };
+  }
+
+  /** Side-file copy first (content survives a failed drop), then remove the failing chunk from the buffer. Never throws:
+   *  the drop must not fail the extraction task that already settled its result. */
+  private async dropFailedChunk(
+    bufferKey: string,
+    chunk: readonly BufferTurn[],
+    dropAfter: number,
+    result: ExtractionRunResult,
+    byIdentity: boolean,
+  ): Promise<boolean> {
+    const reason =
+      `dropped after ${dropAfter} repeated extraction failures ` +
+      `(last result: status=${result.status} reason=${result.reason ?? "n/a"})`;
+    try {
+      const storage = await this.deps.getStorage();
+      const path = await storage.saveDroppedExtractionChunk(bufferKey, reason, chunk);
+      log.warn(`buffer[${bufferKey}]: dropping ${chunk.length} turns — ${reason}; copy saved to ${path}`);
+    } catch (err) {
+      // Never destroy the only buffered copy without a recovery file.
+      log.warn(
+        `buffer[${bufferKey}]: could not save the dropped-chunk recovery copy; keeping the chunk in the buffer: ${describeError(err)}`,
+      );
+      return false;
+    }
+    try {
+      const dropped = await this.deps.buffer.dropExtractionChunk(bufferKey, chunk, { allowNonPrefix: byIdentity });
+      if (dropped > 0) {
+        log.warn(`buffer[${bufferKey}]: removed ${dropped} buffered turns — ${reason}`);
+        return true;
+      }
+      // Nothing removed (the live turns changed): the chunk is still buffered, so this is not a drop.
+      log.warn(`buffer[${bufferKey}]: dropped chunk no longer matches the buffer; keeping it and its failure count`);
+    } catch (err) {
+      log.warn(
+        `buffer[${bufferKey}]: failed to remove dropped chunk from buffer (will retry next trigger): ${describeError(err)}`,
+      );
+    }
+    return false;
+  }
+}

package/src/orchestration/extraction-run-helpers.ts

@@ -1,6 +1,7 @@
 import type { BufferTurn, ExtractionFailureClass, ExtractionResult, MetaState } from "../types.js";
 import type { StorageManager } from "../index.js";
 import { log } from "../logger.js";
+import type { ExtractionRunResult } from "./extraction-run.js";
 
 export function combineExtractionAbortSignals(
   primary: AbortSignal | undefined,
@@ -11,6 +12,13 @@ export function combineExtractionAbortSignals(
 }
 
 
+/** Result of a run whose extraction found nothing durable; `retained` marks a recorded failure that left the window in the buffer. */
+export function emptyExtractionResult(retained: boolean, failureClass?: ExtractionFailureClass): ExtractionRunResult {
+  const result: ExtractionRunResult = { status: "skipped", reason: "empty_extraction_result", persistedCount: 0, durableOutputCount: 0 };
+  if (retained) Object.assign(result, { retainedBuffer: true, failureClass: failureClass ?? "provider_retryable" }); // same default as the retry bookkeeping
+  return result;
+}
+
 function isRecord(value: unknown): value is Record<string, unknown> {
   return typeof value === "object" && value !== null;
 }