@librechat/agents 3.7.13 → 3.7.15

package/src/llm/preempt.ts

@@ -1,6 +1,11 @@
 // src/llm/preempt.ts
 import type { AIMessageChunk } from '@langchain/core/messages';
-import { ContentTypes, DEFAULT_MAX_SEALS } from '@/common';
+import {
+  ContentTypes,
+  DEFAULT_MAX_SEALS,
+  DEFAULT_PREEMPT_RESTART_GRACE_MS,
+} from '@/common';
+import { isReasoningContentBlock } from '@/messages/core';
 
 /**
  * Normalizes a host-supplied seal budget.
@@ -119,6 +124,61 @@ function countOpenToolCalls(
   return open;
 }
 
+/**
+ * True for a text block a provider emitted before it had anything to say.
+ *
+ * Whitespace-only string content is already accepted above, and the block form
+ * of the same thing must agree: several providers open their message with an
+ * empty text part, and rejecting it would leave a turn neither sealable nor
+ * discardable — an armed interrupt would then wait out the whole turn for the
+ * sake of a block carrying nothing.
+ */
+function isBlankTextBlock(block: { type?: string; text?: unknown }): boolean {
+  if (block.type !== ContentTypes.TEXT) {
+    return false;
+  }
+  return typeof block.text === 'string' && block.text.trim() === '';
+}
+
+/**
+ * True when an OpenAI Responses turn already carries provider-side tool output.
+ *
+ * Responses reports its built-in tools differently from every other shape this
+ * module checks: a completed web search, code interpreter run, or apply-patch
+ * lands in `additional_kwargs.tool_outputs` or the authoritative
+ * `response_metadata.output`, while `content` stays empty and the `tool_calls`
+ * arrays are never populated. The ordinary gates therefore see an empty turn
+ * and would call it disposable — and reissuing the prompt would run that tool
+ * a SECOND time, rebilling a search and repeating whatever a shell or
+ * apply-patch call already did to the world.
+ *
+ * `output` is inspected item by item rather than treated as fatal on sight,
+ * because a reasoning-only Responses turn populates it too at natural
+ * completion, and that turn is precisely the one a restart should be able to
+ * discard.
+ */
+function hasResponsesProviderOutput(chunk: AIMessageChunk): boolean {
+  if (chunk.additional_kwargs.tool_outputs != null) {
+    return true;
+  }
+  const metadata = chunk.response_metadata as {
+    tool_outputs?: unknown;
+    output?: unknown;
+  };
+  if (metadata.tool_outputs != null) {
+    return true;
+  }
+  if (!Array.isArray(metadata.output)) {
+    return false;
+  }
+  for (const item of metadata.output) {
+    if (!isReasoningContentBlock(item as { type?: string })) {
+      return true;
+    }
+  }
+  return false;
+}
+
 /**
  * Cooperative mid-generation seal gate. Returns true ONLY when sealing here
  * yields a message sequence valid on EVERY supported provider:
@@ -176,3 +236,262 @@ export function canSealPreempt(chunk: AIMessageChunk | undefined): boolean {
   }
   return hasNonEmptyTextContent(chunk.content);
 }
+
+/**
+ * What a host's armed preempt request may do to the turn in flight.
+ *
+ * `seal` keeps the partial assistant turn and injects after it; `restart`
+ * throws the partial away and re-issues the model call with the injected turn
+ * appended to the prompt. They are not two flavors of the same move — one
+ * preserves work, the other deliberately discards it — so the decision is
+ * resolved once, here, rather than re-derived at each trigger.
+ */
+export type PreemptAction = 'none' | 'seal' | 'restart';
+
+/**
+ * True when the accumulated turn holds nothing a seal would have to preserve,
+ * so the whole model call can be thrown away and re-issued instead.
+ *
+ * The membership test is a WHITELIST: every content block must be a known
+ * reasoning block, and an unrecognized block refuses the restart. Blacklisting
+ * would silently discard whatever a future provider adds, and the failure is
+ * asymmetric — refusing costs the user a slower interrupt (exactly today's
+ * behavior), while wrongly discarding destroys work the model already did and
+ * the host may already have rendered.
+ *
+ * Tool machinery of any kind refuses, and deliberately without the settled-id
+ * allowance {@link canSealPreempt} makes: a settled `web_search_tool_result`
+ * means the provider already ran and billed a search, and re-issuing the call
+ * would run it again. A seal there is free; a restart is not. The same
+ * argument covers Responses' sidecar shapes — see
+ * {@link hasResponsesProviderOutput}, which the ordinary tool-call gates
+ * cannot see at all.
+ *
+ * An accumulation that carries visible text is not handled here at all — the
+ * caller resolves `seal` first, because keeping the user's answer always beats
+ * discarding it.
+ *
+ * `undefined` — the provider has sent nothing at all — is the case this whole
+ * path exists for: it is the silent window between the request and the first
+ * chunk, where there is by definition nothing to lose.
+ */
+export function canRestartPreempt(chunk: AIMessageChunk | undefined): boolean {
+  if (chunk == null) {
+    return true;
+  }
+  if ((chunk.tool_calls?.length ?? 0) > 0) {
+    return false;
+  }
+  if ((chunk.tool_call_chunks?.length ?? 0) > 0) {
+    return false;
+  }
+  if ((chunk.invalid_tool_calls?.length ?? 0) > 0) {
+    return false;
+  }
+  if (hasResponsesProviderOutput(chunk)) {
+    return false;
+  }
+  const { content } = chunk;
+  if (typeof content === 'string') {
+    return content.trim() === '';
+  }
+  for (const block of content) {
+    if (isReasoningContentBlock(block) || isBlankTextBlock(block)) {
+      continue;
+    }
+    return false;
+  }
+  return true;
+}
+
+/**
+ * Normalizes a host-supplied restart grace, on the same rules as
+ * {@link resolveMaxSeals}: `0` is honored as "never wait", anything not finite
+ * falls back to the default. Negative values collapse to `0` rather than
+ * inverting the comparison in {@link resolvePreemptAction}.
+ */
+export function resolveRestartGraceMs(graceMs: number | undefined): number {
+  if (graceMs == null || !Number.isFinite(graceMs)) {
+    return DEFAULT_PREEMPT_RESTART_GRACE_MS;
+  }
+  return graceMs > 0 ? graceMs : 0;
+}
+
+/**
+ * The single decision point both preempt triggers share: the per-chunk poll in
+ * the stream loop, and the host wake-up that fires while the provider is
+ * silent.
+ *
+ * A SEAL is preferred wherever one is available, so a turn that has already
+ * produced an answer never loses it to a restart. A RESTART converts only once
+ * the request has outlived `graceMs`.
+ *
+ * The window is not politeness, it is correctness, and it applies to a silent
+ * provider exactly as it does to a reasoning one:
+ *  - reasoning usually precedes text by moments, and discarding a turn that
+ *    was about to become sealable trades a kept answer for a re-issued
+ *    request. Only a genuinely long thinking stretch — the one an interrupt
+ *    can otherwise wait out entirely — should convert.
+ *  - `chunk` is what the CONSUMER has accumulated, and the provider stream
+ *    buffers a chunk ahead of it. An empty accumulation therefore does not
+ *    prove the provider produced nothing; it can also mean the first chunk is
+ *    in flight. Converting on emptiness alone would discard that chunk — text
+ *    included — on a race no caller can see. A provider that is still silent a
+ *    window later has no such chunk outstanding.
+ *
+ * Non-mutating and allocation-free, like the poll that guards it: the seal
+ * budget is only spent once the caller acts on a non-`none` result.
+ */
+export function resolvePreemptAction({
+  chunk,
+  requestAgeMs,
+  graceMs,
+}: {
+  chunk: AIMessageChunk | undefined;
+  requestAgeMs: number;
+  graceMs: number;
+}): PreemptAction {
+  if (canSealPreempt(chunk)) {
+    return 'seal';
+  }
+  if (!canRestartPreempt(chunk)) {
+    return 'none';
+  }
+  return requestAgeMs >= graceMs ? 'restart' : 'none';
+}
+
+/**
+ * How long a restarted model run stays recorded when nothing consumes it.
+ *
+ * The consumer is LangChain's error callback, dispatched on a non-awaited
+ * queue, so a marker may legitimately sit unread for a while. A COUNT cap
+ * cannot tell that apart from a leak: a burst of concurrent restarts would
+ * evict markers whose callbacks were still queued, and each eviction turns an
+ * expected restart back into an error in the host's traces — the very thing
+ * the marker exists to prevent. Age can tell them apart, so the bound is time.
+ *
+ * It is a LEAK bound, not a deadline: nothing here can observe when the
+ * callback queue drains, so the window is set far past any latency that queue
+ * exhibits rather than tuned to it. A stall long enough to outlast this has
+ * already broken every other time-based assumption in the run.
+ */
+const PREEMPT_RESTARTED_RUN_TTL_MS = 300_000;
+
+/**
+ * Model runs whose provider stream was torn down for a restart, with the
+ * accumulated turn so the close can carry its usage.
+ *
+ * Recorded rather than inferred from the thrown error: aborting makes the
+ * provider adapter raise whatever IT raises for cancellation, and matching on
+ * that shape would bind the tracing layer to per-provider error identity. The
+ * run id is unambiguous and already captured for the seal path.
+ *
+ * The message rides along because the run closes through the error path, which
+ * carries no output: without it a discarded attempt would report no usage at
+ * all, and the reasoning tokens the provider already billed would vanish from
+ * cost accounting. The later synthetic `CHAT_MODEL_END` cannot repair that —
+ * by then the generation is closed.
+ *
+ * Insertion-ordered, so expiry sweeps from the front and stops at the first
+ * live entry.
+ */
+const preemptRestartedRuns = new Map<string, PreemptRestartedRun>();
+
+/**
+ * The `llmOutput` a discarded attempt closes with, so a restart reads as
+ * control flow rather than as an ordinary generation — `AGENTS.md` states that
+ * control flow is not an error, and a discarded turn is not an answer either.
+ *
+ * Shared so every close route emits it: which trigger fired must not change
+ * how the run looks in a trace.
+ */
+export const PREEMPT_RESTART_CONTROL_FLOW = {
+  controlFlow: 'PreemptRestart',
+} as const;
+
+/** A torn-down model run awaiting its tracing close. */
+export interface PreemptRestartedRun {
+  /** The accumulated turn, carrying whatever usage was resolved for it. */
+  message: AIMessageChunk;
+  recordedAt: number;
+}
+
+/**
+ * Armed while anything is recorded, so expiry does not depend on another
+ * restart arriving to trigger it. Without this the final burst before a quiet
+ * period would hold its messages for the life of the process — and a host that
+ * enables preemption without tracing never consumes a single record, so that
+ * burst is the common case, not the corner one.
+ *
+ * `unref`'d: reclaiming a few messages is never a reason to keep a process
+ * alive.
+ */
+let sweepTimer: ReturnType<typeof setTimeout> | undefined;
+
+function sweepExpiredRestartedRuns(now: number): void {
+  for (const [runId, record] of preemptRestartedRuns) {
+    if (now - record.recordedAt < PREEMPT_RESTARTED_RUN_TTL_MS) {
+      break;
+    }
+    preemptRestartedRuns.delete(runId);
+  }
+  if (preemptRestartedRuns.size === 0) {
+    if (sweepTimer != null) {
+      clearTimeout(sweepTimer);
+      sweepTimer = undefined;
+    }
+    return;
+  }
+  if (sweepTimer != null) {
+    return;
+  }
+  sweepTimer = setTimeout(() => {
+    sweepTimer = undefined;
+    sweepExpiredRestartedRuns(Date.now());
+  }, PREEMPT_RESTARTED_RUN_TTL_MS);
+  sweepTimer.unref();
+}
+
+/** Records a model run whose stream was torn down for a restart. */
+export function notePreemptRestartedRun(
+  runId: string,
+  message: AIMessageChunk
+): void {
+  const now = Date.now();
+  preemptRestartedRuns.set(runId, { message, recordedAt: now });
+  sweepExpiredRestartedRuns(now);
+}
+
+/**
+ * The record for a run that ended because a preempt discarded it, or
+ * `undefined`.
+ *
+ * Deliberately NON-consuming. A run can be closed through more than one
+ * tracing handler — a run-level one plus another supplied through the model's
+ * own `clientOptions.callbacks`, which `endSealedModelRun` already composes
+ * for exactly that reason — and each receives the same `handleLLMError`. A
+ * read that deleted the record would let only the first handler recognize the
+ * restart, and every other destination would export the same expected abort as
+ * an error. Reuse is not a concern the way it would be for a counter: run ids
+ * are UUIDs, and the sweep is what reclaims them.
+ */
+export function readPreemptRestartedRun(
+  runId: string
+): PreemptRestartedRun | undefined {
+  return preemptRestartedRuns.get(runId);
+}
+
+/**
+ * Drops a record whose run turned out not to need it — the adapter ignored the
+ * abort, so the tear-down never reached LangChain's error path and the close
+ * happens manually instead.
+ */
+export function forgetPreemptRestartedRun(runId: string): void {
+  if (!preemptRestartedRuns.delete(runId)) {
+    return;
+  }
+  if (preemptRestartedRuns.size === 0 && sweepTimer != null) {
+    clearTimeout(sweepTimer);
+    sweepTimer = undefined;
+  }
+}

package/src/messages/core.ts

@@ -224,6 +224,34 @@ function getAdditionalReasoningContent(
   return getReasoningDetailsText(additionalKwargs.reasoning_details);
 }
 
+/**
+ * True when a content block is a provider reasoning block: Anthropic
+ * `thinking` / `redacted_thinking`, Google `reasoning`, Bedrock
+ * `reasoning_content`, and the streamed delta variants that prefix each of
+ * those names.
+ *
+ * Shared rather than re-derived because two very different readers must agree
+ * on the same answer: the stream classifier, which decides what the host has
+ * been shown, and the preempt gate, which decides whether an accumulated turn
+ * holds anything worth keeping. A turn the host renders as thinking-only must
+ * never look like a visible answer to the gate.
+ *
+ * Prefix matching, not equality, because providers stream these as suffixed
+ * delta types. `reasoning_content` needs no clause of its own — it already
+ * carries the `reasoning` prefix.
+ */
+export function isReasoningContentBlock(block: { type?: string }): boolean {
+  const type = block.type;
+  if (type == null) {
+    return false;
+  }
+  return (
+    type.startsWith(ContentTypes.THINKING) ||
+    type.startsWith(ContentTypes.REASONING) ||
+    type === 'redacted_thinking'
+  );
+}
+
 function hasReasoningContent(content: BaseMessage['content']): boolean {
   if (!Array.isArray(content)) {
     return false;

package/src/run.ts

@@ -942,12 +942,20 @@ export class Run<_T extends t.BaseGraphState> {
   }
 
   /**
-   * Cooperative-seal counters for this run. `emptyBoundaries` is the one to
-   * watch: it counts seals whose `PreemptBoundary` produced nothing to
-   * inject, which ends the turn early and leaves the answer unfinished.
+   * Cooperative-preemption counters for this run. `emptyBoundaries` is the one
+   * to watch: it counts SEALS whose `PreemptBoundary` produced nothing to
+   * inject, which ends the turn early and leaves the answer unfinished. A
+   * restart that injects nothing simply reissues the call, so it is not
+   * counted there.
    */
   getPreemptStats(): t.PreemptStats {
-    return this.Graph?.getPreemptStats() ?? { seals: 0, emptyBoundaries: 0 };
+    return (
+      this.Graph?.getPreemptStats() ?? {
+        seals: 0,
+        restarts: 0,
+        emptyBoundaries: 0,
+      }
+    );
   }
 
   getToolCount(): number {

package/src/stream.ts

@@ -56,6 +56,7 @@ import {
 } from '@/utils/truncation';
 import { resolveToolOutcome, outcomeFieldsFromResult } from '@/tools/intentArg';
 import { TOOL_OUTPUT_REF_PATTERN } from '@/tools/toolOutputReferences';
+import { isReasoningContentBlock } from '@/messages/core';
 import { safeDispatchCustomEvent } from '@/utils/events';
 import { composeAbortSignals } from '@/utils/misc';
 import { isGoogleLike } from '@/utils/llm';
@@ -454,15 +455,6 @@ function isTextContentPart(contentPart: t.MessageContentComplex): boolean {
   return contentPart.type?.startsWith(ContentTypes.TEXT) ?? false;
 }
 
-function isReasoningContentPart(contentPart: t.MessageContentComplex): boolean {
-  return (
-    (contentPart.type?.startsWith(ContentTypes.THINKING) ?? false) ||
-    (contentPart.type?.startsWith(ContentTypes.REASONING) ?? false) ||
-    (contentPart.type?.startsWith(ContentTypes.REASONING_CONTENT) ?? false) ||
-    contentPart.type === 'redacted_thinking'
-  );
-}
-
 function getReasoningTextFromContentPart(
   contentPart: t.MessageContentComplex
 ): string {
@@ -532,7 +524,7 @@ function shouldStartFreshMessageStepAfterGoogleServerSideTool({
   }
   return (
     content.every((c) => isTextContentPart(c)) ||
-    content.every((c) => isReasoningContentPart(c))
+    content.every((c) => isReasoningContentBlock(c))
   );
 }
 
@@ -649,7 +641,7 @@ async function dispatchGoogleServerSideToolStreamContent({
   }
   reasoningContent.push(
     ...content
-      .filter((contentPart) => isReasoningContentPart(contentPart))
+      .filter((contentPart) => isReasoningContentBlock(contentPart))
       .map((contentPart) => ({
         type: ContentTypes.THINK,
         think: getReasoningTextFromContentPart(contentPart),
@@ -2034,7 +2026,7 @@ hasToolCallChunks: ${hasToolCallChunks}
         },
         metadata
       );
-    } else if (content.every((c) => isReasoningContentPart(c))) {
+    } else if (content.every((c) => isReasoningContentBlock(c))) {
       await graph.dispatchReasoningDelta(
         stepId,
         {

package/src/types/run.ts

@@ -123,15 +123,23 @@ export type StandardGraphConfig = Omit<
 > & { type?: 'standard'; signal?: AbortSignal };
 
 /**
- * Cooperative mid-generation preemption. Lets a host seal the live model
- * stream at the next provider-safe token boundary — the run is never
- * aborted, the partial assistant turn is kept, and the graph self-loops
- * into a fresh model call once the `PreemptBoundary` hook has injected
- * whatever the host queued.
+ * Cooperative mid-generation preemption. Lets a host end the live model stream
+ * early and inject whatever it queued, in one of two ways:
+ *
+ *  - a SEAL, at the next provider-safe token boundary: the run is never
+ *    aborted, the partial assistant turn is kept, and the graph self-loops into
+ *    a fresh model call once the `PreemptBoundary` hook has injected.
+ *  - a RESTART, when the turn holds nothing worth keeping — no text, no tool
+ *    call, at most reasoning. The provider request IS torn down, its output is
+ *    discarded, and the model is called again with the injection appended to
+ *    the same prompt. Requires `subscribe`; see it for why the per-chunk poll
+ *    alone cannot reach this case.
  *
  * Preconditions the host MUST satisfy:
  *   - `shouldPreempt` is polled once per streamed chunk on the top-level
- *     graph. It must be synchronous, allocation-free and O(1) — never I/O.
+ *     graph, and once more per model attempt if `subscribe` is supplied — at
+ *     the attempt's start and on each wake. It must be synchronous,
+ *     allocation-free and O(1) — never I/O.
  *     It must also be LEVEL-TRIGGERED (non-consuming): the SDK never clears
  *     the host's request, and a true result is only honored once the
  *     accumulated chunk is provider-safe, so the predicate may be polled
@@ -168,6 +176,40 @@ export interface StreamPreemption {
    * consumes the request; a self-clearing read loses it on an unsafe chunk.
    */
   shouldPreempt: () => boolean;
+  /**
+   * Optional wake-up channel for the window where the poll cannot reach: a
+   * provider that has sent nothing yet, or that is streaming reasoning the
+   * seal gate will never accept. `shouldPreempt` is only read per chunk, so a
+   * request armed during a long silent stretch would otherwise wait for the
+   * whole turn.
+   *
+   * The SDK subscribes once per model attempt and calls the returned
+   * unsubscribe when the attempt ends. `wake` is a HINT, never the request
+   * itself: it wakes the SDK, which then re-reads `shouldPreempt` as the sole
+   * authority. That keeps the level-triggered contract above intact and makes
+   * spurious wakes free — a host may call it on every arm without tracking
+   * which ones the SDK already saw.
+   *
+   * A wake is only honored when the accumulated turn holds nothing worth
+   * keeping, and it is honored by DISCARDING that turn: the in-flight provider
+   * request is torn down, its partial output is dropped, the `PreemptBoundary`
+   * hook injects, and the model is called again with the injection appended.
+   * Reasoning tokens already spent are billed and lost, which is the trade the
+   * host is asking for. A turn that has produced visible text seals instead,
+   * through the ordinary per-chunk path, and a wake never discards it.
+   */
+  subscribe?: (wake: () => void) => () => void;
+  /**
+   * How long a request waits for the turn to become sealable before it may
+   * discard it instead. Defaults to `DEFAULT_PREEMPT_RESTART_GRACE_MS`; `0`
+   * converts as soon as the shape allows.
+   *
+   * The window keeps a restart from stealing a seal that was moments away, and
+   * from discarding a first chunk still in flight between the provider and the
+   * consumer. Both are why it applies to a silent turn as well as a thinking
+   * one.
+   */
+  restartGraceMs?: number;
   /**
    * Max cooperative seals per run. Each seal costs one extra superstep, so
    * this also bounds the recursion-limit headroom the run reserves.
@@ -175,9 +217,18 @@ export interface StreamPreemption {
   maxSeals?: number;
 }
 
-/** Seals honored and boundaries that had nothing to inject, per run. */
+/**
+ * Per run: seals honored, discards honored, and boundaries that had nothing to
+ * inject.
+ *
+ * `seals` counts only turns that were KEPT — a partial assistant message
+ * survived into the next prompt. `restarts` counts turns that were discarded
+ * and reissued, which preserve nothing, so a consumer classifying a run as
+ * "sealed" must not read them together.
+ */
 export type PreemptStats = {
   seals: number;
+  restarts: number;
   emptyBoundaries: number;
 };