@librechat/agents 3.3.11 → 3.3.12

package/src/graphs/__tests__/Graph.breakerLifecycle.test.ts

@@ -0,0 +1,234 @@
+import { describe, it, expect } from '@jest/globals';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type * as t from '@/types';
+import { StreamLimitExceededError } from '@/llm/streamLimits';
+import { Providers } from '@/common';
+import { StandardGraph } from '../Graph';
+
+const makeAgent = (agentId: string): t.AgentInputs => ({
+  agentId,
+  provider: Providers.OPENAI,
+  instructions: `You are ${agentId}.`,
+});
+
+const makeGraph = (): StandardGraph =>
+  new StandardGraph({
+    runId: 'run_1',
+    agents: [makeAgent('agent')],
+  });
+
+const makeTrip = (): StreamLimitExceededError =>
+  new StreamLimitExceededError({
+    kind: 'tool_call_args',
+    limit: 10,
+    observed: 11,
+    toolName: 'db_query',
+  });
+
+describe('run breaker lifecycle', () => {
+  it('replaces the breaker at every run start, even when un-aborted', () => {
+    const graph = makeGraph();
+    const previous = graph.breakerAbort;
+    expect(previous.signal.aborted).toBe(false);
+
+    graph.resetValues();
+
+    expect(graph.breakerAbort).not.toBe(previous);
+    /** A straggler from the failed run trips the controller it captured;
+     * the run starting now must not observe it. */
+    previous.abort(makeTrip());
+    expect(graph.breakerAbort.signal.aborted).toBe(false);
+  });
+
+  it('keeps straggler accounting through cleanup and one reset, then sweeps it', () => {
+    const graph = makeGraph();
+    const epoch = graph.breakerEpoch;
+    graph.streamedToolCallArgTallies.set('gen:i:0', { bytes: 42, epoch });
+    graph.streamDeltaEventCounts.set('gen', { count: 7, epoch });
+    graph.streamLimitChargeCredits = new WeakMap();
+
+    /** Cleanup runs while sibling attempts can still be unwinding on the
+     * retained breaker; clearing here would hand a cancellation-ignoring
+     * provider's late chunks a fresh budget. */
+    graph.clearHeavyState();
+    expect(graph.streamedToolCallArgTallies.get('gen:i:0')?.bytes).toBe(42);
+    expect(graph.streamDeltaEventCounts.get('gen')?.count).toBe(7);
+    expect(graph.streamLimitChargeCredits).toBeDefined();
+
+    /** Producer loops of straggling attempts sit OUTSIDE the consumer-only
+     * epoch gate, so their entries must survive the next run start too —
+     * one grace reset keeps them on their original budgets. */
+    graph.resetValues();
+    expect(graph.streamedToolCallArgTallies.get('gen:i:0')?.bytes).toBe(42);
+    expect(graph.streamDeltaEventCounts.get('gen')?.count).toBe(7);
+    expect(graph.streamLimitChargeCredits).toBeUndefined();
+
+    /** A second reset sweeps entries from older epochs. */
+    graph.resetValues();
+    expect(graph.streamedToolCallArgTallies.size).toBe(0);
+    expect(graph.streamDeltaEventCounts.size).toBe(0);
+  });
+
+  it('holds a leased producer straggler to its budget across two run resets', async () => {
+    const { attemptInvoke } = await import('@/llm/invoke');
+    const { resolveStreamLimits } = await import('@/llm/streamLimits');
+    const { AIMessageChunk, HumanMessage } = await import(
+      '@langchain/core/messages'
+    );
+    const graph = makeGraph();
+    graph.streamLimits = resolveStreamLimits({ maxToolCallArgBytes: 100 });
+    const chunkOf = (args: string): InstanceType<typeof AIMessageChunk> =>
+      new AIMessageChunk({
+        content: '',
+        tool_call_chunks: [
+          {
+            type: 'tool_call_chunk',
+            id: 'call_1',
+            name: 'writer',
+            args,
+            index: 0,
+          },
+        ],
+      });
+    let yields = 0;
+    const model = {
+      stream: async function* (): AsyncGenerator<
+        InstanceType<typeof AIMessageChunk>
+        > {
+        yields += 1;
+        yield chunkOf('x'.repeat(60));
+        /** Two prompt runs start and reset while this attempt is still
+         * draining. Its lease must keep the tally alive — reset counting
+         * cannot prove the attempt finished. */
+        graph.resetValues();
+        graph.resetValues();
+        yields += 1;
+        yield chunkOf('x'.repeat(60));
+        yields += 1;
+        yield chunkOf('x');
+      },
+    };
+
+    await expect(
+      attemptInvoke(
+        {
+          model: model as unknown as t.ChatModel,
+          messages: [new HumanMessage('hi')],
+          provider: Providers.OPENAI,
+          context: graph as Parameters<
+            typeof attemptInvoke
+          >[0]['context'],
+          onChunk: () => {
+            /* producer loop enforcement is under test */
+          },
+        },
+        { metadata: {} }
+      )
+    ).rejects.toMatchObject({ kind: 'tool_call_args', limit: 100 });
+    expect(yields).toBe(2);
+    /** The attempt's finally released its lease and entries. */
+    expect(graph.activeStreamLimitGenerations?.size ?? 0).toBe(0);
+  });
+
+  it('takes no attempt lease when every guard is disabled', async () => {
+    const { attemptInvoke } = await import('@/llm/invoke');
+    const { resolveStreamLimits } = await import('@/llm/streamLimits');
+    const { AIMessage, HumanMessage } = await import(
+      '@langchain/core/messages'
+    );
+    const graph = makeGraph();
+    graph.streamLimits = resolveStreamLimits({
+      maxToolCallArgBytes: 0,
+      maxToolCallArgBytesByTool: { writer: 0 },
+    });
+    const model = {
+      invoke: async (): Promise<InstanceType<typeof AIMessage>> =>
+        new AIMessage({ content: 'ok' }),
+    };
+
+    await attemptInvoke(
+      {
+        model: model as unknown as t.ChatModel,
+        messages: [new HumanMessage('hi')],
+        provider: Providers.OPENAI,
+        context: graph as Parameters<typeof attemptInvoke>[0]['context'],
+      },
+      { metadata: {} }
+    );
+
+    /** The lease only protects accounting entries; with no guard able to
+     * fire there are none, and per-attempt bookkeeping must not allocate. */
+    expect(graph.activeStreamLimitGenerations).toBeUndefined();
+  });
+
+  it('holds a post-reset producer straggler to its original byte budget', async () => {
+    const { enforceStreamLimitsForWireChunk } = await import(
+      '@/llm/streamLimits'
+    );
+    const graph = makeGraph();
+    graph.streamLimits = {
+      maxToolCallArgBytes: 100,
+      maxDeltaEventsPerTurn: 0,
+      hasEnforceableToolCallArgLimit: true,
+    };
+    const metadata = {
+      langgraph_checkpoint_ns: '',
+      langgraph_node: 'agent',
+      langgraph_step: 1,
+    };
+    const chunkOf = (
+      args: string
+    ): { tool_call_chunks: Array<Record<string, unknown>> } => ({
+      tool_call_chunks: [{ id: 'call_1', name: 'writer', args, index: 0 }],
+    });
+
+    enforceStreamLimitsForWireChunk({
+      graph,
+      metadata,
+      chunk: chunkOf('x'.repeat(60)) as Parameters<
+        typeof enforceStreamLimitsForWireChunk
+      >[0]['chunk'],
+    });
+
+    /** The next run starts while this producer is still draining. Its next
+     * chunk must land on the ORIGINAL 60-byte tally and trip, not open a
+     * fresh allowance. */
+    graph.resetValues();
+    expect(() =>
+      enforceStreamLimitsForWireChunk({
+        graph,
+        metadata,
+        chunk: chunkOf('x'.repeat(60)) as Parameters<
+          typeof enforceStreamLimitsForWireChunk
+        >[0]['chunk'],
+      })
+    ).toThrow(StreamLimitExceededError);
+  });
+
+  it('rejects a model node at entry when the breaker has already tripped', async () => {
+    const graph = makeGraph();
+    const trip = makeTrip();
+    graph.breakerAbort.abort(trip);
+
+    const node = graph.createCallModel('agent');
+    await expect(
+      node(
+        { messages: [] } as unknown as t.AgentSubgraphState,
+        {} as RunnableConfig
+      )
+    ).rejects.toBe(trip);
+  });
+
+  it('lets a model node proceed past entry when the breaker is live', async () => {
+    const graph = makeGraph();
+    const node = graph.createCallModel('agent');
+    /** With a live breaker the entry guard must not fire; the node then
+     * fails later, on the missing config, proving it got past the guard. */
+    await expect(
+      node(
+        { messages: [] } as unknown as t.AgentSubgraphState,
+        undefined
+      )
+    ).rejects.toThrow('No config provided');
+  });
+});

package/src/index.ts

@@ -82,3 +82,14 @@ export { initializeModel } from './llm/init';
 export { attemptInvoke, tryFallbackProviders } from './llm/invoke';
 export { canSealPreempt } from './llm/preempt';
 export { isThinkingEnabled, getMaxOutputTokensKey } from './llm/request';
+export {
+  DEFAULT_MAX_TOOL_CALL_ARG_BYTES,
+  StreamLimitExceededError,
+  resolveStreamLimits,
+} from './llm/streamLimits';
+export type {
+  StreamedToolCallArgTally,
+  ResolvedStreamLimits,
+  StreamLimitState,
+  StreamLimitKind,
+} from './llm/streamLimits';

package/src/llm/bedrock/index.ts

@@ -46,6 +46,7 @@ import {
   supportsBedrockToolCache,
   type PromptCacheTtl,
 } from '@/messages/cache';
+import { linkStreamLimitCanonical } from '@/llm/streamLimits';
 import { applyCachePointsToConversePayload } from './cachePoints';
 import { insertBedrockToolCachePoint } from './toolCache';
 
@@ -572,8 +573,17 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
           }
 
           const deltaChunk = handleConverseStreamContentBlockDelta(splitDelta);
+          const enrichedChunk = this.enrichChunk(deltaChunk, seenBlockIndices);
+          if (enrichedChunk !== deltaChunk) {
+            /** The callback copy is the same emission as the enriched yield;
+             * without the link, stream-limit accounting charges both message
+             * objects and Bedrock tool arguments falsely trip near half the
+             * cap. Linked on the messages, which are what the accounting
+             * observes. */
+            linkStreamLimitCanonical(deltaChunk.message, enrichedChunk.message);
+          }
           await enqueueChunk({
-            chunk: this.enrichChunk(deltaChunk, seenBlockIndices),
+            chunk: enrichedChunk,
             callbackChunk: deltaChunk,
             callbackToken: deltaChunk.text,
             smooth,
@@ -602,8 +612,18 @@ export class CustomChatBedrockConverse extends ChatBedrockConverse {
                     toolUseBlockIndices.add(idx);
                   }
                 }
+                const enrichedStart = this.enrichChunk(
+                  startChunk,
+                  seenBlockIndices
+                );
+                if (enrichedStart !== startChunk) {
+                  linkStreamLimitCanonical(
+                    startChunk.message,
+                    enrichedStart.message
+                  );
+                }
                 await enqueueChunk({
-                  chunk: this.enrichChunk(startChunk, seenBlockIndices),
+                  chunk: enrichedStart,
                   callbackChunk: startChunk,
                   callbackToken: startChunk.text,
                 });

package/src/llm/invoke.streamLimits.test.ts

@@ -0,0 +1,142 @@
+/**
+ * `attemptInvoke`'s local stream branch (no registered SDK dispatcher) must
+ * charge the event budget even for chunks that produce no handling chunk:
+ * a pure OpenRouter reasoning-replay chunk is skipped for content handling,
+ * and in this branch no `streamEvents` consumer counts the wire event
+ * either, so a looping replay stream would otherwise bypass an enabled
+ * `maxDeltaEventsPerTurn` indefinitely.
+ */
+import { AIMessageChunk, HumanMessage } from '@langchain/core/messages';
+import { describe, it, expect, jest } from '@jest/globals';
+import type { AgentContext } from '@/agents/AgentContext';
+import type { StandardGraph } from '@/graphs';
+import type * as t from '@/types';
+import {
+  StreamLimitExceededError,
+  resolveStreamLimits,
+} from '@/llm/streamLimits';
+import { attemptInvoke } from '@/llm/invoke';
+import { ContentTypes, Providers } from '@/common';
+
+function createContext(
+  overrides: Partial<StandardGraph> = {}
+): StandardGraph {
+  const runSteps = new Map<string, t.RunStep>();
+  const stepIdsByKey = new Map<string, string>();
+  let stepCounter = 0;
+
+  const graph = {
+    config: { configurable: { user_id: 'user_1' }, metadata: { run_id: 'r1' } },
+    eagerEventToolExecution: undefined,
+    eagerEventToolExecutions: new Map(),
+    eagerEventToolCallChunks: new Map(),
+    eagerEventToolSuppressions: new Set<string>(),
+    handlerRegistry: undefined,
+    hookRegistry: undefined,
+    humanInTheLoop: undefined,
+    toolOutputReferences: undefined,
+    sessions: new Map(),
+    toolCallStepIds: new Map(),
+    messageIdsByStepKey: new Map(),
+    messageStepHasToolCalls: new Map(),
+    prelimMessageIdsByStepKey: new Map([['step-key', 'msg_1']]),
+    getAgentContext: jest.fn(
+      (): Partial<AgentContext> => ({
+        provider: Providers.OPENAI,
+        reasoningKey: 'reasoning_content',
+        currentTokenType: ContentTypes.TEXT,
+        toolDefinitions: [],
+        graphTools: [],
+        agentId: 'agent_1',
+      })
+    ),
+    getOrCreateToolOutputRegistry: jest.fn(() => undefined),
+    shouldPreemptStream: jest.fn(() => false),
+    claimPreemptSeal: jest.fn(() => false),
+    getStepKey: jest.fn(() => 'step-key'),
+    getStepIdByKey: jest.fn((stepKey: string) => {
+      const stepId = stepIdsByKey.get(stepKey);
+      if (stepId == null) {
+        throw new Error('no current step');
+      }
+      return stepId;
+    }),
+    getRunStep: jest.fn((stepId: string) => runSteps.get(stepId)),
+    dispatchRunStep: jest.fn(async (stepKey: string, details: unknown) => {
+      const id = `step_${++stepCounter}`;
+      stepIdsByKey.set(stepKey, id);
+      runSteps.set(id, {
+        id,
+        type: (details as { type: t.RunStep['type'] }).type,
+        stepDetails: details as t.RunStep['stepDetails'],
+      } as t.RunStep);
+      return id;
+    }),
+    dispatchRunStepDelta: jest.fn(async () => undefined),
+    dispatchMessageDelta: jest.fn(async () => undefined),
+    dispatchReasoningDelta: jest.fn(async () => undefined),
+    ...overrides,
+  };
+  return graph as unknown as StandardGraph;
+}
+
+const replayChunk = (): AIMessageChunk =>
+  new AIMessageChunk({
+    content: 'abc',
+    additional_kwargs: { reasoning_details: [{ type: 'reasoning.text' }] },
+  });
+
+function createReplayModel(replays: number): t.ChatModel {
+  return {
+    stream: async function* stream(): AsyncGenerator<AIMessageChunk> {
+      yield new AIMessageChunk({ content: 'abc' });
+      for (let i = 0; i < replays; i++) {
+        yield replayChunk();
+      }
+    },
+  } as unknown as t.ChatModel;
+}
+
+describe('attemptInvoke local branch stream limits', () => {
+  it('charges skipped OpenRouter replay chunks against the event budget', async () => {
+    const context = createContext({
+      streamLimits: resolveStreamLimits({ maxDeltaEventsPerTurn: 3 }),
+    });
+
+    let caught: unknown;
+    try {
+      await attemptInvoke(
+        {
+          model: createReplayModel(10),
+          messages: [new HumanMessage('hi')],
+          provider: Providers.OPENROUTER,
+          context: context as unknown as Parameters<
+            typeof attemptInvoke
+          >[0]['context'],
+        },
+        { metadata: { langgraph_node: 'agent', langgraph_step: 1 } }
+      );
+    } catch (error) {
+      caught = error;
+    }
+    expect(caught).toBeInstanceOf(StreamLimitExceededError);
+    expect((caught as StreamLimitExceededError).kind).toBe('delta_events');
+  });
+
+  it('leaves replay streams unbounded when the cap is disabled', async () => {
+    const context = createContext({ streamLimits: resolveStreamLimits() });
+
+    const result = await attemptInvoke(
+      {
+        model: createReplayModel(10),
+        messages: [new HumanMessage('hi')],
+        provider: Providers.OPENROUTER,
+        context: context as unknown as Parameters<
+          typeof attemptInvoke
+        >[0]['context'],
+      },
+      { metadata: { langgraph_node: 'agent', langgraph_step: 1 } }
+    );
+    expect(result.messages).toHaveLength(1);
+  });
+});

package/src/llm/invoke.test.ts

@@ -19,6 +19,7 @@ import { _convertMessagesToAnthropicPayload } from '@/llm/anthropic/utils/messag
 import { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
 import { convertMessageContentToParts } from '@/llm/google/utils/common';
 import { _convertMessagesToOpenAIParams } from '@/llm/openai/utils';
+import { StreamLimitExceededError } from '@/llm/streamLimits';
 import { attemptInvoke, tryFallbackProviders } from '@/llm/invoke';
 import { toLangChainContent } from '@/messages/langchain';
 import { Constants, Providers } from '@/common';
@@ -556,7 +557,7 @@ describe('attemptInvoke applies lazy ref annotation', () => {
     responses.model.defaultOptions = { tools: [] };
     responses.model._useResponsesApi = (options) =>
       Array.isArray((options as { tools?: unknown[] } | undefined)?.tools) &&
-      ((options as { tools: unknown[] }).tools.length ?? 0) > 0;
+      (options as { tools: unknown[] }).tools.length > 0;
 
     await attemptInvoke(
       {
@@ -829,6 +830,93 @@ describe('tryFallbackProviders applies the same lazy annotation transform', () =
     jest.resetModules();
   });
 
+  it('stops draining a cancellation-ignoring stream once the breaker trips', async () => {
+    const abort = new AbortController();
+    /** Statically imported class: neighboring tests reset the module
+     * registry, and a dynamically imported instance would fail the
+     * statically-linked drain loop's instanceof check. */
+    const trip = new StreamLimitExceededError({
+      kind: 'tool_call_args',
+      limit: 10,
+      observed: 11,
+      toolName: 'db_query',
+    });
+    let yields = 0;
+    const model: StubModel = {
+      stream: jest.fn(async function* (): AsyncGenerator<AIMessageChunk> {
+        for (let i = 0; i < 5; i++) {
+          yields += 1;
+          yield new AIMessageChunk({ content: `c${i}` });
+        }
+      }),
+    };
+
+    /** The stub ignores the abort signal entirely; only the per-chunk
+     * breaker check in the drain loop can stop it. */
+    await expect(
+      attemptInvoke(
+        {
+          model: model as t.ChatModel,
+          messages: [new HumanMessage('hi')],
+          provider: Providers.ANTHROPIC,
+          onChunk: () => {
+            abort.abort(trip);
+          },
+        },
+        { signal: abort.signal }
+      )
+    ).rejects.toBe(trip);
+    expect(yields).toBeLessThanOrEqual(2);
+  });
+
+  it('rejects before invoking a fallback when the breaker tripped during preparation', async () => {
+    const abort = new AbortController();
+    const { invokeMessages, model } = buildCapturingModel();
+    await jest.isolateModulesAsync(async () => {
+      jest.doMock('@/llm/init', () => ({
+        initializeModel: (): unknown => model,
+      }));
+      const { tryFallbackProviders: freshTry } = (await import(
+        '@/llm/invoke'
+      )) as { tryFallbackProviders: typeof tryFallbackProviders };
+      const { StreamLimitExceededError: FreshStreamLimitError } = (await import(
+        '@/llm/streamLimits'
+      )) as typeof import('@/llm/streamLimits');
+      const trip = new FreshStreamLimitError({
+        kind: 'tool_call_args',
+        limit: 10,
+        observed: 11,
+        toolName: 'db_query',
+      });
+
+      /** The sibling's trip lands while fallback preparation is awaited; the
+       * fallback must not be invoked at all — a provider that ignores the
+       * aborted signal and SUCCEEDS would resolve a run that must reject. */
+      await expect(
+        freshTry({
+          fallbacks: [{ provider: Providers.ANTHROPIC }],
+          messages: [
+            new ToolMessage({
+              name: 'echo',
+              tool_call_id: 'tc1',
+              status: 'success',
+              content: 'output',
+            }),
+          ],
+          primaryError: new Error('primary failed'),
+          config: { signal: abort.signal },
+          prepareProviderMessages: async ({ messages: preparedMessages }) => {
+            abort.abort(trip);
+            return preparedMessages;
+          },
+        })
+      ).rejects.toBe(trip);
+    });
+    expect(invokeMessages).toHaveLength(0);
+
+    jest.dontMock('@/llm/init');
+  });
+
   it('neutralizes preempted Responses history before an OpenAI Chat fallback', async () => {
     const reasoning = {
       id: 'rs_fallback',