@librechat/agents 3.2.61 → 3.2.63

package/src/tools/__tests__/hitl.test.ts

@@ -1566,6 +1566,118 @@ describe('ToolNode HITL — PostToolBatch hook', () => {
     expect(injected).toBeDefined();
     expect(String(injected!.content)).toContain('format the response as JSON');
   });
+
+  it('PostToolBatch injectedMessages land as individual HumanMessages after the consolidated context', async () => {
+    mockEventDispatch([
+      { toolCallId: 'call_1', content: 'ok', status: 'success' },
+    ]);
+
+    const registry = new HookRegistry();
+    registry.register('PostToolBatch', {
+      hooks: [
+        async (): Promise<PostToolBatchHookOutput> => ({
+          additionalContext: 'batch convention',
+          injectedMessages: [
+            { role: 'user', content: 'steer one', source: 'steer' },
+            { role: 'user', content: 'steer two', source: 'steer' },
+          ],
+        }),
+      ],
+    });
+
+    const node = new ToolNode({
+      tools: [createSchemaStub('echo')],
+      eventDrivenMode: true,
+      agentId: 'agent-x',
+      toolCallStepIds: new Map([['call_1', 'step_1']]),
+      hookRegistry: registry,
+      humanInTheLoop: { enabled: false },
+    });
+
+    const graph = buildHITLGraph(node, [
+      { id: 'call_1', name: 'echo', args: { command: 'a' } },
+    ]);
+    const result = (await graph.invoke(
+      { messages: [] },
+      { configurable: { thread_id: 'batch-steer-thread' } }
+    )) as { messages: BaseMessage[] };
+
+    type KwargMessage = {
+      additional_kwargs?: { source?: string; role?: string };
+    };
+    const humanMessages = result.messages.filter(
+      (m) => m._getType() === 'human'
+    );
+    const contextIndex = humanMessages.findIndex(
+      (m) => (m as KwargMessage).additional_kwargs?.source === 'hook'
+    );
+    const steerMessages = humanMessages.filter(
+      (m) => (m as KwargMessage).additional_kwargs?.source === 'steer'
+    );
+
+    expect(contextIndex).toBeGreaterThanOrEqual(0);
+    expect(steerMessages).toHaveLength(2);
+    expect(String(steerMessages[0].content)).toBe('steer one');
+    expect(String(steerMessages[1].content)).toBe('steer two');
+    for (const steer of steerMessages) {
+      expect((steer as KwargMessage).additional_kwargs?.role).toBe('user');
+      expect(humanMessages.indexOf(steer)).toBeGreaterThan(contextIndex);
+    }
+    const toolIndex = result.messages.findIndex((m) => m._getType() === 'tool');
+    const firstSteerIndex = result.messages.indexOf(steerMessages[0]);
+    expect(firstSteerIndex).toBeGreaterThan(toolIndex);
+  });
+
+  it('PostToolBatch injectedMessages work without additionalContext', async () => {
+    mockEventDispatch([
+      { toolCallId: 'call_1', content: 'ok', status: 'success' },
+    ]);
+
+    const registry = new HookRegistry();
+    registry.register('PostToolBatch', {
+      hooks: [
+        async (): Promise<PostToolBatchHookOutput> => ({
+          injectedMessages: [
+            { role: 'user', content: 'solo steer', source: 'steer' },
+          ],
+        }),
+      ],
+    });
+
+    const node = new ToolNode({
+      tools: [createSchemaStub('echo')],
+      eventDrivenMode: true,
+      agentId: 'agent-x',
+      toolCallStepIds: new Map([['call_1', 'step_1']]),
+      hookRegistry: registry,
+      humanInTheLoop: { enabled: false },
+    });
+
+    const graph = buildHITLGraph(node, [
+      { id: 'call_1', name: 'echo', args: { command: 'a' } },
+    ]);
+    const result = (await graph.invoke(
+      { messages: [] },
+      { configurable: { thread_id: 'solo-steer-thread' } }
+    )) as { messages: BaseMessage[] };
+
+    type KwargMessage = {
+      additional_kwargs?: { source?: string; role?: string };
+    };
+    const consolidated = result.messages.find(
+      (m) =>
+        m._getType() === 'human' &&
+        (m as KwargMessage).additional_kwargs?.source === 'hook'
+    );
+    const steer = result.messages.find(
+      (m) =>
+        m._getType() === 'human' &&
+        (m as KwargMessage).additional_kwargs?.source === 'steer'
+    );
+    expect(consolidated).toBeUndefined();
+    expect(steer).toBeDefined();
+    expect(String(steer!.content)).toBe('solo steer');
+  });
 });
 
 describe('ToolNode HITL — per-hook allowedDecisions override', () => {

package/src/tools/__tests__/subagentHooks.test.ts

@@ -4,6 +4,7 @@ import type { ToolCall } from '@langchain/core/messages/tool';
 import type {
   HookCallback,
   PermissionDeniedHookOutput,
+  PostToolBatchHookOutput,
   PostToolUseHookOutput,
   PreToolUseHookOutput,
   SubagentStartHookInput,
@@ -365,6 +366,129 @@ describe('Subagent hook integration (end-to-end via Run)', () => {
     expect(postExecEvents).toContain('researcher-child:calculator');
   });
 
+  it('top-level event-driven dispatches leave agentId unset (subagent-scope marker)', async () => {
+    getChatModelClassSpy.mockImplementation(((provider: Providers) => {
+      if (provider === Providers.OPENAI) {
+        return class extends FakeChatModel {
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          constructor(_options: any) {
+            super({
+              responses: ['Calculating.', 'All done.'],
+              sleep: 1,
+              toolCalls: [createCalculatorToolCall()],
+            });
+          }
+          bindTools(tools: unknown): ReturnType<FakeChatModel['withConfig']> {
+            const config = {
+              tools,
+            } as Parameters<FakeChatModel['withConfig']>[0];
+            return this.withConfig(config);
+          }
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        } as any;
+      }
+      return originalGetChatModelClass(provider);
+    }) as typeof providers.getChatModelClass);
+
+    const registry = new HookRegistry();
+    const scopeEvents: string[] = [];
+
+    const preHook: HookCallback<'PreToolUse'> = async (
+      input
+    ): Promise<PreToolUseHookOutput> => {
+      scopeEvents.push(
+        `pre:${input.agentId ?? '-'}:${input.executingAgentId ?? '-'}`
+      );
+      return { decision: 'allow' };
+    };
+    registry.register('PreToolUse', { hooks: [preHook] });
+
+    const postHook: HookCallback<'PostToolUse'> = async (
+      input
+    ): Promise<PostToolUseHookOutput> => {
+      scopeEvents.push(
+        `post:${input.agentId ?? '-'}:${input.executingAgentId ?? '-'}`
+      );
+      return {};
+    };
+    registry.register('PostToolUse', { hooks: [postHook] });
+
+    const batchHook: HookCallback<'PostToolBatch'> = async (
+      input
+    ): Promise<PostToolBatchHookOutput> => {
+      scopeEvents.push(
+        `batch:${input.agentId ?? '-'}:${input.executingAgentId ?? '-'}`
+      );
+      return {};
+    };
+    registry.register('PostToolBatch', { hooks: [batchHook] });
+
+    const dispatchAgentIds: Array<string | undefined> = [];
+    const customHandlers: Record<string, t.EventHandler> = {
+      [GraphEvents.TOOL_END]: new ToolEndHandler(),
+      [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(),
+      [GraphEvents.ON_TOOL_EXECUTE]: {
+        handle: (_event, rawData): void => {
+          const request = rawData as t.ToolExecuteBatchRequest;
+          dispatchAgentIds.push(request.agentId);
+          const results: t.ToolExecuteResult[] = request.toolCalls.map(
+            (call) => ({
+              toolCallId: call.id,
+              status: 'success',
+              content: '42',
+            })
+          );
+          request.resolve(results);
+        },
+      },
+    };
+
+    /**
+     * The LibreChat shape: a TOP-LEVEL agent whose tools ride
+     * `toolDefinitions` (event-driven ToolNode, host executes via
+     * ON_TOOL_EXECUTE). Hook inputs here must NOT carry the subagent-scope
+     * marker — a host hook keying on `agentId != null` (e.g. a steering
+     * drain that must never inject into child state) would otherwise skip
+     * every top-level batch. The DISPATCH payload is the opposite: hosts
+     * key tool/credential lookup on `request.agentId`, so it must keep
+     * identifying the owning agent.
+     */
+    const run = await Run.create<t.IState>({
+      runId: `toplevel-event-hook-${Date.now()}`,
+      graphConfig: {
+        type: 'standard',
+        agents: [
+          {
+            agentId: 'hook-parent',
+            provider: Providers.OPENAI,
+            clientOptions: { modelName: 'gpt-4o-mini', apiKey: 'test-key' },
+            instructions: 'Use the calculator.',
+            maxContextTokens: 8000,
+            toolDefinitions: [calculatorDef],
+          },
+        ],
+      },
+      returnContent: true,
+      skipCleanup: true,
+      customHandlers,
+      hooks: registry,
+    });
+
+    run.Graph!.overrideTestModel(['Calculating.', 'All done.'], 5, [
+      createCalculatorToolCall(),
+    ]);
+
+    await run.processStream(
+      { messages: [new HumanMessage('what is 21 * 2?')] },
+      callerConfig
+    );
+
+    expect(scopeEvents).toContain('pre:-:hook-parent');
+    expect(scopeEvents).toContain('post:-:hook-parent');
+    expect(scopeEvents).toContain('batch:-:hook-parent');
+    expect(dispatchAgentIds).toEqual(['hook-parent']);
+  });
+
   it('child subagent tool ask hooks fail closed instead of starting unsupported nested HITL', async () => {
     getChatModelClassSpy.mockImplementation(((provider: Providers) => {
       if (provider === Providers.OPENAI) {

package/src/tools/subagent/SubagentExecutor.ts

@@ -6,6 +6,7 @@ import type { ChatGeneration, LLMResult } from '@langchain/core/outputs';
 import type { Callbacks } from '@langchain/core/callbacks/manager';
 import type {
   AgentInputs,
+  InjectedMessage,
   MessageDeltaEvent,
   ProcessedToolCall,
   ReasoningDeltaEvent,
@@ -36,6 +37,7 @@ const MAX_PENDING_SUBAGENT_UPDATES = 64;
 
 const HOOK_FALLBACK: AggregatedHookResult = Object.freeze({
   additionalContexts: [] as string[],
+  injectedMessages: [] as InjectedMessage[],
   errors: [] as string[],
 });
 
@@ -371,6 +373,7 @@ export class SubagentExecutor {
       agents: [childInputs],
       langfuse: this.langfuse,
       tokenCounter: this.tokenCounter,
+      subagentScope: true,
       /**
        * Forwarded so the child graph's own `SubagentExecutor` (created in
        * its `createAgentNode` when `allowNested` keeps subagentConfigs)

package/src/types/graph.ts

@@ -336,6 +336,16 @@ export type StandardGraphInput = {
    * they already flow through the registry's `CHAT_MODEL_END` handler.
    */
   subagentUsageSink?: SubagentUsageSink;
+  /**
+   * True when this graph IS a subagent child run (set by `SubagentExecutor`
+   * when it constructs the child graph). Drives the hook-input `agentId`
+   * subagent-scope marker: hook dispatches from this graph's tool nodes
+   * carry `agentId` so run-scoped host hooks — which fire for child scopes
+   * too, because children inherit the parent's `run_id` — can tell child
+   * scope from the top level. Top-level graphs leave this unset and their
+   * hook inputs carry only `executingAgentId`.
+   */
+  subagentScope?: boolean;
 };
 
 export type GraphEdge = {

package/src/types/llm.ts

@@ -45,7 +45,7 @@ export type AzureClientOptions = Partial<OpenAIChatInput> &
     deploymentName?: string;
   } & BaseChatModelParams & {
     configuration?: OAIClientOptions;
-  };
+  } & ManagedRequestOptions;
 /**
  * Controls whether Claude's reasoning content is returned in adaptive
  * thinking responses. Added for Claude Opus 4.7, which omits thinking by
@@ -73,7 +73,13 @@ export type GoogleThinkingConfig = {
   includeThoughts?: boolean;
   thinkingLevel?: 'THINKING_LEVEL_UNSPECIFIED' | 'LOW' | 'MEDIUM' | 'HIGH';
 };
-export type OpenAIClientOptions = ChatOpenAIFields;
+/** GPT-5.6 managed-request passthrough fields, shared by the OpenAI and
+ *  Azure wrappers that both read them. */
+export type ManagedRequestOptions = {
+  promptCacheExplicit?: boolean;
+  safety_identifier?: string;
+};
+export type OpenAIClientOptions = ChatOpenAIFields & ManagedRequestOptions;
 export type AnthropicClientOptions = Omit<AnthropicInput, 'thinking'> & {
   thinking?: ThinkingConfig;
   promptCache?: boolean;
@@ -100,6 +106,10 @@ export type BedrockAnthropicInput = ChatBedrockConverseInput & {
    * server-side, so the default is safe to leave on.
    */
   promptCacheTtl?: PromptCacheTtl;
+  /**
+   * Minimum delay in milliseconds between visible streamed content deltas.
+   */
+  _lc_stream_delay?: number;
 };
 export type BedrockConverseClientOptions = BedrockAnthropicInput;
 export type BedrockAnthropicClientOptions = BedrockAnthropicInput;

package/src/types/tools.ts

@@ -516,7 +516,7 @@ export type InjectedMessage = {
   /** When true, the message is framework-internal: not shown in UI, not counted as a user turn */
   isMeta?: boolean;
   /** Origin tag for downstream consumers (UI, pruner, compaction) */
-  source?: 'skill' | 'hook' | 'system';
+  source?: 'skill' | 'hook' | 'system' | 'steer';
   /** Only set when source is 'skill', for compaction preservation */
   skillName?: string;
 };

package/src/utils/tokens.ts

@@ -17,7 +17,7 @@ const OPENAI_IMAGE_TOKENS_PER_TILE = 170;
 /** Google Gemini fixed per-image cost. */
 const _GEMINI_IMAGE_TOKENS = 258;
 /** Safety margin for image and document token estimates (5% overestimate). */
-const IMAGE_TOKEN_SAFETY_MARGIN = 1.05;
+export const IMAGE_TOKEN_SAFETY_MARGIN = 1.05;
 
 /**
  * Anthropic PDF: each page costs image tokens + text tokens.
@@ -33,6 +33,33 @@ const BASE64_BYTES_PER_PDF_PAGE = 75_000;
 /** Fallback token cost for URL-referenced documents without local data. */
 const URL_DOCUMENT_FALLBACK_TOKENS = 2000;
 
+/**
+ * Timed media (video/audio) is priced by DURATION, not size, and the content
+ * carries no duration — so duration is estimated from encoded size at
+ * representative bitrates and priced at Gemini's rates (the dominant provider
+ * for native video/audio). Deliberately rough; superseded by real provider
+ * usage after the first turn.
+ */
+/** Gemini video ≈ 258 tokens/frame (1 fps) + 32 tokens/s audio ≈ 300 tokens/s. */
+const VIDEO_TOKENS_PER_SECOND = 300;
+/** Gemini audio: 32 tokens/s (single channel). */
+const AUDIO_TOKENS_PER_SECOND = 32;
+/** Representative encoded byte rates for duration-from-size estimation. */
+const VIDEO_BYTES_PER_SECOND = 250_000; // ~2 Mbps
+const AUDIO_BYTES_PER_SECOND = 16_000; // ~128 kbps
+/** Flat fallback when only a URL is present (no size to estimate from) — ~30s. */
+const VIDEO_URL_FALLBACK_TOKENS = 9000;
+const AUDIO_URL_FALLBACK_TOKENS = 960;
+/** Content block types that can carry timed media (video/audio). `media` is
+ *  generic (Google) so it is classified by MIME; the rest are unambiguous. */
+const TIMED_MEDIA_TYPES = new Set([
+  'media',
+  'video',
+  'audio',
+  'video_url',
+  'input_audio',
+]);
+
 /**
  * Extracts image dimensions from the first bytes of a base64-encoded
  * PNG, JPEG, GIF, or WebP without decoding the full image.
@@ -135,7 +162,7 @@ export function estimateOpenAIImageTokens(
  * Extracts dimensions from base64 header when available.
  * Falls back to Anthropic minimum (1024) when dimensions can't be determined.
  */
-function estimateImageBlockTokens(
+export function estimateImageBlockTokens(
   block: Record<string, unknown>,
   encoding: EncodingName
 ): number {
@@ -180,7 +207,7 @@ function estimateImageBlockTokens(
  * - Base64 PDF: page count estimated from base64 length × per-page cost.
  * - URL reference: conservative flat estimate.
  */
-function estimateDocumentBlockTokens(
+export function estimateDocumentBlockTokens(
   block: Record<string, unknown>,
   encoding: EncodingName,
   getTokenCount: (text: string) => number
@@ -287,6 +314,150 @@ function estimateDocumentBlockTokens(
   return URL_DOCUMENT_FALLBACK_TOKENS;
 }
 
+/** Decoded byte length of base64 or a `data:` URL; 0 for any remote URI (http,
+ *  gs, s3, file, …) or empty. Base64 never contains `:`, so any scheme prefix
+ *  marks a remote reference with no local size. */
+function base64ByteLength(value: string | undefined): number {
+  if (typeof value !== 'string' || value.length === 0) {
+    return 0;
+  }
+  if (value.startsWith('data:')) {
+    const comma = value.indexOf(',');
+    return comma < 0 ? 0 : Math.floor(((value.length - comma - 1) * 3) / 4);
+  }
+  if (/^[a-z][a-z0-9+.-]*:/i.test(value)) {
+    return 0;
+  }
+  return Math.floor((value.length * 3) / 4);
+}
+
+function timedMediaTokens(
+  bytes: number,
+  bytesPerSecond: number,
+  tokensPerSecond: number,
+  urlFallback: number
+): number {
+  if (bytes <= 0) {
+    return urlFallback;
+  }
+  return Math.max(
+    tokensPerSecond,
+    Math.ceil((bytes / bytesPerSecond) * tokensPerSecond)
+  );
+}
+
+/** Decoded byte length of a media block payload — top-level `data` (base64
+ *  string or `Uint8Array`) or base64 `url`, or the nested `video|audio.{data,
+ *  url, source.bytes}` shapes (`formatMessage` media arrays / native Bedrock);
+ *  0 for a bare remote URL, `fileId`/`fileUri`, S3 location, or empty. */
+function mediaBlockByteLength(block: Record<string, unknown>): number {
+  const data = block.data;
+  if (typeof data === 'string') {
+    return base64ByteLength(data);
+  }
+  if (data instanceof Uint8Array) {
+    return data.length;
+  }
+  if (typeof block.url === 'string') {
+    return base64ByteLength(block.url);
+  }
+  const nested = (block.video ?? block.audio) as
+    | { data?: unknown; url?: unknown; source?: { bytes?: unknown } }
+    | undefined;
+  if (nested != null) {
+    if (typeof nested.data === 'string') {
+      return base64ByteLength(nested.data);
+    }
+    if (typeof nested.url === 'string') {
+      return base64ByteLength(nested.url);
+    }
+    if (nested.source?.bytes instanceof Uint8Array) {
+      return nested.source.bytes.length;
+    }
+  }
+  return 0;
+}
+
+/** Classifies a block as timed media, or null when it is not video/audio — e.g.
+ *  a generic Google `media` block carrying an image/document MIME, which must
+ *  NOT be priced as video. Also handles the Google shape where `type` IS the
+ *  MIME string (e.g. `{ type: 'audio/wav', data }`). */
+function timedMediaKind(
+  block: Record<string, unknown>
+): 'video' | 'audio' | null {
+  const type = typeof block.type === 'string' ? block.type : '';
+  if (type === 'input_audio' || type === 'audio') {
+    return 'audio';
+  }
+  if (type === 'video_url' || type === 'video') {
+    return 'video';
+  }
+  const mime =
+    type === 'media' && typeof block.mimeType === 'string' ? block.mimeType : type;
+  if (mime.startsWith('audio/')) {
+    return 'audio';
+  }
+  if (mime.startsWith('video/')) {
+    return 'video';
+  }
+  return null;
+}
+
+/** Whether a content-block `type` can carry timed media — the fixed set plus the
+ *  Google MIME-as-type shape (`audio/*` / `video/*`). Gates callers before they
+ *  hand the block to {@link estimateTimedMediaBlockTokens}. */
+function isTimedMediaType(type: string): boolean {
+  return (
+    TIMED_MEDIA_TYPES.has(type) ||
+    type.startsWith('audio/') ||
+    type.startsWith('video/')
+  );
+}
+
+/**
+ * Estimates token cost for a timed-media block (video/audio). Handles Google
+ * `{ type: 'media', mimeType, data|url|fileUri }`, OpenRouter
+ * `{ type: 'video_url' }` / `{ type: 'input_audio' }`, and standard
+ * `{ type: 'video' }` / `{ type: 'audio' }` blocks (payload as `data` base64 or
+ * `Uint8Array`, base64 `url`, or `fileId`). Duration is inferred from encoded
+ * size (providers price by duration, which the block does not carry) at Gemini's
+ * rates; a payload with no size (bare URL / file id) falls back to a ~30s
+ * estimate. Returns 0 for non-timed media (e.g. an image/document `media` block)
+ * so it is never mispriced as video.
+ */
+export function estimateTimedMediaBlockTokens(
+  block: Record<string, unknown>
+): number {
+  const kind = timedMediaKind(block);
+  if (kind == null) {
+    return 0;
+  }
+  let bytes: number;
+  if (block.type === 'input_audio') {
+    const audio = block.input_audio as { data?: string } | undefined;
+    bytes = base64ByteLength(audio?.data);
+  } else if (block.type === 'video_url') {
+    const video = block.video_url as string | { url?: string } | undefined;
+    bytes = base64ByteLength(typeof video === 'string' ? video : video?.url);
+  } else {
+    bytes = mediaBlockByteLength(block);
+  }
+  if (kind === 'audio') {
+    return timedMediaTokens(
+      bytes,
+      AUDIO_BYTES_PER_SECOND,
+      AUDIO_TOKENS_PER_SECOND,
+      AUDIO_URL_FALLBACK_TOKENS
+    );
+  }
+  return timedMediaTokens(
+    bytes,
+    VIDEO_BYTES_PER_SECOND,
+    VIDEO_TOKENS_PER_SECOND,
+    VIDEO_URL_FALLBACK_TOKENS
+  );
+}
+
 const tokenizers: Partial<Record<EncodingName, Tokenizer>> = {};
 
 async function getTokenizer(
@@ -358,6 +529,13 @@ export function getTokenCountForMessage(
           continue;
         }
 
+        if (isTimedMediaType(item.type)) {
+          numTokens += Math.ceil(
+            estimateTimedMediaBlockTokens(item) * IMAGE_TOKEN_SAFETY_MARGIN
+          );
+          continue;
+        }
+
         if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {
           const toolName = item.tool_call.name;
           if (typeof toolName === 'string' && toolName.length > 0) {