@librechat/agents 3.2.61 → 3.2.62

package/src/tools/__tests__/hitl.test.ts

@@ -1566,6 +1566,118 @@ describe('ToolNode HITL — PostToolBatch hook', () => {
     expect(injected).toBeDefined();
     expect(String(injected!.content)).toContain('format the response as JSON');
   });
+
+  it('PostToolBatch injectedMessages land as individual HumanMessages after the consolidated context', async () => {
+    mockEventDispatch([
+      { toolCallId: 'call_1', content: 'ok', status: 'success' },
+    ]);
+
+    const registry = new HookRegistry();
+    registry.register('PostToolBatch', {
+      hooks: [
+        async (): Promise<PostToolBatchHookOutput> => ({
+          additionalContext: 'batch convention',
+          injectedMessages: [
+            { role: 'user', content: 'steer one', source: 'steer' },
+            { role: 'user', content: 'steer two', source: 'steer' },
+          ],
+        }),
+      ],
+    });
+
+    const node = new ToolNode({
+      tools: [createSchemaStub('echo')],
+      eventDrivenMode: true,
+      agentId: 'agent-x',
+      toolCallStepIds: new Map([['call_1', 'step_1']]),
+      hookRegistry: registry,
+      humanInTheLoop: { enabled: false },
+    });
+
+    const graph = buildHITLGraph(node, [
+      { id: 'call_1', name: 'echo', args: { command: 'a' } },
+    ]);
+    const result = (await graph.invoke(
+      { messages: [] },
+      { configurable: { thread_id: 'batch-steer-thread' } }
+    )) as { messages: BaseMessage[] };
+
+    type KwargMessage = {
+      additional_kwargs?: { source?: string; role?: string };
+    };
+    const humanMessages = result.messages.filter(
+      (m) => m._getType() === 'human'
+    );
+    const contextIndex = humanMessages.findIndex(
+      (m) => (m as KwargMessage).additional_kwargs?.source === 'hook'
+    );
+    const steerMessages = humanMessages.filter(
+      (m) => (m as KwargMessage).additional_kwargs?.source === 'steer'
+    );
+
+    expect(contextIndex).toBeGreaterThanOrEqual(0);
+    expect(steerMessages).toHaveLength(2);
+    expect(String(steerMessages[0].content)).toBe('steer one');
+    expect(String(steerMessages[1].content)).toBe('steer two');
+    for (const steer of steerMessages) {
+      expect((steer as KwargMessage).additional_kwargs?.role).toBe('user');
+      expect(humanMessages.indexOf(steer)).toBeGreaterThan(contextIndex);
+    }
+    const toolIndex = result.messages.findIndex((m) => m._getType() === 'tool');
+    const firstSteerIndex = result.messages.indexOf(steerMessages[0]);
+    expect(firstSteerIndex).toBeGreaterThan(toolIndex);
+  });
+
+  it('PostToolBatch injectedMessages work without additionalContext', async () => {
+    mockEventDispatch([
+      { toolCallId: 'call_1', content: 'ok', status: 'success' },
+    ]);
+
+    const registry = new HookRegistry();
+    registry.register('PostToolBatch', {
+      hooks: [
+        async (): Promise<PostToolBatchHookOutput> => ({
+          injectedMessages: [
+            { role: 'user', content: 'solo steer', source: 'steer' },
+          ],
+        }),
+      ],
+    });
+
+    const node = new ToolNode({
+      tools: [createSchemaStub('echo')],
+      eventDrivenMode: true,
+      agentId: 'agent-x',
+      toolCallStepIds: new Map([['call_1', 'step_1']]),
+      hookRegistry: registry,
+      humanInTheLoop: { enabled: false },
+    });
+
+    const graph = buildHITLGraph(node, [
+      { id: 'call_1', name: 'echo', args: { command: 'a' } },
+    ]);
+    const result = (await graph.invoke(
+      { messages: [] },
+      { configurable: { thread_id: 'solo-steer-thread' } }
+    )) as { messages: BaseMessage[] };
+
+    type KwargMessage = {
+      additional_kwargs?: { source?: string; role?: string };
+    };
+    const consolidated = result.messages.find(
+      (m) =>
+        m._getType() === 'human' &&
+        (m as KwargMessage).additional_kwargs?.source === 'hook'
+    );
+    const steer = result.messages.find(
+      (m) =>
+        m._getType() === 'human' &&
+        (m as KwargMessage).additional_kwargs?.source === 'steer'
+    );
+    expect(consolidated).toBeUndefined();
+    expect(steer).toBeDefined();
+    expect(String(steer!.content)).toBe('solo steer');
+  });
 });
 
 describe('ToolNode HITL — per-hook allowedDecisions override', () => {

package/src/tools/subagent/SubagentExecutor.ts

@@ -6,6 +6,7 @@ import type { ChatGeneration, LLMResult } from '@langchain/core/outputs';
 import type { Callbacks } from '@langchain/core/callbacks/manager';
 import type {
   AgentInputs,
+  InjectedMessage,
   MessageDeltaEvent,
   ProcessedToolCall,
   ReasoningDeltaEvent,
@@ -36,6 +37,7 @@ const MAX_PENDING_SUBAGENT_UPDATES = 64;
 
 const HOOK_FALLBACK: AggregatedHookResult = Object.freeze({
   additionalContexts: [] as string[],
+  injectedMessages: [] as InjectedMessage[],
   errors: [] as string[],
 });
 

package/src/types/llm.ts

@@ -45,7 +45,7 @@ export type AzureClientOptions = Partial<OpenAIChatInput> &
     deploymentName?: string;
   } & BaseChatModelParams & {
     configuration?: OAIClientOptions;
-  };
+  } & ManagedRequestOptions;
 /**
  * Controls whether Claude's reasoning content is returned in adaptive
  * thinking responses. Added for Claude Opus 4.7, which omits thinking by
@@ -73,7 +73,13 @@ export type GoogleThinkingConfig = {
   includeThoughts?: boolean;
   thinkingLevel?: 'THINKING_LEVEL_UNSPECIFIED' | 'LOW' | 'MEDIUM' | 'HIGH';
 };
-export type OpenAIClientOptions = ChatOpenAIFields;
+/** GPT-5.6 managed-request passthrough fields, shared by the OpenAI and
+ *  Azure wrappers that both read them. */
+export type ManagedRequestOptions = {
+  promptCacheExplicit?: boolean;
+  safety_identifier?: string;
+};
+export type OpenAIClientOptions = ChatOpenAIFields & ManagedRequestOptions;
 export type AnthropicClientOptions = Omit<AnthropicInput, 'thinking'> & {
   thinking?: ThinkingConfig;
   promptCache?: boolean;
@@ -100,6 +106,10 @@ export type BedrockAnthropicInput = ChatBedrockConverseInput & {
    * server-side, so the default is safe to leave on.
    */
   promptCacheTtl?: PromptCacheTtl;
+  /**
+   * Minimum delay in milliseconds between visible streamed content deltas.
+   */
+  _lc_stream_delay?: number;
 };
 export type BedrockConverseClientOptions = BedrockAnthropicInput;
 export type BedrockAnthropicClientOptions = BedrockAnthropicInput;

package/src/types/tools.ts

@@ -516,7 +516,7 @@ export type InjectedMessage = {
   /** When true, the message is framework-internal: not shown in UI, not counted as a user turn */
   isMeta?: boolean;
   /** Origin tag for downstream consumers (UI, pruner, compaction) */
-  source?: 'skill' | 'hook' | 'system';
+  source?: 'skill' | 'hook' | 'system' | 'steer';
   /** Only set when source is 'skill', for compaction preservation */
   skillName?: string;
 };

package/src/utils/tokens.ts

@@ -17,7 +17,7 @@ const OPENAI_IMAGE_TOKENS_PER_TILE = 170;
 /** Google Gemini fixed per-image cost. */
 const _GEMINI_IMAGE_TOKENS = 258;
 /** Safety margin for image and document token estimates (5% overestimate). */
-const IMAGE_TOKEN_SAFETY_MARGIN = 1.05;
+export const IMAGE_TOKEN_SAFETY_MARGIN = 1.05;
 
 /**
  * Anthropic PDF: each page costs image tokens + text tokens.
@@ -33,6 +33,33 @@ const BASE64_BYTES_PER_PDF_PAGE = 75_000;
 /** Fallback token cost for URL-referenced documents without local data. */
 const URL_DOCUMENT_FALLBACK_TOKENS = 2000;
 
+/**
+ * Timed media (video/audio) is priced by DURATION, not size, and the content
+ * carries no duration — so duration is estimated from encoded size at
+ * representative bitrates and priced at Gemini's rates (the dominant provider
+ * for native video/audio). Deliberately rough; superseded by real provider
+ * usage after the first turn.
+ */
+/** Gemini video ≈ 258 tokens/frame (1 fps) + 32 tokens/s audio ≈ 300 tokens/s. */
+const VIDEO_TOKENS_PER_SECOND = 300;
+/** Gemini audio: 32 tokens/s (single channel). */
+const AUDIO_TOKENS_PER_SECOND = 32;
+/** Representative encoded byte rates for duration-from-size estimation. */
+const VIDEO_BYTES_PER_SECOND = 250_000; // ~2 Mbps
+const AUDIO_BYTES_PER_SECOND = 16_000; // ~128 kbps
+/** Flat fallback when only a URL is present (no size to estimate from) — ~30s. */
+const VIDEO_URL_FALLBACK_TOKENS = 9000;
+const AUDIO_URL_FALLBACK_TOKENS = 960;
+/** Content block types that can carry timed media (video/audio). `media` is
+ *  generic (Google) so it is classified by MIME; the rest are unambiguous. */
+const TIMED_MEDIA_TYPES = new Set([
+  'media',
+  'video',
+  'audio',
+  'video_url',
+  'input_audio',
+]);
+
 /**
  * Extracts image dimensions from the first bytes of a base64-encoded
  * PNG, JPEG, GIF, or WebP without decoding the full image.
@@ -135,7 +162,7 @@ export function estimateOpenAIImageTokens(
  * Extracts dimensions from base64 header when available.
  * Falls back to Anthropic minimum (1024) when dimensions can't be determined.
  */
-function estimateImageBlockTokens(
+export function estimateImageBlockTokens(
   block: Record<string, unknown>,
   encoding: EncodingName
 ): number {
@@ -180,7 +207,7 @@ function estimateImageBlockTokens(
  * - Base64 PDF: page count estimated from base64 length × per-page cost.
  * - URL reference: conservative flat estimate.
  */
-function estimateDocumentBlockTokens(
+export function estimateDocumentBlockTokens(
   block: Record<string, unknown>,
   encoding: EncodingName,
   getTokenCount: (text: string) => number
@@ -287,6 +314,150 @@ function estimateDocumentBlockTokens(
   return URL_DOCUMENT_FALLBACK_TOKENS;
 }
 
+/** Decoded byte length of base64 or a `data:` URL; 0 for any remote URI (http,
+ *  gs, s3, file, …) or empty. Base64 never contains `:`, so any scheme prefix
+ *  marks a remote reference with no local size. */
+function base64ByteLength(value: string | undefined): number {
+  if (typeof value !== 'string' || value.length === 0) {
+    return 0;
+  }
+  if (value.startsWith('data:')) {
+    const comma = value.indexOf(',');
+    return comma < 0 ? 0 : Math.floor(((value.length - comma - 1) * 3) / 4);
+  }
+  if (/^[a-z][a-z0-9+.-]*:/i.test(value)) {
+    return 0;
+  }
+  return Math.floor((value.length * 3) / 4);
+}
+
+function timedMediaTokens(
+  bytes: number,
+  bytesPerSecond: number,
+  tokensPerSecond: number,
+  urlFallback: number
+): number {
+  if (bytes <= 0) {
+    return urlFallback;
+  }
+  return Math.max(
+    tokensPerSecond,
+    Math.ceil((bytes / bytesPerSecond) * tokensPerSecond)
+  );
+}
+
+/** Decoded byte length of a media block payload — top-level `data` (base64
+ *  string or `Uint8Array`) or base64 `url`, or the nested `video|audio.{data,
+ *  url, source.bytes}` shapes (`formatMessage` media arrays / native Bedrock);
+ *  0 for a bare remote URL, `fileId`/`fileUri`, S3 location, or empty. */
+function mediaBlockByteLength(block: Record<string, unknown>): number {
+  const data = block.data;
+  if (typeof data === 'string') {
+    return base64ByteLength(data);
+  }
+  if (data instanceof Uint8Array) {
+    return data.length;
+  }
+  if (typeof block.url === 'string') {
+    return base64ByteLength(block.url);
+  }
+  const nested = (block.video ?? block.audio) as
+    | { data?: unknown; url?: unknown; source?: { bytes?: unknown } }
+    | undefined;
+  if (nested != null) {
+    if (typeof nested.data === 'string') {
+      return base64ByteLength(nested.data);
+    }
+    if (typeof nested.url === 'string') {
+      return base64ByteLength(nested.url);
+    }
+    if (nested.source?.bytes instanceof Uint8Array) {
+      return nested.source.bytes.length;
+    }
+  }
+  return 0;
+}
+
+/** Classifies a block as timed media, or null when it is not video/audio — e.g.
+ *  a generic Google `media` block carrying an image/document MIME, which must
+ *  NOT be priced as video. Also handles the Google shape where `type` IS the
+ *  MIME string (e.g. `{ type: 'audio/wav', data }`). */
+function timedMediaKind(
+  block: Record<string, unknown>
+): 'video' | 'audio' | null {
+  const type = typeof block.type === 'string' ? block.type : '';
+  if (type === 'input_audio' || type === 'audio') {
+    return 'audio';
+  }
+  if (type === 'video_url' || type === 'video') {
+    return 'video';
+  }
+  const mime =
+    type === 'media' && typeof block.mimeType === 'string' ? block.mimeType : type;
+  if (mime.startsWith('audio/')) {
+    return 'audio';
+  }
+  if (mime.startsWith('video/')) {
+    return 'video';
+  }
+  return null;
+}
+
+/** Whether a content-block `type` can carry timed media — the fixed set plus the
+ *  Google MIME-as-type shape (`audio/*` / `video/*`). Gates callers before they
+ *  hand the block to {@link estimateTimedMediaBlockTokens}. */
+function isTimedMediaType(type: string): boolean {
+  return (
+    TIMED_MEDIA_TYPES.has(type) ||
+    type.startsWith('audio/') ||
+    type.startsWith('video/')
+  );
+}
+
+/**
+ * Estimates token cost for a timed-media block (video/audio). Handles Google
+ * `{ type: 'media', mimeType, data|url|fileUri }`, OpenRouter
+ * `{ type: 'video_url' }` / `{ type: 'input_audio' }`, and standard
+ * `{ type: 'video' }` / `{ type: 'audio' }` blocks (payload as `data` base64 or
+ * `Uint8Array`, base64 `url`, or `fileId`). Duration is inferred from encoded
+ * size (providers price by duration, which the block does not carry) at Gemini's
+ * rates; a payload with no size (bare URL / file id) falls back to a ~30s
+ * estimate. Returns 0 for non-timed media (e.g. an image/document `media` block)
+ * so it is never mispriced as video.
+ */
+export function estimateTimedMediaBlockTokens(
+  block: Record<string, unknown>
+): number {
+  const kind = timedMediaKind(block);
+  if (kind == null) {
+    return 0;
+  }
+  let bytes: number;
+  if (block.type === 'input_audio') {
+    const audio = block.input_audio as { data?: string } | undefined;
+    bytes = base64ByteLength(audio?.data);
+  } else if (block.type === 'video_url') {
+    const video = block.video_url as string | { url?: string } | undefined;
+    bytes = base64ByteLength(typeof video === 'string' ? video : video?.url);
+  } else {
+    bytes = mediaBlockByteLength(block);
+  }
+  if (kind === 'audio') {
+    return timedMediaTokens(
+      bytes,
+      AUDIO_BYTES_PER_SECOND,
+      AUDIO_TOKENS_PER_SECOND,
+      AUDIO_URL_FALLBACK_TOKENS
+    );
+  }
+  return timedMediaTokens(
+    bytes,
+    VIDEO_BYTES_PER_SECOND,
+    VIDEO_TOKENS_PER_SECOND,
+    VIDEO_URL_FALLBACK_TOKENS
+  );
+}
+
 const tokenizers: Partial<Record<EncodingName, Tokenizer>> = {};
 
 async function getTokenizer(
@@ -358,6 +529,13 @@ export function getTokenCountForMessage(
           continue;
         }
 
+        if (isTimedMediaType(item.type)) {
+          numTokens += Math.ceil(
+            estimateTimedMediaBlockTokens(item) * IMAGE_TOKEN_SAFETY_MARGIN
+          );
+          continue;
+        }
+
         if (item.type === ContentTypes.TOOL_CALL && item.tool_call != null) {
           const toolName = item.tool_call.name;
           if (typeof toolName === 'string' && toolName.length > 0) {