@librechat/agents 3.2.61 → 3.2.63

package/src/specs/cache.simple.test.ts

@@ -11,7 +11,7 @@ import {
 } from '@langchain/core/messages';
 import type * as t from '@/types';
 import { ModelEndHandler, ToolEndHandler } from '@/events';
-import { capitalizeFirstLetter } from './spec.utils';
+import { capitalizeFirstLetter, hasEnv, hasEveryEnv } from './spec.utils';
 import { createContentAggregator } from '@/stream';
 import { GraphEvents, Providers } from '@/common';
 import { getLLMConfig } from '@/utils/llmConfig';
@@ -82,7 +82,20 @@ describe('Prompt Caching Integration Tests', () => {
     version: 'v2' as const,
   };
 
-  describe('Anthropic Prompt Caching', () => {
+  const requiredBedrockEnv = [
+    'BEDROCK_AWS_REGION',
+    'BEDROCK_AWS_ACCESS_KEY_ID',
+    'BEDROCK_AWS_SECRET_ACCESS_KEY',
+  ];
+  const hasAnthropicApiKey = hasEnv('ANTHROPIC_API_KEY');
+  const hasBedrockCredentials = hasEveryEnv(requiredBedrockEnv);
+
+  const describeIfAnthropic = hasAnthropicApiKey ? describe : describe.skip;
+  const describeIfBedrock = hasBedrockCredentials ? describe : describe.skip;
+  const describeIfAnthropicAndBedrock =
+    hasAnthropicApiKey && hasBedrockCredentials ? describe : describe.skip;
+
+  describeIfAnthropic('Anthropic Prompt Caching', () => {
     const provider = Providers.ANTHROPIC;
 
     test(`${capitalizeFirstLetter(provider)}: multi-turn conversation with caching should not corrupt messages`, async () => {
@@ -212,7 +225,7 @@ describe('Prompt Caching Integration Tests', () => {
     });
   });
 
-  describe('Bedrock Prompt Caching', () => {
+  describeIfBedrock('Bedrock Prompt Caching', () => {
     const provider = Providers.BEDROCK;
 
     test(`${capitalizeFirstLetter(provider)}: multi-turn conversation with caching should not corrupt messages`, async () => {
@@ -340,7 +353,7 @@ describe('Prompt Caching Integration Tests', () => {
     });
   });
 
-  describe('Cross-provider message isolation', () => {
+  describeIfAnthropicAndBedrock('Cross-provider message isolation', () => {
     test('Messages processed by Anthropic should not affect Bedrock processing', async () => {
       const anthropicConfig = getLLMConfig(Providers.ANTHROPIC);
       const bedrockConfig = getLLMConfig(Providers.BEDROCK);

package/src/specs/langfuse-callbacks.test.ts

@@ -1,7 +1,9 @@
 import { HumanMessage } from '@langchain/core/messages';
+import { LangfuseOtelSpanAttributes } from '@langfuse/tracing';
 import { CallbackManager } from '@langchain/core/callbacks/manager';
 import { context as otelContext, trace as otelTrace } from '@opentelemetry/api';
 import type * as t from '@/types';
+import { handleConverseStreamMetadata } from '@/llm/bedrock/utils/message_outputs';
 import { traceIdFromSeed } from '@/langfuseRuntimeContext';
 import { Providers } from '@/common';
 import { Run } from '@/run';
@@ -387,6 +389,65 @@ describe('Langfuse callback composition', () => {
     });
   });
 
+  it('exports Bedrock prompt-cache usage buckets to Langfuse', async () => {
+    const { createLangfuseHandler } = await import('@/langfuse');
+    const { initializeLangfuseTracing } = await import('@/instrumentation');
+    initializeLangfuseTracing({
+      publicKey: 'pk-test',
+      secretKey: 'sk-test',
+    });
+    const handler = createLangfuseHandler({
+      langfuse: {
+        publicKey: 'pk-test',
+        secretKey: 'sk-test',
+      },
+    });
+    const runId = 'test-langfuse-bedrock-cache-usage';
+
+    await handler?.handleChatModelStart(
+      {
+        lc: 1,
+        type: 'constructor',
+        id: ['LibreChatBedrockConverse'],
+        kwargs: {},
+      },
+      [[new HumanMessage('hello')]],
+      runId
+    );
+
+    const generation = handleConverseStreamMetadata(
+      {
+        usage: {
+          inputTokens: 13,
+          outputTokens: 5,
+          totalTokens: 20849,
+          cacheReadInputTokens: 10831,
+          cacheWriteInputTokens: 10000,
+        },
+        metrics: { latencyMs: 1000 },
+      },
+      { streamUsage: true }
+    );
+    await handler?.handleLLMEnd({ generations: [[generation]] }, runId);
+
+    const usageDetails = mockSpanAttributeSets
+      .map(
+        (attributes) =>
+          attributes[LangfuseOtelSpanAttributes.OBSERVATION_USAGE_DETAILS]
+      )
+      .find((value): value is string => typeof value === 'string');
+
+    expect(usageDetails).toBe(
+      JSON.stringify({
+        input: 13,
+        output: 5,
+        total: 20849,
+        input_cache_read: 10831,
+        input_cache_creation: 10000,
+      })
+    );
+  });
+
   it('uses deterministic trace ids when tracing is configured from env only', async () => {
     process.env.LANGFUSE_PUBLIC_KEY = 'pk-env';
     process.env.LANGFUSE_SECRET_KEY = 'sk-env';

package/src/specs/openai.simple.test.ts

@@ -16,14 +16,16 @@ import {
   createMetadataAggregator,
 } from '@/events';
 import { ContentTypes, GraphEvents, Providers, TitleMethod } from '@/common';
-import { capitalizeFirstLetter } from './spec.utils';
+import { capitalizeFirstLetter, hasEnv } from './spec.utils';
 import { createContentAggregator } from '@/stream';
 import { getLLMConfig } from '@/utils/llmConfig';
 import { getArgs } from '@/scripts/args';
 import { Run } from '@/run';
 
 const provider = Providers.OPENAI;
-describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
+const describeIfOpenAI = hasEnv('OPENAI_API_KEY') ? describe : describe.skip;
+
+describeIfOpenAI(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
   jest.setTimeout(30000);
   let run: Run<t.IState>;
   let runningHistory: BaseMessage[];

package/src/specs/spec.utils.ts

@@ -1,3 +1,15 @@
 export function capitalizeFirstLetter(string: string): string {
   return string.charAt(0).toUpperCase() + string.slice(1);
 }
+
+export function hasEnv(key: string): boolean {
+  return (process.env[key]?.trim() ?? '') !== '';
+}
+
+export function hasEveryEnv(keys: readonly string[]): boolean {
+  return keys.every(hasEnv);
+}
+
+export function hasAnyEnv(keys: readonly string[]): boolean {
+  return keys.some(hasEnv);
+}

package/src/specs/summarization.test.ts

@@ -21,6 +21,7 @@ import { Run } from '@/run';
 import { formatAgentMessages } from '@/messages/format';
 import { FakeListChatModel } from '@langchain/core/utils/testing';
 import * as providers from '@/llm/providers';
+import { hasAnyEnv, hasEnv, hasEveryEnv } from './spec.utils';
 
 const SUMMARY_WRAPPER_OVERHEAD_TOKENS = 33;
 
@@ -287,7 +288,7 @@ function logTurn(
 // Anthropic Summarization Tests
 // ---------------------------------------------------------------------------
 
-const hasAnthropic = process.env.ANTHROPIC_API_KEY != null;
+const hasAnthropic = hasEnv('ANTHROPIC_API_KEY');
 (hasAnthropic ? describe : describe.skip)('Anthropic Summarization E2E', () => {
   jest.setTimeout(180_000);
 
@@ -599,7 +600,7 @@ const hasAnthropic = process.env.ANTHROPIC_API_KEY != null;
   }, 180_000);
 
   test('cross-provider summarization: Anthropic agent with OpenAI summarizer', async () => {
-    const hasOpenAI = process.env.OPENAI_API_KEY != null;
+    const hasOpenAI = hasEnv('OPENAI_API_KEY');
     if (!hasOpenAI) {
       console.log('  Skipping cross-provider test (no OPENAI_API_KEY)');
       return;
@@ -1026,7 +1027,7 @@ const requiredBedrockEnv = [
   'BEDROCK_AWS_ACCESS_KEY_ID',
   'BEDROCK_AWS_SECRET_ACCESS_KEY',
 ];
-const hasBedrock = requiredBedrockEnv.every((k) => process.env[k] != null);
+const hasBedrock = hasEveryEnv(requiredBedrockEnv);
 
 (hasBedrock ? describe : describe.skip)('Bedrock Summarization E2E', () => {
   jest.setTimeout(180_000);
@@ -1159,7 +1160,7 @@ const hasBedrock = requiredBedrockEnv.every((k) => process.env[k] != null);
 // OpenAI Summarization Tests
 // ---------------------------------------------------------------------------
 
-const hasOpenAI = process.env.OPENAI_API_KEY != null;
+const hasOpenAI = hasEnv('OPENAI_API_KEY');
 (hasOpenAI ? describe : describe.skip)('OpenAI Summarization E2E', () => {
   jest.setTimeout(120_000);
 
@@ -2111,7 +2112,6 @@ describe('Tight context with oversized tool results (no API keys)', () => {
             constructor(_options: any) {
               super({ responses: [SUMMARY_RESPONSE] });
             }
-            // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
             async *_streamResponseChunks(
               messages: any[],
               options: any,
@@ -2419,17 +2419,14 @@ describe('Tight context with oversized tool results (no API keys)', () => {
 // Token accounting audit (requires API keys)
 // ---------------------------------------------------------------------------
 
-const hasAnyApiKey =
-  process.env.ANTHROPIC_API_KEY != null || process.env.OPENAI_API_KEY != null;
+const hasAnyApiKey = hasAnyEnv(['ANTHROPIC_API_KEY', 'OPENAI_API_KEY']);
 
 (hasAnyApiKey ? describe : describe.skip)('Token accounting audit', () => {
   jest.setTimeout(180_000);
 
-  const agentProvider =
-    process.env.ANTHROPIC_API_KEY != null &&
-    process.env.ANTHROPIC_API_KEY !== ''
-      ? Providers.ANTHROPIC
-      : Providers.OPENAI;
+  const agentProvider = hasEnv('ANTHROPIC_API_KEY')
+    ? Providers.ANTHROPIC
+    : Providers.OPENAI;
   const summarizationProvider = agentProvider;
   const summarizationModel =
     agentProvider === Providers.ANTHROPIC ? 'claude-haiku-4-5' : 'gpt-4.1-mini';
@@ -2960,7 +2957,6 @@ describe('Summarization deduplication correctness (no API keys)', () => {
               chunkCallCount++;
               super({ responses: [response] });
             }
-            // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
             async *_streamResponseChunks(
               messages: any[],
               options: any,

package/src/specs/token-accounting-pipeline.test.ts

@@ -727,7 +727,7 @@ describe('Token accounting pipeline — end-to-end multi-turn with calibration',
 });
 
 describe('Token accounting pipeline — Anthropic vs OpenAI cache semantics', () => {
-  it('Anthropic additive cache inflates calibration input correctly', () => {
+  it('Anthropic normalized cache input calibrates from input_tokens only', () => {
     const messages = [new HumanMessage('Hello'), new AIMessage('Hi')];
 
     const indexTokenCountMap: Record<string, number | undefined> = {
@@ -744,12 +744,12 @@ describe('Token accounting pipeline — Anthropic vs OpenAI cache semantics', ()
       reserveRatio: 0,
     });
 
-    // Anthropic: input_tokens=30, cache_read=60, cache_creation=20
-    // cacheSum=80 > baseInput=30 → additive → totalInput=30+80=110
+    // Anthropic output normalization stores cache-inclusive prompt tokens in
+    // input_tokens and repeats the cache components in input_token_details.
     const result = pruneMessages({
       messages,
       usageMetadata: {
-        input_tokens: 30,
+        input_tokens: 110,
         output_tokens: 15,
         input_token_details: { cache_read: 60, cache_creation: 20 },
       },
@@ -761,6 +761,132 @@ describe('Token accounting pipeline — Anthropic vs OpenAI cache semantics', ()
     expect(result.calibrationRatio).toBeCloseTo(1.1, 1);
   });
 
+  it('skips Bedrock cache-artifact turns instead of poisoning instruction overhead', () => {
+    const messages = [
+      new SystemMessage('system instructions'),
+      new HumanMessage('first user turn'),
+      new AIMessage('first response'),
+      new HumanMessage('second user turn'),
+      new AIMessage('second response'),
+    ];
+    const indexTokenCountMap: Record<string, number | undefined> = {
+      0: 3197,
+      1: 8000,
+      2: 7000,
+      3: 9506,
+      4: 1,
+    };
+    const instructionTokens = 5450;
+    const logs: Array<{
+      level: string;
+      message: string;
+      data?: Record<string, unknown>;
+    }> = [];
+
+    const pruneMessages = createPruneMessages({
+      provider: Providers.BEDROCK,
+      maxTokens: 126935,
+      startIndex: messages.length,
+      tokenCounter: charCounter,
+      indexTokenCountMap,
+      reserveRatio: 0.05,
+      getInstructionTokens: () => instructionTokens,
+      log: (level, message, data) => logs.push({ level, message, data }),
+    });
+
+    const result = pruneMessages({
+      messages,
+      usageMetadata: {
+        input_tokens: 6,
+        output_tokens: 1,
+        input_token_details: {
+          cache_creation: 380974,
+          cache_read: 13165,
+        },
+      },
+    });
+
+    expect(result.calibrationRatio).toBe(1);
+    expect(result.resolvedInstructionOverhead).toBeUndefined();
+    expect(result.remainingContextTokens).toBeGreaterThan(0);
+    expect(
+      logs.find((entry) => entry.message === 'Calibration skipped')?.data
+    ).toMatchObject({
+      reason: 'input_below_instruction_overhead',
+      providerInputTokens: 6,
+      instructionOverhead: instructionTokens,
+    });
+    expect(
+      logs.find((entry) => entry.message === 'Budget')?.data
+    ).toMatchObject({
+      instructionTokens,
+    });
+  });
+
+  it('accepts provider usage above the application budget and prunes', () => {
+    // maxTokens is an application budget, not the provider's context window.
+    // The provider accepts requests past the budget and reports true usage —
+    // exactly the turns where calibration must feed pruning/summarization.
+    const messages = [
+      new SystemMessage('system instructions'),
+      new HumanMessage('first user turn'),
+      new AIMessage('first response'),
+      new HumanMessage('second user turn'),
+      new AIMessage('second response'),
+    ];
+    const indexTokenCountMap: Record<string, number | undefined> = {
+      0: 40,
+      1: 120,
+      2: 110,
+      3: 120,
+      4: 80,
+    };
+    const rawTotal = 470;
+    const providerInputTokens = 620;
+    const logs: Array<{
+      level: string;
+      message: string;
+      data?: Record<string, unknown>;
+    }> = [];
+
+    const pruneMessages = createPruneMessages({
+      provider: Providers.ANTHROPIC,
+      maxTokens: 500,
+      startIndex: messages.length,
+      tokenCounter: charCounter,
+      indexTokenCountMap,
+      summarizationEnabled: true,
+      reserveRatio: 0,
+      log: (level, message, data) => logs.push({ level, message, data }),
+    });
+
+    const result = pruneMessages({
+      messages,
+      usageMetadata: {
+        input_tokens: providerInputTokens,
+        output_tokens: 25,
+      },
+    });
+
+    // The above-budget measurement calibrates instead of being discarded.
+    expect(
+      logs.find((entry) => entry.message === 'Calibration skipped')
+    ).toBeUndefined();
+    expect(
+      logs.find((entry) => entry.message === 'Calibration observed')?.data
+    ).toMatchObject({ providerInputTokens });
+    expect(result.calibrationRatio).toBeCloseTo(
+      providerInputTokens / rawTotal,
+      2
+    );
+
+    // Calibrated context (620) exceeds the budget (500), so messages are
+    // yielded for summarization instead of returning the full context.
+    expect(result.messagesToRefine).toBeDefined();
+    expect(result.messagesToRefine!.length).toBeGreaterThan(0);
+    expect(result.context.length).toBeLessThan(messages.length);
+  });
+
   it('OpenAI inclusive cache does NOT inflate calibration input', () => {
     const messages = [new HumanMessage('Hello'), new AIMessage('Hi')];
 

package/src/specs/tokens.test.ts

@@ -3,8 +3,27 @@ import {
   encodingForModel,
   createTokenCounter,
   TokenEncoderManager,
+  estimateImageBlockTokens,
+  estimateDocumentBlockTokens,
+  estimateTimedMediaBlockTokens,
 } from '@/utils/tokens';
 
+/** Builds a minimal PNG data URI whose IHDR encodes the given dimensions. */
+function pngDataUri(width: number, height: number): string {
+  const buf = Buffer.alloc(48);
+  buf[0] = 0x89;
+  buf[1] = 0x50;
+  buf[2] = 0x4e;
+  buf[3] = 0x47;
+  buf[4] = 0x0d;
+  buf[5] = 0x0a;
+  buf[6] = 0x1a;
+  buf[7] = 0x0a;
+  buf.writeUInt32BE(width, 16);
+  buf.writeUInt32BE(height, 20);
+  return `data:image/png;base64,${buf.toString('base64')}`;
+}
+
 describe('encodingForModel', () => {
   test('returns claude for Claude model strings', () => {
     expect(encodingForModel('claude-3-5-sonnet-20241022')).toBe('claude');
@@ -62,3 +81,198 @@ describe('createTokenCounter with different encodings', () => {
     expect(o200kCounter(msg)).toBeGreaterThan(0);
   });
 });
+
+describe('estimateImageBlockTokens', () => {
+  test('claude: tokens = ceil(w*h/750), floored at 1024', () => {
+    const block = { type: 'image_url', image_url: { url: pngDataUri(1024, 768) } };
+    // 1024*768/750 = 1048.58 -> ceil 1049 (> 1024 floor)
+    expect(estimateImageBlockTokens(block, 'claude')).toBe(1049);
+  });
+
+  test('openai: tokens = 85 + tiles*170 (512px tiles)', () => {
+    const block = { type: 'image_url', image_url: { url: pngDataUri(1024, 768) } };
+    // ceil(1024/512)*ceil(768/512) = 2*2 = 4 tiles -> 85 + 680 = 765
+    expect(estimateImageBlockTokens(block, 'o200k_base')).toBe(765);
+  });
+
+  test('falls back to the Anthropic minimum (1024) without base64 data', () => {
+    expect(
+      estimateImageBlockTokens(
+        { type: 'image_url', image_url: { url: 'https://example.com/a.png' } },
+        'claude'
+      )
+    ).toBe(1024);
+  });
+});
+
+describe('estimateDocumentBlockTokens', () => {
+  const countChars = (text: string): number => text.length;
+
+  test('text document is tokenized directly via getTokenCount', () => {
+    const block = { type: 'file', source_type: 'text', text: 'hello world' };
+    expect(estimateDocumentBlockTokens(block, 'o200k_base', countChars)).toBe(11);
+  });
+
+  test('base64 PDF is priced per estimated page', () => {
+    const block = {
+      type: 'file',
+      source_type: 'base64',
+      mime_type: 'application/pdf',
+      data: 'x'.repeat(150_000),
+    };
+    // ceil(150000/75000) = 2 pages
+    expect(estimateDocumentBlockTokens(block, 'claude', countChars)).toBe(4000); // 2 * 2000
+    expect(estimateDocumentBlockTokens(block, 'o200k_base', countChars)).toBe(3000); // 2 * 1500
+  });
+
+  test('url-referenced document uses the conservative fallback', () => {
+    const block = { type: 'file', source_type: 'url', url: 'https://x/y.pdf' };
+    expect(estimateDocumentBlockTokens(block, 'o200k_base', countChars)).toBe(2000);
+  });
+});
+
+describe('estimateTimedMediaBlockTokens', () => {
+  const B64 = (chars: number): string => 'A'.repeat(chars);
+
+  test('Google video (type=media, video/*): duration from size at ~300 tok/s', () => {
+    // 1,000,000 b64 -> 750,000 bytes / 250,000 Bps = 3s * 300 = 900
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'video/mp4', data: B64(1_000_000) }),
+    ).toBe(900);
+  });
+
+  test('Google audio (type=media, audio/*): 32 tok/s at ~16KB/s', () => {
+    // 320,000 b64 -> 240,000 bytes / 16,000 Bps = 15s * 32 = 480
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'audio/mp3', data: B64(320_000) }),
+    ).toBe(480);
+  });
+
+  test('OpenRouter input_audio: estimates from base64 data', () => {
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'input_audio', input_audio: { data: B64(320_000) } }),
+    ).toBe(480);
+  });
+
+  test('OpenRouter video_url with a data: URL estimates from size', () => {
+    const url = `data:video/mp4;base64,${B64(1_000_000)}`;
+    expect(estimateTimedMediaBlockTokens({ type: 'video_url', video_url: { url } })).toBe(900);
+  });
+
+  test('bare remote URL falls back to the flat ~30s estimate', () => {
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'video_url', video_url: { url: 'https://x/v.mp4' } }),
+    ).toBe(9000);
+    expect(estimateTimedMediaBlockTokens({ type: 'input_audio', input_audio: {} })).toBe(960);
+  });
+
+  test('clamps to at least one second of tokens for tiny payloads', () => {
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'audio/wav', data: 'AAAA' }),
+    ).toBe(32);
+  });
+
+  test('standard type=video / type=audio blocks (Bedrock converter shape)', () => {
+    // 750,000 bytes video / 250,000 = 3s * 300 = 900
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'video', mimeType: 'video/mp4', data: B64(1_000_000) }),
+    ).toBe(900);
+    // 240,000 bytes audio / 16,000 = 15s * 32 = 480
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'audio', mimeType: 'audio/mpeg', data: B64(320_000) }),
+    ).toBe(480);
+  });
+
+  test('reads Uint8Array data and base64 url payloads', () => {
+    // 240,000-byte Uint8Array audio -> 15s * 32 = 480
+    expect(
+      estimateTimedMediaBlockTokens({
+        type: 'audio',
+        mimeType: 'audio/wav',
+        data: new Uint8Array(240_000),
+      }),
+    ).toBe(480);
+    // base64 data url on a bare video block
+    expect(
+      estimateTimedMediaBlockTokens({
+        type: 'video',
+        mimeType: 'video/mp4',
+        url: `data:video/mp4;base64,${B64(1_000_000)}`,
+      }),
+    ).toBe(900);
+  });
+
+  test('non-video/audio media (image/document MIME) is NOT priced as video', () => {
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'image/png', fileUri: 's3://x' }),
+    ).toBe(0);
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'image/png', data: B64(400_000) }),
+    ).toBe(0);
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'application/pdf', data: B64(400) }),
+    ).toBe(0);
+  });
+
+  test('fileId / bare URL with no size falls back to the ~30s estimate', () => {
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'video', mimeType: 'video/mp4', fileId: 's3://v' }),
+    ).toBe(9000);
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'audio', mimeType: 'audio/mp3', fileId: 's3://a' }),
+    ).toBe(960);
+  });
+
+  test('reads native Bedrock nested source.bytes (video/audio)', () => {
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'video', video: { source: { bytes: new Uint8Array(750_000) } } }),
+    ).toBe(900); // 750,000 / 250,000 = 3s * 300
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'audio', audio: { source: { bytes: new Uint8Array(240_000) } } }),
+    ).toBe(480); // 240,000 / 16,000 = 15s * 32
+  });
+
+  test('reads nested video/audio data-URL and data payloads, remote stays fallback', () => {
+    // nested data URL sized by its bytes (750,000 -> 3s * 300)
+    expect(
+      estimateTimedMediaBlockTokens({
+        type: 'video',
+        video: { url: `data:video/mp4;base64,${B64(1_000_000)}` },
+      }),
+    ).toBe(900);
+    // nested base64 data (240,000 -> 15s * 32)
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'audio', audio: { data: B64(320_000) } }),
+    ).toBe(480);
+    // nested REMOTE url has no size -> ~30s fallback
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'video', video: { url: 'https://x/v.mp4' } }),
+    ).toBe(9000);
+  });
+
+  test('non-data URI schemes (gs://, s3://) are treated as remote, not base64', () => {
+    // gs:// audio must hit the ~30s remote fallback, not clamp to 32
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'audio', mimeType: 'audio/mp3', url: 'gs://bucket/a.mp3' }),
+    ).toBe(960);
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'media', mimeType: 'video/mp4', data: 's3://bucket/v.mp4' }),
+    ).toBe(9000);
+  });
+
+  test('classifies Google MIME-as-type blocks (type is the mime string)', () => {
+    // { type: 'audio/wav', data } -> 240,000 bytes / 16,000 = 15s * 32 = 480
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'audio/wav', data: B64(320_000) }),
+    ).toBe(480);
+    // { type: 'video/mp4', data } -> 750,000 / 250,000 = 3s * 300 = 900
+    expect(
+      estimateTimedMediaBlockTokens({ type: 'video/mp4', data: B64(1_000_000) }),
+    ).toBe(900);
+  });
+
+  test('returns 0 for non-timed-media blocks', () => {
+    expect(estimateTimedMediaBlockTokens({ type: 'text' })).toBe(0);
+    expect(estimateTimedMediaBlockTokens({ type: 'image/png', data: B64(400) })).toBe(0);
+  });
+});

package/src/stream.ts

@@ -108,8 +108,27 @@ function getNonEmptyValue(possibleValues: string[]): string | undefined {
   return undefined;
 }
 
-function isBatchSensitiveToolExecution(graph: StandardGraph): boolean {
-  return graph.hookRegistry != null || graph.humanInTheLoop?.enabled === true;
+function isBatchSensitiveToolExecution(
+  graph: StandardGraph,
+  metadata?: Record<string, unknown>
+): boolean {
+  /**
+   * Resolve the hook-session id exactly the way ToolNode will: its hook
+   * lookups read `config.configurable.run_id` ONLY, so that source wins here
+   * too (a subagent child graph carries its own `graph.runId`, but its
+   * ToolNode executes hooks under the PARENT's inherited configurable run
+   * id). The metadata / graph.runId fallbacks only apply when no
+   * configurable id exists and can only be conservative — a false positive
+   * merely skips eager prestart.
+   */
+  const runId =
+    (graph.config?.configurable?.run_id as string | undefined) ??
+    (metadata?.run_id as string | undefined) ??
+    graph.runId;
+  return (
+    graph.hookRegistry?.hasResultAlteringHooks(runId) === true ||
+    graph.humanInTheLoop?.enabled === true
+  );
 }
 
 function hasToolOutputReference(value: unknown): boolean {
@@ -246,7 +265,7 @@ function isEagerToolExecutionEnabledForBatch(args: {
   if ((agentContext?.toolDefinitions?.length ?? 0) === 0) {
     return false;
   }
-  if (isBatchSensitiveToolExecution(graph)) {
+  if (isBatchSensitiveToolExecution(graph, metadata)) {
     return false;
   }
   if (