@librechat/agents 3.2.34 → 3.2.35

package/src/llm/invoke.test.ts

@@ -12,8 +12,8 @@ import type { BaseMessage } from '@langchain/core/messages';
 import type * as t from '@/types';
 import { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
 import { attemptInvoke, tryFallbackProviders } from '@/llm/invoke';
+import { Constants, Providers } from '@/common';
 import { ToolNode } from '@/tools/ToolNode';
-import { Providers } from '@/common';
 
 /**
  * Minimal stub model shape `attemptInvoke` reads. Either `invoke` or
@@ -341,6 +341,84 @@ describe('tryFallbackProviders applies the same lazy annotation transform', () =
   });
 });
 
+describe('invocation attribution metadata', () => {
+  it('stamps INVOKED_PROVIDER on the config passed to the model', async () => {
+    const capturedConfigs: unknown[] = [];
+    const model: StubModel = {
+      invoke: jest.fn(
+        async (_m: BaseMessage[], config?: unknown): Promise<AIMessage> => {
+          capturedConfigs.push(config);
+          return new AIMessage({ content: 'ok' });
+        }
+      ),
+    };
+
+    await attemptInvoke(
+      {
+        model: model as t.ChatModel,
+        messages: [new HumanMessage('hi')],
+        /** A ChatOpenAI-derived provider — `ls_provider` would lie here. */
+        provider: Providers.DEEPSEEK,
+      },
+      { configurable: { run_id: 'run-attr' }, metadata: { existing: true } }
+    );
+
+    const config = capturedConfigs[0] as {
+      metadata?: Record<string, unknown>;
+    };
+    expect(config.metadata?.[Constants.INVOKED_PROVIDER]).toBe(
+      Providers.DEEPSEEK
+    );
+    /** Pre-existing metadata is preserved, not replaced. */
+    expect(config.metadata?.existing).toBe(true);
+  });
+
+  it('stamps INVOKED_MODEL from the fallback clientOptions in tryFallbackProviders', async () => {
+    const capturedConfigs: unknown[] = [];
+    const model: StubModel = {
+      invoke: jest.fn(
+        async (_m: BaseMessage[], config?: unknown): Promise<AIMessage> => {
+          capturedConfigs.push(config);
+          return new AIMessage({ content: 'ok' });
+        }
+      ),
+    };
+
+    jest.doMock('@/llm/init', () => ({
+      initializeModel: (): unknown => model,
+    }));
+    jest.resetModules();
+    const { tryFallbackProviders: freshTry } = (await import(
+      '@/llm/invoke'
+    )) as { tryFallbackProviders: typeof tryFallbackProviders };
+
+    await freshTry({
+      fallbacks: [
+        {
+          provider: Providers.ANTHROPIC,
+          clientOptions: { model: 'claude-fallback-1' },
+        },
+      ],
+      messages: [new HumanMessage('hi')],
+      primaryError: new Error('primary failed'),
+      config: { configurable: { run_id: 'run-attr-fb' } },
+    });
+
+    const config = capturedConfigs[0] as {
+      metadata?: Record<string, unknown>;
+    };
+    expect(config.metadata?.[Constants.INVOKED_MODEL]).toBe(
+      'claude-fallback-1'
+    );
+    expect(config.metadata?.[Constants.INVOKED_PROVIDER]).toBe(
+      Providers.ANTHROPIC
+    );
+
+    jest.dontMock('@/llm/init');
+    jest.resetModules();
+  });
+});
+
 describe('cross-run hydration through ToolNode + attemptInvoke', () => {
   it('annotates run 2 refs but leaves hydrated run 1 ToolMessages untouched', async () => {
     /**

package/src/llm/invoke.ts

@@ -6,10 +6,10 @@ import type { BaseMessage } from '@langchain/core/messages';
 import type { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
 import type * as t from '@/types';
 import { annotateMessagesForLLM } from '@/tools/toolOutputReferences';
+import { Constants, GraphEvents, Providers } from '@/common';
 import { manualToolStreamProviders } from '@/llm/providers';
 import { modifyDeltaProperties } from '@/messages';
 import { ChatModelStreamHandler } from '@/stream';
-import { GraphEvents, Providers } from '@/common';
 import { initializeModel } from '@/llm/init';
 
 /**
@@ -208,6 +208,23 @@ export async function attemptInvoke(
   const runId = config?.configurable?.run_id as string | undefined;
   const messagesForProvider = annotateMessagesForLLM(messages, registry, runId);
 
+  /**
+   * Stamp the provider that is ACTUALLY serving this invocation onto the
+   * callback metadata. `attemptInvoke` is the single funnel for primary,
+   * fallback, and summarization model calls, so consumers that need
+   * provider attribution per call (the subagent usage-capture handler)
+   * read this key instead of trusting static agent config — which is
+   * wrong for fallback-served calls — or `ls_provider` — which derived
+   * providers inherit from their base class.
+   */
+  config = {
+    ...config,
+    metadata: {
+      ...(config?.metadata ?? {}),
+      [Constants.INVOKED_PROVIDER]: provider,
+    },
+  };
+
   if (model.stream) {
     const stream = await model.stream(messagesForProvider, config);
     let finalChunk: AIMessageChunk | undefined;
@@ -224,7 +241,7 @@ export async function attemptInvoke(
         });
       }
     } else if (registeredStreamHandler == null) {
-      const metadata = config?.metadata as Record<string, unknown> | undefined;
+      const metadata = config.metadata as Record<string, unknown> | undefined;
       const streamHandler = new ChatModelStreamHandler();
       for await (const chunk of stream) {
         const handlingChunk = getStreamHandlingChunk({
@@ -247,7 +264,7 @@ export async function attemptInvoke(
         });
       }
     } else {
-      const metadata = config?.metadata as Record<string, unknown> | undefined;
+      const metadata = config.metadata as Record<string, unknown> | undefined;
       for await (const chunk of stream) {
         const handlingChunk = getStreamHandlingChunk({
           current: finalChunk,
@@ -292,6 +309,25 @@ export async function attemptInvoke(
   return { messages: [finalMessage] };
 }
 
+/**
+ * Best-effort read of the configured model name from client options.
+ * Providers disagree on the key (`model` vs `modelName`).
+ */
+function extractClientOptionsModel(
+  clientOptions: t.ClientOptions | undefined
+): string | undefined {
+  const options = clientOptions as
+    | { model?: unknown; modelName?: unknown }
+    | undefined;
+  if (typeof options?.model === 'string' && options.model !== '') {
+    return options.model;
+  }
+  if (typeof options?.modelName === 'string' && options.modelName !== '') {
+    return options.modelName;
+  }
+  return undefined;
+}
+
 /**
  * Attempts each fallback provider in order until one succeeds.
  * Throws the last error if all fallbacks fail.
@@ -321,6 +357,24 @@ export async function tryFallbackProviders({
         clientOptions: fb.clientOptions,
         tools,
       });
+      /**
+       * Stamp the fallback's configured model onto callback metadata so
+       * per-call attribution (subagent usage capture) doesn't fall back to
+       * the PRIMARY config's model when the provider reports no
+       * `ls_model_name`. The serving provider is stamped uniformly by
+       * `attemptInvoke` (`INVOKED_PROVIDER`).
+       */
+      const fbModelName = extractClientOptionsModel(fb.clientOptions);
+      const fbConfig: RunnableConfig | undefined =
+        fbModelName == null
+          ? config
+          : {
+            ...config,
+            metadata: {
+              ...(config?.metadata ?? {}),
+              [Constants.INVOKED_MODEL]: fbModelName,
+            },
+          };
       const result = await attemptInvoke(
         {
           model: fbModel as t.ChatModel,
@@ -329,7 +383,7 @@ export async function tryFallbackProviders({
           context,
           onChunk,
         },
-        config
+        fbConfig
       );
       return result;
     } catch (e) {

package/src/messages/content.ts

@@ -1,6 +1,26 @@
-import type { BaseMessage } from '@langchain/core/messages';
+import type {
+  BaseMessage,
+  MessageContentComplex,
+} from '@langchain/core/messages';
 import { ContentTypes } from '@/common';
 
+/**
+ * Whether {@link formatContentStrings} will flatten this message's content:
+ * a human/ai/system message whose content is an array of text-only blocks.
+ */
+export const isLegacyConvertible = (message: BaseMessage): boolean => {
+  const messageType = message.getType();
+  const isValidMessage =
+    messageType === 'human' || messageType === 'ai' || messageType === 'system';
+  if (!isValidMessage) {
+    return false;
+  }
+  if (!Array.isArray(message.content)) {
+    return false;
+  }
+  return message.content.every((block) => block.type === ContentTypes.TEXT);
+};
+
 /**
  * Formats an array of messages for LangChain, making sure all content fields are strings
  * @param {Array<HumanMessage | AIMessage | SystemMessage | ToolMessage>} payload - The array of messages to format.
@@ -13,42 +33,14 @@ export const formatContentStrings = (
   const result: Array<BaseMessage> = [];
 
   for (const message of payload) {
-    const messageType = message.getType();
-    const isValidMessage =
-      messageType === 'human' ||
-      messageType === 'ai' ||
-      messageType === 'system';
-
-    if (!isValidMessage) {
-      result.push(message);
-      continue;
-    }
-
-    // If content is already a string, add as-is
-    if (typeof message.content === 'string') {
-      result.push(message);
-      continue;
-    }
-
-    // If content is not an array, add as-is
-    if (!Array.isArray(message.content)) {
-      result.push(message);
-      continue;
-    }
-
-    // Check if all content blocks are text type
-    const allTextBlocks = message.content.every(
-      (block) => block.type === ContentTypes.TEXT
-    );
-
-    // Only convert to string if all blocks are text type
-    if (!allTextBlocks) {
+    if (!isLegacyConvertible(message)) {
       result.push(message);
       continue;
     }
 
     // Reduce text types to a single string
-    const content = message.content.reduce((acc, curr) => {
+    const blocks = message.content as MessageContentComplex[];
+    const content = blocks.reduce((acc, curr) => {
       if (curr.type === ContentTypes.TEXT) {
         return `${acc}${curr[ContentTypes.TEXT] || ''}\n`;
       }

package/src/messages/prune.ts

@@ -1312,16 +1312,36 @@ export function createPruneMessages(factoryParams: PruneMessagesFactoryParams) {
     originalToolContent?: Map<number, string>;
     calibrationRatio?: number;
     resolvedInstructionOverhead?: number;
+    /** Usable budget this call: maxTokens minus output reserve */
+    contextBudget?: number;
+    /** Calibrated instruction overhead actually applied this call */
+    effectiveInstructionTokens?: number;
   } {
     if (params.messages.length === 0) {
+      /** Post-compaction calls still invoke the model — report the same
+       *  reserve-adjusted budget fields as the populated paths */
+      const emptyInstructionTokens =
+        factoryParams.getInstructionTokens?.() ?? 0;
+      const emptyReserveRatio =
+        factoryParams.reserveRatio ?? DEFAULT_RESERVE_RATIO;
+      const emptyBudget =
+        factoryParams.maxTokens -
+        (emptyReserveRatio > 0 && emptyReserveRatio < 1
+          ? Math.round(factoryParams.maxTokens * emptyReserveRatio)
+          : 0);
       return {
         context: [],
         indexTokenCountMap,
         messagesToRefine: [],
         prePruneContextTokens: 0,
-        remainingContextTokens: factoryParams.maxTokens,
+        remainingContextTokens: Math.max(
+          0,
+          emptyBudget - emptyInstructionTokens
+        ),
         calibrationRatio,
         resolvedInstructionOverhead: bestInstructionOverhead,
+        contextBudget: emptyBudget,
+        effectiveInstructionTokens: emptyInstructionTokens,
       };
     }
 
@@ -1549,6 +1569,8 @@ export function createPruneMessages(factoryParams: PruneMessagesFactoryParams) {
           pruningBudget > 0 ? calibratedTotalTokens / pruningBudget : 0,
         calibrationRatio,
         resolvedInstructionOverhead: bestInstructionOverhead,
+        contextBudget: pruningBudget,
+        effectiveInstructionTokens: currentInstructionTokens,
       };
     }
 
@@ -1752,6 +1774,8 @@ export function createPruneMessages(factoryParams: PruneMessagesFactoryParams) {
           originalToolContent.size > 0 ? originalToolContent : undefined,
         calibrationRatio,
         resolvedInstructionOverhead: bestInstructionOverhead,
+        contextBudget: pruningBudget,
+        effectiveInstructionTokens: currentInstructionTokens,
       };
     }
 
@@ -2099,9 +2123,20 @@ export function createPruneMessages(factoryParams: PruneMessagesFactoryParams) {
       }
     }
 
+    /** Scale raw-space remaining back to calibrated/provider units so it is
+     *  directly comparable with pruningBudget and prePruneContextTokens */
+    const rawRemaining = Math.max(
+      0,
+      initialRemainingContextTokens + reclaimedTokens
+    );
     const remainingContextTokens = Math.max(
       0,
-      Math.min(pruningBudget, initialRemainingContextTokens + reclaimedTokens)
+      Math.min(
+        pruningBudget,
+        calibrationRatio > 0
+          ? Math.round(rawRemaining * calibrationRatio)
+          : rawRemaining
+      )
     );
 
     runThinkingStartIndex = thinkingStartIndex ?? -1;
@@ -2123,6 +2158,8 @@ export function createPruneMessages(factoryParams: PruneMessagesFactoryParams) {
         originalToolContent.size > 0 ? originalToolContent : undefined,
       calibrationRatio,
       resolvedInstructionOverhead: bestInstructionOverhead,
+      contextBudget: pruningBudget,
+      effectiveInstructionTokens: currentInstructionTokens,
     };
   };
 }

package/src/run.ts

@@ -78,6 +78,7 @@ const CUSTOM_GRAPH_EVENTS = new Set<string>([
   GraphEvents.ON_SUMMARIZE_COMPLETE,
   GraphEvents.ON_SUBAGENT_UPDATE,
   GraphEvents.ON_AGENT_LOG,
+  GraphEvents.ON_CONTEXT_USAGE,
   GraphEvents.ON_CUSTOM_EVENT,
 ]);
 
@@ -129,6 +130,7 @@ export class Run<_T extends t.BaseGraphState> {
   private toolOutputReferences?: t.ToolOutputReferencesConfig;
   private eagerEventToolExecution?: t.EagerEventToolExecutionConfig;
   private toolExecution?: t.ToolExecutionConfig;
+  private subagentUsageSink?: t.SubagentUsageSink;
   private indexTokenCountMap?: Record<string, number>;
   calibrationRatio: number = 1;
   graphRunnable?: t.CompiledStateWorkflow;
@@ -176,6 +178,7 @@ export class Run<_T extends t.BaseGraphState> {
     this.toolOutputReferences = config.toolOutputReferences;
     this.eagerEventToolExecution = config.eagerEventToolExecution;
     this.toolExecution = config.toolExecution;
+    this.subagentUsageSink = config.subagentUsageSink;
 
     if (!config.graphConfig) {
       throw new Error('Graph config not provided');
@@ -249,6 +252,7 @@ export class Run<_T extends t.BaseGraphState> {
       tokenCounter: this.tokenCounter,
       indexTokenCountMap: this.indexTokenCountMap,
       calibrationRatio: this.calibrationRatio,
+      subagentUsageSink: this.subagentUsageSink,
     });
     /** Propagate compile options from graph config */
     standardGraph.compileOptions = this.applyHITLCheckpointerFallback(
@@ -276,6 +280,7 @@ export class Run<_T extends t.BaseGraphState> {
       tokenCounter: this.tokenCounter,
       indexTokenCountMap: this.indexTokenCountMap,
       calibrationRatio: this.calibrationRatio,
+      subagentUsageSink: this.subagentUsageSink,
     });
 
     multiAgentGraph.compileOptions =

package/src/scripts/subagent-usage-sink.ts

@@ -0,0 +1,176 @@
+import { config } from 'dotenv';
+config();
+
+import { HumanMessage } from '@langchain/core/messages';
+import type { UsageMetadata } from '@langchain/core/messages';
+import type * as t from '@/types';
+import { ToolEndHandler, ModelEndHandler } from '@/events';
+import { Providers, GraphEvents } from '@/common';
+import { Run } from '@/run';
+
+/**
+ * Live verification for `subagentUsageSink` (host billing of subagent
+ * child-run model usage).
+ *
+ * Runs a supervisor that MUST delegate to a "researcher" subagent, then
+ * asserts:
+ * 1. The host's CHAT_MODEL_END handler collected the PARENT's calls only.
+ * 2. The sink received one event per CHILD model call, tagged with the
+ *    subagent type, child run id, and the child's model/provider.
+ * 3. Child usage has real token counts (the previously-unbilled tokens).
+ *
+ * Usage:
+ *   OPENAI_API_KEY=... npx ts-node -r tsconfig-paths/register src/scripts/subagent-usage-sink.ts
+ *
+ * Or with Anthropic:
+ *   ANTHROPIC_API_KEY=... npx ts-node -r tsconfig-paths/register src/scripts/subagent-usage-sink.ts --provider anthropic
+ */
+
+const useAnthropic =
+  process.argv.includes('--provider') &&
+  process.argv[process.argv.indexOf('--provider') + 1] === 'anthropic';
+
+const provider = useAnthropic ? Providers.ANTHROPIC : Providers.OPENAI;
+const apiKey = useAnthropic
+  ? process.env.ANTHROPIC_API_KEY
+  : process.env.OPENAI_API_KEY;
+const modelName = useAnthropic ? 'claude-sonnet-4-20250514' : 'gpt-4o-mini';
+
+if (!apiKey) {
+  console.error(
+    `Missing ${useAnthropic ? 'ANTHROPIC_API_KEY' : 'OPENAI_API_KEY'} environment variable`
+  );
+  process.exit(1);
+}
+
+async function main(): Promise<void> {
+  console.log('=== Subagent Usage Sink Live Verification ===\n');
+  console.log(`Provider: ${provider}`);
+  console.log(`Model: ${modelName}\n`);
+
+  const parentAgent: t.AgentInputs = {
+    agentId: 'supervisor',
+    provider,
+    clientOptions: { modelName, apiKey },
+    instructions: `You are a supervisor agent. For ANY user question, you MUST delegate to the "researcher" subagent via the subagent tool — never answer directly. After the subagent returns, give the user a one-sentence final answer.`,
+    maxContextTokens: 16000,
+    subagentConfigs: [
+      {
+        type: 'researcher',
+        name: 'Research Specialist',
+        description: 'Researches questions and returns concise answers.',
+        agentInputs: {
+          agentId: 'researcher',
+          provider,
+          clientOptions: { modelName, apiKey },
+          instructions:
+            'You are a research specialist. Answer the task in one or two sentences.',
+          maxContextTokens: 8000,
+        },
+      },
+    ],
+  };
+
+  const collectedUsage: UsageMetadata[] = [];
+  const sunkEvents: t.SubagentUsageEvent[] = [];
+
+  const runId = `usage-sink-live-${Date.now()}`;
+  const run = await Run.create<t.IState>({
+    runId,
+    graphConfig: {
+      type: 'standard',
+      agents: [parentAgent],
+    },
+    returnContent: true,
+    customHandlers: {
+      [GraphEvents.TOOL_END]: new ToolEndHandler(),
+      [GraphEvents.CHAT_MODEL_END]: new ModelEndHandler(collectedUsage),
+    },
+    subagentUsageSink: (event) => {
+      sunkEvents.push(event);
+    },
+  });
+
+  const callerConfig = {
+    configurable: { thread_id: `usage-sink-${Date.now()}` },
+    streamMode: 'values' as const,
+    version: 'v2' as const,
+  };
+
+  await run.processStream(
+    {
+      messages: [
+        new HumanMessage(
+          'In what year was the Eiffel Tower completed? Use the researcher subagent.'
+        ),
+      ],
+    },
+    callerConfig
+  );
+
+  console.log('\n--- Parent collectedUsage (CHAT_MODEL_END handler) ---');
+  console.dir(collectedUsage, { depth: null });
+
+  console.log('\n--- Subagent usage sink events ---');
+  console.dir(sunkEvents, { depth: null });
+
+  const failures: string[] = [];
+
+  if (collectedUsage.length < 2) {
+    failures.push(
+      `expected >= 2 parent model calls in collectedUsage, got ${collectedUsage.length}`
+    );
+  }
+  if (sunkEvents.length === 0) {
+    failures.push('sink received NO child usage events');
+  }
+  for (const event of sunkEvents) {
+    if (event.subagentType !== 'researcher') {
+      failures.push(`unexpected subagentType: ${event.subagentType}`);
+    }
+    if (event.runId !== runId) {
+      failures.push(`event.runId mismatch: ${event.runId}`);
+    }
+    if (!event.subagentRunId.startsWith(`${runId}_sub_`)) {
+      failures.push(`event.subagentRunId mismatch: ${event.subagentRunId}`);
+    }
+    if (event.provider !== provider) {
+      failures.push(`event.provider mismatch: ${event.provider}`);
+    }
+    if (event.model == null || event.model === '') {
+      failures.push('event.model missing');
+    }
+    const input = Number(event.usage.input_tokens) || 0;
+    const output = Number(event.usage.output_tokens) || 0;
+    if (input <= 0 || output <= 0) {
+      failures.push(
+        `child usage has non-positive tokens: input=${input} output=${output}`
+      );
+    }
+  }
+
+  const childTotal = sunkEvents.reduce(
+    (sum, e) =>
+      sum +
+      (Number(e.usage.input_tokens) || 0) +
+      (Number(e.usage.output_tokens) || 0),
+    0
+  );
+  console.log(
+    `\nChild tokens that were previously invisible to billing: ${childTotal}`
+  );
+
+  if (failures.length > 0) {
+    console.error('\nFAIL:');
+    for (const failure of failures) {
+      console.error(`  - ${failure}`);
+    }
+    process.exit(1);
+  }
+  console.log('\nPASS: subagent child usage reported through the sink.');
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});