@librechat/agents 3.1.96 → 3.1.98

package/dist/types/types/graph.d.ts

@@ -195,6 +195,7 @@ export type StandardGraphInput = {
     runId?: string;
     signal?: AbortSignal;
     agents: AgentInputs[];
+    langfuse?: LangfuseConfig;
     tokenCounter?: TokenCounter;
     indexTokenCountMap?: Record<string, number>;
     calibrationRatio?: number;
@@ -288,11 +289,40 @@ export interface SubagentUpdateEvent {
     /** ISO timestamp for ordering / display. */
     timestamp: string;
 }
+export type LangfuseToolOutputTracingConfig = {
+    /**
+     * Whether tool outputs should be exported to Langfuse. Defaults to
+     * `true`. Set to `false` to keep tool spans and redact their output.
+     */
+    enabled?: boolean;
+    /**
+     * Optional allowlist of tool names whose outputs should be redacted even
+     * when `enabled` is true.
+     */
+    redactedToolNames?: string[];
+    /**
+     * Match strategy for `redactedToolNames`. Defaults to `exact`; use
+     * `partial` to redact tools whose names contain a configured value.
+     */
+    redactedToolNameMatchMode?: 'exact' | 'partial';
+    /** Replacement text used for redacted tool outputs. */
+    redactionText?: string;
+};
+export type LangfuseToolNodeTracingConfig = {
+    /**
+     * Overrides ToolNode callback tracing. ToolNode spans are exported by the
+     * env-backed Langfuse callback, so this only enables tracing when that
+     * callback is configured.
+     */
+    enabled?: boolean;
+};
 export interface LangfuseConfig {
     enabled?: boolean;
     publicKey?: string;
     secretKey?: string;
     baseUrl?: string;
+    toolNodeTracing?: LangfuseToolNodeTracingConfig;
+    toolOutputTracing?: LangfuseToolOutputTracingConfig;
 }
 export interface AgentInputs {
     agentId: string;

package/dist/types/types/llm.d.ts

@@ -65,7 +65,7 @@ export type BedrockAnthropicInput = ChatBedrockConverseInput & {
     additionalModelRequestFields?: ChatBedrockConverseInput['additionalModelRequestFields'] & AnthropicReasoning;
     promptCache?: boolean;
 };
-export type BedrockConverseClientOptions = ChatBedrockConverseInput;
+export type BedrockConverseClientOptions = BedrockAnthropicInput;
 export type BedrockAnthropicClientOptions = BedrockAnthropicInput;
 export type GoogleClientOptions = GoogleGenerativeAIChatInput & {
     customHeaders?: RequestOptions['customHeaders'];
@@ -95,7 +95,7 @@ export type ProviderOptionsMap = {
     [Providers.MISTRALAI]: MistralAIClientOptions;
     [Providers.MISTRAL]: MistralAIClientOptions;
     [Providers.OPENROUTER]: ChatOpenRouterCallOptions;
-    [Providers.BEDROCK]: BedrockConverseClientOptions;
+    [Providers.BEDROCK]: BedrockAnthropicClientOptions;
     [Providers.XAI]: XAIClientOptions;
     [Providers.MOONSHOT]: OpenAIClientOptions;
 };

package/dist/types/types/run.d.ts

@@ -104,6 +104,12 @@ export type StandardGraphConfig = Omit<MultiAgentGraphConfig, 'edges' | 'type'>
 export type RunConfig = {
     runId: string;
     graphConfig: LegacyGraphConfig | StandardGraphConfig | MultiAgentGraphConfig;
+    /**
+     * Run-scoped Langfuse configuration. Per-agent `AgentInputs.langfuse`
+     * takes precedence for agent-specific spans; this object supplies defaults
+     * for run-wide tracing controls such as tool-output redaction.
+     */
+    langfuse?: g.LangfuseConfig;
     customHandlers?: Record<string, g.EventHandler>;
     /**
      * Pre-constructed hook registry for this run. Hooks fire at lifecycle

package/dist/types/types/tools.d.ts

@@ -5,6 +5,7 @@ import type { HookRegistry } from '@/hooks';
 import type { ToolOutputReferenceRegistry } from '@/tools/toolOutputReferences';
 import type { MessageContentComplex, ToolErrorData } from './stream';
 import type { HumanInTheLoopConfig } from './hitl';
+import type { LangfuseConfig } from './graph';
 /** Replacement type for `import type { ToolCall } from '@langchain/core/messages/tool'` in order to have stringified args typed */
 export type CustomToolCall = {
     name: string;
@@ -62,6 +63,12 @@ export type EagerEventToolCallChunkState = {
 export type ToolNodeOptions = {
     name?: string;
     tags?: string[];
+    /** Enables LangChain/LangGraph tracing for this ToolNode. Defaults to false. */
+    trace?: boolean;
+    /** Run-level Langfuse config used to scope ToolNode trace redaction. */
+    runLangfuse?: LangfuseConfig;
+    /** Agent-level Langfuse config used to scope ToolNode trace redaction. */
+    agentLangfuse?: LangfuseConfig;
     handleToolErrors?: boolean;
     loadRuntimeTools?: ToolRefGenerator;
     toolCallStepIds?: Map<string, string>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@librechat/agents",
-  "version": "3.1.96",
+  "version": "3.1.98",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",
@@ -230,6 +230,7 @@
     "@langfuse/langchain": "^5.3.0",
     "@langfuse/otel": "^5.3.0",
     "@langfuse/tracing": "^5.3.0",
+    "@opentelemetry/context-async-hooks": "2.7.1",
     "@opentelemetry/sdk-node": "^0.218.0",
     "@scarf/scarf": "^1.4.0",
     "@types/diff": "^7.0.2",

package/src/agents/__tests__/AgentContext.anthropic.live.test.ts

@@ -9,6 +9,14 @@ import { config as dotenvConfig } from 'dotenv';
 dotenvConfig();
 
 import { describe, expect, it } from '@jest/globals';
+import {
+  AIMessage,
+  BaseMessage,
+  HumanMessage,
+  SystemMessage,
+  ToolMessage,
+  type MessageContentComplex,
+} from '@langchain/core/messages';
 import type * as t from '@/types';
 import {
   runLiveTurn,
@@ -18,6 +26,9 @@ import {
   waitForCachePropagation,
 } from './promptCacheLiveHelpers';
 import { Providers } from '@/common';
+import { addCacheControl } from '@/messages/cache';
+import { toLangChainContent } from '@/messages/langchain';
+import { _convertMessagesToAnthropicPayload } from '@/llm/anthropic/utils/message_inputs';
 
 const shouldRunLive =
   process.env.RUN_ANTHROPIC_PROMPT_CACHE_LIVE_TESTS === '1' &&
@@ -46,6 +57,268 @@ function createClientOptions(): t.AnthropicClientOptions {
   };
 }
 
+type AnthropicCacheUsage = {
+  inputTokens: number;
+  outputTokens: number;
+  cacheCreation: number;
+  cacheRead: number;
+  latencyMs: number;
+};
+
+type AnthropicUsageResponse = {
+  usage?: {
+    input_tokens?: number;
+    output_tokens?: number;
+    cache_creation_input_tokens?: number | null;
+    cache_read_input_tokens?: number | null;
+  };
+};
+
+type AnthropicMessagesClient = {
+  messages: {
+    create: (
+      request: Record<string, unknown>,
+      options: { headers: Record<string, string> }
+    ) => Promise<AnthropicUsageResponse>;
+  };
+};
+
+const benchmarkTool = {
+  name: 'lookup_cache_probe',
+  description: 'Returns prompt cache benchmark data.',
+  input_schema: {
+    type: 'object',
+    properties: {
+      step: { type: 'integer' },
+    },
+    required: ['step'],
+  },
+};
+
+function cloneLiveMessage(
+  message: BaseMessage,
+  content: MessageContentComplex[]
+): BaseMessage {
+  const baseParams = {
+    content: toLangChainContent(content),
+    additional_kwargs: { ...message.additional_kwargs },
+    response_metadata: { ...message.response_metadata },
+    id: message.id,
+    name: message.name,
+  };
+
+  const messageType = message.getType();
+  if (messageType === 'ai') {
+    return new AIMessage({
+      ...baseParams,
+      tool_calls: (message as AIMessage).tool_calls,
+    });
+  }
+  if (messageType === 'human') {
+    return new HumanMessage(baseParams);
+  }
+  if (messageType === 'system') {
+    return new SystemMessage(baseParams);
+  }
+  if (messageType === 'tool') {
+    return new ToolMessage({
+      ...baseParams,
+      tool_call_id: (message as ToolMessage).tool_call_id,
+    });
+  }
+
+  return message;
+}
+
+function addLatestUserOnlyAnthropicCacheControl(
+  messages: BaseMessage[]
+): BaseMessage[] {
+  const updatedMessages = [...messages];
+  let addedCacheControl = false;
+
+  for (let i = updatedMessages.length - 1; i >= 0; i--) {
+    const message = updatedMessages[i];
+    const content = message.content;
+    const hasArrayContent = Array.isArray(content);
+    const canAddCache =
+      !addedCacheControl &&
+      message.getType() === 'human' &&
+      (typeof content === 'string' || hasArrayContent);
+
+    if (!canAddCache && !hasArrayContent) {
+      continue;
+    }
+
+    let workingContent: MessageContentComplex[];
+    let modified = false;
+
+    if (hasArrayContent) {
+      workingContent = [];
+      let lastTextIndex = -1;
+      for (const block of content as MessageContentComplex[]) {
+        if ('cachePoint' in block && !('type' in block)) {
+          modified = true;
+          continue;
+        }
+        const cloned = { ...block };
+        if ('cache_control' in cloned) {
+          delete (cloned as Record<string, unknown>).cache_control;
+          modified = true;
+        }
+        if ('type' in cloned && cloned.type === 'text') {
+          const text = (cloned as { text?: string }).text;
+          if (text != null && text.trim() !== '') {
+            lastTextIndex = workingContent.length;
+          }
+        }
+        workingContent.push(cloned as MessageContentComplex);
+      }
+
+      if (canAddCache && lastTextIndex >= 0) {
+        (
+          workingContent[lastTextIndex] as MessageContentComplex & {
+            cache_control?: { type: 'ephemeral' };
+          }
+        ).cache_control = { type: 'ephemeral' };
+        addedCacheControl = true;
+        modified = true;
+      }
+
+      if (!modified) {
+        continue;
+      }
+    } else if (typeof content === 'string' && content.trim() !== '' && canAddCache) {
+      workingContent = [
+        {
+          type: 'text',
+          text: content,
+          cache_control: { type: 'ephemeral' },
+        },
+      ] as unknown as MessageContentComplex[];
+      addedCacheControl = true;
+    } else {
+      continue;
+    }
+
+    updatedMessages[i] = cloneLiveMessage(message, workingContent);
+  }
+
+  return updatedMessages;
+}
+
+function repeated(label: string, count: number): string {
+  return Array.from(
+    { length: count },
+    (_, index) =>
+      `${label} reference ${index}: stable schema, metric definition, access policy, dashboard note, and query planning guidance.`
+  ).join('\n');
+}
+
+function buildMultiTurnToolMessages({
+  nonce,
+  marker,
+}: {
+  nonce: string;
+  marker: string;
+}): BaseMessage[] {
+  const stableFirstUser = [
+    `Anthropic prompt cache placement benchmark ${nonce}.`,
+    'This first user turn is intentionally stable across calls in the same benchmark case.',
+    repeated(`${nonce} stable-user-context`, 190),
+  ].join('\n');
+  const latestUser = [
+    `Current user request marker: ${marker}.`,
+    'Use the final tool result to answer with the marker only.',
+    repeated(`${nonce} latest-user-${marker}`, 18),
+  ].join('\n');
+  const volatileToolPayload = repeated(`${nonce} volatile-tool-${marker}`, 70);
+
+  return [
+    new HumanMessage(stableFirstUser),
+    new AIMessage('I will keep this stable context in mind.'),
+    new HumanMessage(latestUser),
+    new AIMessage({
+      content: `I will inspect cache probe step 1 for ${marker}.\n${volatileToolPayload}`,
+      tool_calls: [
+        {
+          id: `call_${marker}_1`,
+          name: 'lookup_cache_probe',
+          args: { step: 1 },
+        },
+      ],
+    }),
+    new ToolMessage({
+      content: `Tool result 1 for ${marker}.\n${volatileToolPayload}`,
+      tool_call_id: `call_${marker}_1`,
+    }),
+    new AIMessage({
+      content: `I will inspect cache probe step 2 for ${marker}.\n${volatileToolPayload}`,
+      tool_calls: [
+        {
+          id: `call_${marker}_2`,
+          name: 'lookup_cache_probe',
+          args: { step: 2 },
+        },
+      ],
+    }),
+    new ToolMessage({
+      content: [
+        `Final tool result marker: ${marker}.`,
+        'Reply with the marker and no extra explanation.',
+        volatileToolPayload,
+      ].join('\n'),
+      tool_call_id: `call_${marker}_2`,
+    }),
+  ];
+}
+
+function extractCacheUsage(
+  response: AnthropicUsageResponse,
+  latencyMs: number
+): AnthropicCacheUsage {
+  if (response.usage == null) {
+    throw new Error('Missing Anthropic usage metadata for cache benchmark');
+  }
+
+  return {
+    inputTokens: response.usage.input_tokens ?? 0,
+    outputTokens: response.usage.output_tokens ?? 0,
+    cacheCreation: response.usage.cache_creation_input_tokens ?? 0,
+    cacheRead: response.usage.cache_read_input_tokens ?? 0,
+    latencyMs,
+  };
+}
+
+async function runAnthropicCacheBenchmarkTurn({
+  client,
+  messages,
+}: {
+  client: AnthropicMessagesClient;
+  messages: BaseMessage[];
+}): Promise<AnthropicCacheUsage> {
+  const payload = _convertMessagesToAnthropicPayload(messages);
+  const startedAt = Date.now();
+  const response = await client.messages.create(
+    {
+      ...payload,
+      model: modelName,
+      max_tokens: 16,
+      temperature: 0,
+      tools: [benchmarkTool],
+    },
+    {
+      headers: {
+        'anthropic-beta': 'prompt-caching-2024-07-31',
+      },
+    }
+  );
+
+  return extractCacheUsage(
+    response as AnthropicUsageResponse,
+    Date.now() - startedAt
+  );
+}
+
 describeIfLive('AgentContext Anthropic prompt cache live API', () => {
   it('caches only the stable system prefix while dynamic tail changes', async () => {
     const nonce = `agent-cache-live-${Date.now()}`;
@@ -109,4 +382,63 @@ describeIfLive('AgentContext Anthropic prompt cache live API', () => {
     expect(second.text.toLowerCase()).toContain('bravo');
     expect(second.usage.input_token_details?.cache_read).toBeGreaterThan(0);
   }, 120_000);
+
+  it('compares current two-user cache placement against latest-user-only', async () => {
+    const Anthropic = (await import('@anthropic-ai/sdk')).default;
+    const client = new Anthropic({
+      apiKey: process.env.ANTHROPIC_API_KEY,
+    }) as unknown as AnthropicMessagesClient;
+    const nonce = `anthropic-cache-placement-${Date.now()}`;
+    const currentNonce = `${nonce}-current`;
+    const latestOnlyNonce = `${nonce}-latest-only`;
+
+    const currentFirst = await runAnthropicCacheBenchmarkTurn({
+      client,
+      messages: addCacheControl(
+        buildMultiTurnToolMessages({ nonce: currentNonce, marker: 'alpha' })
+      ),
+    });
+
+    await waitForCachePropagation();
+
+    const currentSecond = await runAnthropicCacheBenchmarkTurn({
+      client,
+      messages: addCacheControl(
+        buildMultiTurnToolMessages({ nonce: currentNonce, marker: 'bravo' })
+      ),
+    });
+
+    const latestOnlyFirst = await runAnthropicCacheBenchmarkTurn({
+      client,
+      messages: addLatestUserOnlyAnthropicCacheControl(
+        buildMultiTurnToolMessages({ nonce: latestOnlyNonce, marker: 'alpha' })
+      ),
+    });
+
+    await waitForCachePropagation();
+
+    const latestOnlySecond = await runAnthropicCacheBenchmarkTurn({
+      client,
+      messages: addLatestUserOnlyAnthropicCacheControl(
+        buildMultiTurnToolMessages({ nonce: latestOnlyNonce, marker: 'bravo' })
+      ),
+    });
+
+    process.stdout.write(
+      `Anthropic cache placement benchmark ${JSON.stringify({
+        currentFirst,
+        currentSecond,
+        latestOnlyFirst,
+        latestOnlySecond,
+        cacheWriteDelta:
+          currentSecond.cacheCreation - latestOnlySecond.cacheCreation,
+      })}\n`
+    );
+
+    expect(currentSecond.cacheRead).toBeGreaterThan(0);
+    expect(currentSecond.cacheRead).toBeGreaterThan(latestOnlySecond.cacheRead);
+    expect(currentSecond.cacheCreation).toBeLessThan(
+      latestOnlySecond.cacheCreation
+    );
+  }, 180_000);
 });