@librechat/agents 3.1.96 → 3.1.98

package/src/graphs/Graph.ts

@@ -52,7 +52,11 @@ import { attemptInvoke, tryFallbackProviders } from '@/llm/invoke';
 import { shouldTriggerSummarization } from '@/summarization';
 import { createSummarizeNode } from '@/summarization/node';
 import { messagesStateReducer } from '@/messages/reducer';
-import { appendCallbacks } from '@/utils/callbacks';
+import {
+  appendCallbacks,
+  findCallback,
+  type CallbackEntry,
+} from '@/utils/callbacks';
 import { createSchemaOnlyTools } from '@/tools/schema';
 import { AgentContext } from '@/agents/AgentContext';
 import { createFakeStreamingLLM } from '@/llm/fake';
@@ -64,12 +68,20 @@ import { isThinkingEnabled } from '@/llm/request';
 import { initializeModel } from '@/llm/init';
 import {
   createLangfuseHandler,
-  disposeLangfuseHandler,
   createLangfuseTraceMetadata,
+  disposeLangfuseHandler,
+  isLangfuseCallbackHandler,
 } from '@/langfuse';
+import { initializeLangfuseTracing } from '@/instrumentation';
+import {
+  resolveLangfuseConfig,
+  shouldTraceToolNodeForLangfuse,
+  withLangfuseToolOutputTracingConfig,
+} from '@/langfuseToolOutputTracing';
 import { HandlerRegistry } from '@/events';
 import { ChatOpenAI } from '@/llm/openai';
 import { partitionAndMarkOpenRouterToolCache } from '@/llm/openrouter/toolCache';
+import { partitionAndMarkBedrockToolCache } from '@/llm/bedrock/toolCache';
 import type { HookRegistry } from '@/hooks';
 
 const { AGENT, TOOLS, SUMMARIZE } = GraphNodeKeys;
@@ -174,6 +186,10 @@ export abstract class Graph<
    * graph compiles.
    */
   toolOutputReferences: t.ToolOutputReferencesConfig | undefined;
+  /**
+   * Run-scoped Langfuse defaults. Per-agent config wins when present.
+   */
+  langfuse: t.LangfuseConfig | undefined;
   /**
    * Run-scoped opt-in for eager event-driven tool execution. The stream
    * handler may prestart eligible event-driven tools; ToolNode later
@@ -427,6 +443,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     runId,
     signal,
     agents,
+    langfuse,
     tokenCounter,
     indexTokenCountMap,
     calibrationRatio,
@@ -434,6 +451,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     super();
     this.runId = runId;
     this.signal = signal;
+    this.langfuse = langfuse;
 
     if (agents.length === 0) {
       throw new Error('At least one agent configuration is required');
@@ -743,6 +761,10 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     const toolDefinitions = agentContext?.toolDefinitions;
     const eventDrivenMode =
       toolDefinitions != null && toolDefinitions.length > 0;
+    const traceToolNode = shouldTraceToolNodeForLangfuse({
+      runLangfuse: this.langfuse,
+      agentLangfuse: agentContext?.langfuse,
+    });
 
     if (eventDrivenMode) {
       const schemaTools = createSchemaOnlyTools(toolDefinitions);
@@ -770,6 +792,9 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
       const node = new CustomToolNode<t.BaseGraphState>({
         tools: allTools,
         toolMap: allToolMap,
+        trace: traceToolNode,
+        runLangfuse: this.langfuse,
+        agentLangfuse: agentContext?.langfuse,
         eventDrivenMode: true,
         sessions: this.sessions,
         toolDefinitions: toolDefMap,
@@ -815,6 +840,9 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
     const node = new CustomToolNode<t.BaseGraphState>({
       tools: allTraditionalTools,
       toolMap: traditionalToolMap,
+      trace: traceToolNode,
+      runLangfuse: this.langfuse,
+      agentLangfuse: agentContext?.langfuse,
       toolCallStepIds: this.toolCallStepIds,
       errorHandler: (data, metadata): Promise<void> =>
         StandardGraph.handleToolCallErrorStatic(this, data, metadata),
@@ -935,6 +963,19 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
             rawToolsForBinding,
             makeIsDeferred(agentContext.toolDefinitions)
           ) ?? rawToolsForBinding;
+      } else if (
+        agentContext.provider === Providers.BEDROCK &&
+        (
+          agentContext.clientOptions as
+            | t.BedrockAnthropicClientOptions
+            | undefined
+        )?.promptCache === true
+      ) {
+        toolsForBinding =
+          partitionAndMarkBedrockToolCache(
+            rawToolsForBinding,
+            makeIsDeferred(agentContext.toolDefinitions)
+          ) ?? rawToolsForBinding;
       }
 
       let model =
@@ -1338,44 +1379,72 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
         { force: true }
       );
 
-      const langfuseHandler = createLangfuseHandler({
-        langfuse: agentContext.langfuse,
-        userId: config.configurable?.user_id as string | undefined,
-        sessionId: config.configurable?.thread_id as string | undefined,
-        traceMetadata: createLangfuseTraceMetadata({
-          messageId: this.runId,
-          parentMessageId: config.configurable?.requestBody?.parentMessageId,
-          agentId,
-          agentName: agentContext.name,
-        }),
-        tags: ['librechat', 'agent'],
+      const langfuse = resolveLangfuseConfig(
+        this.langfuse,
+        agentContext.langfuse
+      );
+      const traceMetadata = createLangfuseTraceMetadata({
+        messageId: this.runId,
+        parentMessageId: config.configurable?.requestBody?.parentMessageId,
+        agentId,
+        agentName: agentContext.name,
       });
-      const invokeConfig = langfuseHandler
-        ? {
-          ...config,
-          callbacks: appendCallbacks(config.callbacks, [langfuseHandler]),
+      let langfuseHandler: CallbackEntry | undefined;
+      let invokeConfig = {
+        ...config,
+        metadata: {
+          ...(config.metadata ?? {}),
+          ...traceMetadata,
+        },
+      };
+      initializeLangfuseTracing(langfuse);
+      if (findCallback(config.callbacks, isLangfuseCallbackHandler) == null) {
+        langfuseHandler = createLangfuseHandler({
+          langfuse,
+          userId: config.configurable?.user_id as string | undefined,
+          sessionId: config.configurable?.thread_id as string | undefined,
+          traceMetadata,
+          tags: ['librechat', 'agent'],
+        });
+        if (langfuseHandler != null) {
+          invokeConfig = {
+            ...invokeConfig,
+            callbacks: appendCallbacks(invokeConfig.callbacks, [
+              langfuseHandler,
+            ]),
+          };
         }
-        : config;
+      }
 
       try {
-        result = await attemptInvoke(
-          {
-            model: (this.overrideModel ?? model) as t.ChatModel,
-            messages: finalMessages,
-            provider: agentContext.provider,
-            context: this,
-          },
-          invokeConfig
+        result = await withLangfuseToolOutputTracingConfig(
+          this.langfuse,
+          () =>
+            attemptInvoke(
+              {
+                model: (this.overrideModel ?? model) as t.ChatModel,
+                messages: finalMessages,
+                provider: agentContext.provider,
+                context: this,
+              },
+              invokeConfig
+            ),
+          agentContext.langfuse
         );
       } catch (primaryError) {
-        result = await tryFallbackProviders({
-          fallbacks,
-          tools: agentContext.tools,
-          messages: finalMessages,
-          config: invokeConfig,
-          primaryError,
-          context: this,
-        });
+        result = await withLangfuseToolOutputTracingConfig(
+          this.langfuse,
+          () =>
+            tryFallbackProviders({
+              fallbacks,
+              tools: agentContext.tools,
+              messages: finalMessages,
+              config: invokeConfig,
+              primaryError,
+              context: this,
+            }),
+          agentContext.langfuse
+        );
       } finally {
         await disposeLangfuseHandler(langfuseHandler);
       }
@@ -1599,6 +1668,7 @@ export class StandardGraph extends Graph<t.BaseGraphState, t.GraphNode> {
           parentHandlerRegistry: getParentHandlerRegistry,
           parentRunId: this.runId ?? '',
           parentAgentId: agentContext.agentId,
+          langfuse: this.langfuse,
           tokenCounter: agentContext.tokenCounter,
           maxDepth: effectiveSubagentDepth,
           createChildGraph: (input): StandardGraph => {

package/src/instrumentation.ts

@@ -1,17 +1,178 @@
-import { NodeSDK } from '@opentelemetry/sdk-node';
-import { LangfuseSpanProcessor } from '@langfuse/otel';
+import { setLangfuseTracerProvider } from '@langfuse/tracing';
+import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base';
+import { context, ROOT_CONTEXT, createContextKey } from '@opentelemetry/api';
+import { AsyncLocalStorageContextManager } from '@opentelemetry/context-async-hooks';
+import {
+  hasLangfuseConfigCredentials,
+  hasLangfuseEnvCredentials,
+  hasLangfuseEnvConfig,
+} from '@/langfuse';
+import {
+  createLangfuseSpanProcessor,
+  getContextLangfuseConfig,
+} from '@/langfuseToolOutputTracing';
 import { isPresent } from '@/utils/misc';
+import type {
+  ReadableSpan,
+  Span,
+  SpanProcessor,
+} from '@opentelemetry/sdk-trace-base';
+import type { LangfuseSpanProcessorParams } from '@langfuse/otel';
+import type { Context } from '@opentelemetry/api';
+import type * as t from '@/types';
 
-if (
-  isPresent(process.env.LANGFUSE_SECRET_KEY) &&
-  isPresent(process.env.LANGFUSE_PUBLIC_KEY) &&
-  isPresent(process.env.LANGFUSE_BASE_URL ?? process.env.LANGFUSE_BASEURL)
-) {
-  const langfuseSpanProcessor = new LangfuseSpanProcessor();
+let langfuseTracerProvider: BasicTracerProvider | undefined;
+let langfuseRoutingSpanProcessor: RoutingLangfuseSpanProcessor | undefined;
+const contextManagerProbeKey = createContextKey(
+  'langfuse-context-manager-probe'
+);
 
-  const sdk = new NodeSDK({
-    spanProcessors: [langfuseSpanProcessor],
+function hasActiveContextManager(): boolean {
+  return context.with(
+    ROOT_CONTEXT.setValue(contextManagerProbeKey, true),
+    () => context.active().getValue(contextManagerProbeKey) === true
+  );
+}
+
+export function ensureOpenTelemetryContextManager(): void {
+  if (hasActiveContextManager()) {
+    return;
+  }
+
+  const contextManager = new AsyncLocalStorageContextManager();
+  contextManager.enable();
+  if (!context.setGlobalContextManager(contextManager)) {
+    contextManager.disable();
+  }
+}
+
+function getLangfuseSpanProcessorParams(
+  langfuse?: t.LangfuseConfig
+): LangfuseSpanProcessorParams | undefined {
+  if (langfuse?.enabled === false) {
+    return undefined;
+  }
+  if (hasLangfuseConfigCredentials(langfuse)) {
+    return {
+      publicKey: langfuse.publicKey,
+      secretKey: langfuse.secretKey,
+      ...(isPresent(langfuse.baseUrl) ? { baseUrl: langfuse.baseUrl } : {}),
+    };
+  }
+  if (hasLangfuseEnvConfig()) {
+    const baseUrl =
+      langfuse?.baseUrl ??
+      process.env.LANGFUSE_BASE_URL ??
+      process.env.LANGFUSE_BASEURL;
+    return {
+      publicKey: process.env.LANGFUSE_PUBLIC_KEY as string,
+      secretKey: process.env.LANGFUSE_SECRET_KEY as string,
+      ...(isPresent(baseUrl) ? { baseUrl } : {}),
+    };
+  }
+  if (isPresent(langfuse?.baseUrl) && hasLangfuseEnvCredentials()) {
+    return {
+      publicKey: process.env.LANGFUSE_PUBLIC_KEY as string,
+      secretKey: process.env.LANGFUSE_SECRET_KEY as string,
+      baseUrl: langfuse.baseUrl,
+    };
+  }
+  return undefined;
+}
+
+function getLangfuseTracerProviderKey(
+  params: LangfuseSpanProcessorParams,
+  langfuse?: t.LangfuseConfig
+): string {
+  return JSON.stringify({
+    publicKey: params.publicKey,
+    secretKey: params.secretKey,
+    baseUrl: params.baseUrl,
+    environment: params.environment,
+    toolOutputTracing: langfuse?.toolOutputTracing,
   });
+}
+
+class RoutingLangfuseSpanProcessor implements SpanProcessor {
+  private readonly processors = new Map<string, SpanProcessor>();
+  private readonly spanProcessors = new WeakMap<object, SpanProcessor>();
+
+  ensureProcessor(langfuse?: t.LangfuseConfig): SpanProcessor | undefined {
+    const params = getLangfuseSpanProcessorParams(langfuse);
+    if (params == null) {
+      return undefined;
+    }
+
+    const processorKey = getLangfuseTracerProviderKey(params, langfuse);
+    const existing = this.processors.get(processorKey);
+    if (existing != null) {
+      return existing;
+    }
+
+    const processor = createLangfuseSpanProcessor(params, langfuse);
+    this.processors.set(processorKey, processor);
+    return processor;
+  }
+
+  onStart(span: Span, parentContext: Context): void {
+    const processor = this.ensureProcessor(
+      getContextLangfuseConfig(parentContext)
+    );
+    if (processor == null) {
+      return;
+    }
 
-  sdk.start();
+    this.spanProcessors.set(span, processor);
+    processor.onStart(span, parentContext);
+  }
+
+  onEnd(span: ReadableSpan): void {
+    this.spanProcessors.get(span)?.onEnd(span);
+  }
+
+  async forceFlush(): Promise<void> {
+    await Promise.all(
+      Array.from(this.processors.values(), (processor) =>
+        processor.forceFlush()
+      )
+    );
+  }
+
+  async shutdown(): Promise<void> {
+    await Promise.all(
+      Array.from(this.processors.values(), (processor) => processor.shutdown())
+    );
+  }
+}
+
+export function initializeLangfuseTracing(
+  langfuse?: t.LangfuseConfig
+): BasicTracerProvider | undefined {
+  const params = getLangfuseSpanProcessorParams(langfuse);
+  if (params == null) {
+    return undefined;
+  }
+
+  if (langfuseTracerProvider != null) {
+    langfuseRoutingSpanProcessor?.ensureProcessor(langfuse);
+    return langfuseTracerProvider;
+  }
+
+  ensureOpenTelemetryContextManager();
+  langfuseRoutingSpanProcessor = new RoutingLangfuseSpanProcessor();
+  langfuseRoutingSpanProcessor.ensureProcessor(langfuse);
+  langfuseTracerProvider = new BasicTracerProvider({
+    spanProcessors: [langfuseRoutingSpanProcessor],
+  });
+
+  setLangfuseTracerProvider(langfuseTracerProvider);
+  return langfuseTracerProvider;
 }
+
+export function initializeLangfuseTracingFromEnv():
+  | BasicTracerProvider
+  | undefined {
+  return initializeLangfuseTracing();
+}
+
+initializeLangfuseTracingFromEnv();