@librechat/agents 3.1.89 → 3.1.91

package/src/langfuse.ts

@@ -0,0 +1,358 @@
+import { CallbackHandler } from '@langfuse/langchain';
+import { LangfuseSpanProcessor } from '@langfuse/otel';
+import {
+  createObservationAttributes,
+  createTraceAttributes,
+} from '@langfuse/tracing';
+import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
+import { BasicTracerProvider } from '@opentelemetry/sdk-trace-base';
+import { SpanStatusCode } from '@opentelemetry/api';
+import type { Serialized } from '@langchain/core/load/serializable';
+import type { BaseMessage } from '@langchain/core/messages';
+import type { LLMResult } from '@langchain/core/outputs';
+import type { Span } from '@opentelemetry/api';
+import type * as t from '@/types';
+import { isPresent } from '@/utils/misc';
+
+type TraceMetadata = Record<string, unknown>;
+
+type LangfuseHandlerParams = {
+  userId?: string;
+  sessionId?: string;
+  traceMetadata?: TraceMetadata;
+};
+
+type AgentLangfuseHandlerParams = LangfuseHandlerParams & {
+  langfuse?: t.LangfuseConfig;
+};
+
+type ResolvedLangfuseConfig = t.LangfuseConfig & {
+  enabled: true;
+  publicKey: string;
+  secretKey: string;
+};
+
+function getModelName(serialized: Serialized): string {
+  const serializedRecord = serialized as unknown as Record<string, unknown>;
+  const kwargs = serializedRecord.kwargs as Record<string, unknown> | undefined;
+  const modelName =
+    kwargs?.model ??
+    kwargs?.model_name ??
+    kwargs?.modelName ??
+    kwargs?.model_id ??
+    kwargs?.modelId ??
+    serializedRecord.name;
+
+  if (typeof modelName === 'string' && modelName.trim() !== '') {
+    return modelName;
+  }
+
+  if (Array.isArray(serializedRecord.id) && serializedRecord.id.length > 0) {
+    return String(serializedRecord.id[serializedRecord.id.length - 1]);
+  }
+
+  return 'ChatModel';
+}
+
+function getModelParameters(
+  extraParams?: Record<string, unknown>
+): Record<string, string | number> {
+  const invocationParams = extraParams?.invocation_params;
+  const params =
+    invocationParams != null && typeof invocationParams === 'object'
+      ? (invocationParams as Record<string, unknown>)
+      : (extraParams ?? {});
+
+  return Object.fromEntries(
+    Object.entries(params).filter(([, value]) => {
+      return typeof value === 'string' || typeof value === 'number';
+    })
+  ) as Record<string, string | number>;
+}
+
+function getOutput(output: LLMResult): unknown {
+  return output.generations.map((generation) =>
+    generation.map((item) => {
+      if ('message' in item && item.message != null) {
+        return (item.message as { content?: unknown }).content;
+      }
+      return item.text;
+    })
+  );
+}
+
+function getUsageDetails(
+  output: LLMResult
+): Record<string, number> | undefined {
+  const llmOutput = output.llmOutput as Record<string, unknown> | undefined;
+  const usage = llmOutput?.tokenUsage ?? llmOutput?.usage;
+  if (usage == null || typeof usage !== 'object') {
+    return undefined;
+  }
+
+  const usageEntries = Object.entries(usage as Record<string, unknown>).filter(
+    ([, value]) => typeof value === 'number'
+  );
+
+  return usageEntries.length > 0
+    ? (Object.fromEntries(usageEntries) as Record<string, number>)
+    : undefined;
+}
+
+function getTraceName(traceMetadata?: TraceMetadata): string {
+  const agentName = traceMetadata?.agentName;
+  return typeof agentName === 'string' && agentName.trim() !== ''
+    ? `LibreChat Agent: ${agentName}`
+    : 'LibreChat Agent';
+}
+
+export class LangfuseAgentCallbackHandler extends BaseCallbackHandler {
+  name = 'librechat_langfuse_agent_handler';
+
+  private readonly provider: BasicTracerProvider;
+  private readonly processor: LangfuseSpanProcessor;
+  private readonly userId?: string;
+  private readonly sessionId?: string;
+  private readonly traceMetadata?: TraceMetadata;
+  private readonly spans = new Map<string, Span>();
+
+  constructor({
+    langfuse,
+    userId,
+    sessionId,
+    traceMetadata,
+  }: LangfuseHandlerParams & { langfuse: ResolvedLangfuseConfig }) {
+    super();
+    this.userId = userId;
+    this.sessionId = sessionId;
+    this.traceMetadata = traceMetadata;
+    this.processor = new LangfuseSpanProcessor({
+      publicKey: langfuse.publicKey,
+      secretKey: langfuse.secretKey,
+      ...(isPresent(langfuse.baseUrl) ? { baseUrl: langfuse.baseUrl } : {}),
+      environment:
+        process.env.LANGFUSE_TRACING_ENVIRONMENT ??
+        process.env.NODE_ENV ??
+        'development',
+      exportMode: 'immediate',
+    });
+    this.provider = new BasicTracerProvider({
+      spanProcessors: [this.processor],
+    });
+  }
+
+  private startGenerationSpan({
+    llm,
+    input,
+    runId,
+    extraParams,
+    metadata,
+    name,
+  }: {
+    llm: Serialized;
+    input: unknown;
+    runId: string;
+    extraParams?: Record<string, unknown>;
+    metadata?: Record<string, unknown>;
+    name?: string;
+  }): void {
+    if (this.spans.has(runId)) {
+      return;
+    }
+
+    const tracer = this.provider.getTracer('librechat-agents-langfuse');
+    const spanName =
+      typeof name === 'string' && name.trim() !== '' ? name : getModelName(llm);
+    const span = tracer.startSpan(spanName, {
+      attributes: {
+        ...createTraceAttributes({
+          name: getTraceName(this.traceMetadata),
+          userId: this.userId,
+          sessionId: this.sessionId,
+          metadata: this.traceMetadata,
+        }),
+        ...createObservationAttributes('generation', {
+          input,
+          model: getModelName(llm),
+          modelParameters: getModelParameters(extraParams),
+          metadata: {
+            ...metadata,
+            ...this.traceMetadata,
+          },
+        }),
+      },
+    });
+    this.spans.set(runId, span);
+  }
+
+  async handleChatModelStart(
+    llm: Serialized,
+    messages: BaseMessage[][],
+    runId: string,
+    _parentRunId?: string,
+    extraParams?: Record<string, unknown>,
+    _tags?: string[],
+    metadata?: Record<string, unknown>,
+    name?: string
+  ): Promise<void> {
+    this.startGenerationSpan({
+      llm,
+      input: messages,
+      runId,
+      extraParams,
+      metadata,
+      name,
+    });
+  }
+
+  async handleLLMStart(
+    llm: Serialized,
+    prompts: string[],
+    runId: string,
+    _parentRunId?: string,
+    extraParams?: Record<string, unknown>,
+    _tags?: string[],
+    metadata?: Record<string, unknown>,
+    name?: string
+  ): Promise<void> {
+    this.startGenerationSpan({
+      llm,
+      input: prompts,
+      runId,
+      extraParams,
+      metadata,
+      name,
+    });
+  }
+
+  async handleLLMEnd(output: LLMResult, runId: string): Promise<void> {
+    const span = this.spans.get(runId);
+    if (!span) {
+      return;
+    }
+
+    span.setAttributes(
+      createObservationAttributes('generation', {
+        output: getOutput(output),
+        usageDetails: getUsageDetails(output),
+      })
+    );
+    span.end();
+    this.spans.delete(runId);
+    await this.flush();
+  }
+
+  async handleLLMError(err: unknown, runId: string): Promise<void> {
+    const span = this.spans.get(runId);
+    if (!span) {
+      return;
+    }
+
+    const message = err instanceof Error ? err.message : String(err);
+    span.setStatus({ code: SpanStatusCode.ERROR, message });
+    span.setAttributes(
+      createObservationAttributes('generation', {
+        level: 'ERROR',
+        statusMessage: message,
+      })
+    );
+    span.end();
+    this.spans.delete(runId);
+    await this.flush();
+  }
+
+  private async flush(): Promise<void> {
+    try {
+      await this.provider.forceFlush();
+    } catch (error) {
+      process.emitWarning(
+        `[LangfuseAgentCallbackHandler] Failed to flush Langfuse spans: ${
+          error instanceof Error ? error.message : String(error)
+        }`
+      );
+    }
+  }
+
+  async dispose(): Promise<void> {
+    for (const span of this.spans.values()) {
+      span.end();
+    }
+    this.spans.clear();
+    await this.flush();
+    try {
+      await this.provider.shutdown();
+    } catch (error) {
+      process.emitWarning(
+        `[LangfuseAgentCallbackHandler] Failed to shut down Langfuse provider: ${
+          error instanceof Error ? error.message : String(error)
+        }`
+      );
+    }
+  }
+}
+
+function hasRequiredLangfuseConfig(
+  langfuse?: t.LangfuseConfig
+): langfuse is ResolvedLangfuseConfig {
+  return (
+    langfuse?.enabled === true &&
+    isPresent(langfuse.publicKey) &&
+    isPresent(langfuse.secretKey)
+  );
+}
+
+export function createLegacyLangfuseHandler(
+  params: LangfuseHandlerParams
+): CallbackHandler {
+  return new CallbackHandler(params);
+}
+
+export function createLangfuseHandler({
+  langfuse,
+  userId,
+  sessionId,
+  traceMetadata,
+}: AgentLangfuseHandlerParams): LangfuseAgentCallbackHandler | undefined {
+  if (!hasRequiredLangfuseConfig(langfuse)) {
+    return undefined;
+  }
+
+  return new LangfuseAgentCallbackHandler({
+    langfuse,
+    userId,
+    sessionId,
+    traceMetadata,
+  });
+}
+
+export function hasExplicitLangfuseConfig(
+  contexts: Iterable<{ langfuse?: t.LangfuseConfig }>
+): boolean {
+  for (const context of contexts) {
+    if (context.langfuse != null) {
+      return true;
+    }
+  }
+  return false;
+}
+
+export function hasLangfuseEnvConfig(): boolean {
+  return (
+    isPresent(process.env.LANGFUSE_SECRET_KEY) &&
+    isPresent(process.env.LANGFUSE_PUBLIC_KEY) &&
+    isPresent(process.env.LANGFUSE_BASE_URL)
+  );
+}
+
+export function isLangfuseCallbackHandler(value: unknown): boolean {
+  return (
+    value instanceof CallbackHandler ||
+    value instanceof LangfuseAgentCallbackHandler
+  );
+}
+
+export async function disposeLangfuseHandler(value: unknown): Promise<void> {
+  if (value instanceof LangfuseAgentCallbackHandler) {
+    await value.dispose();
+  }
+}

package/src/llm/anthropic/index.ts

@@ -17,9 +17,13 @@ import type {
   ChatAnthropicToolType,
   AnthropicMCPServerURLDefinition,
   AnthropicContextManagementConfigParam,
+  AnthropicRequestOptions,
 } from '@/llm/anthropic/types';
 import { _makeMessageChunkFromAnthropicEvent } from './utils/message_outputs';
-import { _convertMessagesToAnthropicPayload } from './utils/message_inputs';
+import {
+  _convertMessagesToAnthropicPayload,
+  stripUnsupportedAssistantPrefill,
+} from './utils/message_inputs';
 import { handleToolChoice } from './utils/tools';
 
 const DEFAULT_STREAM_DELAY = 25;
@@ -591,6 +595,26 @@ export class CustomAnthropic extends ChatAnthropicMessages {
     });
   }
 
+  protected override async createStreamWithRetry(
+    request: AnthropicStreamingMessageCreateParams,
+    options?: AnthropicRequestOptions
+  ): ReturnType<ChatAnthropicMessages['createStreamWithRetry']> {
+    return super.createStreamWithRetry(
+      stripUnsupportedAssistantPrefill(request),
+      options
+    );
+  }
+
+  protected override async completionWithRetry(
+    request: AnthropicMessageCreateParams,
+    options: AnthropicRequestOptions
+  ): ReturnType<ChatAnthropicMessages['completionWithRetry']> {
+    return super.completionWithRetry(
+      stripUnsupportedAssistantPrefill(request),
+      options
+    );
+  }
+
   async *_streamResponseChunks(
     messages: BaseMessage[],
     options: this['ParsedCallOptions'],
@@ -599,11 +623,11 @@ export class CustomAnthropic extends ChatAnthropicMessages {
     this.resetTokenEvents();
     const params = this.invocationParams(options);
     const formattedMessages = _convertMessagesToAnthropicPayload(messages);
-    const payload = {
+    const payload = stripUnsupportedAssistantPrefill({
       ...params,
       ...formattedMessages,
       stream: true,
-    } as const;
+    } as const);
     const coerceContentToString =
       !_toolsInParams(payload) &&
       !_documentsInParams(payload) &&

package/src/llm/anthropic/llm.spec.ts

@@ -64,7 +64,11 @@ import type {
   ToolEndEvent,
   TPayload,
 } from '@/types';
-import { _convertMessagesToAnthropicPayload } from './utils/message_inputs';
+import {
+  _convertMessagesToAnthropicPayload,
+  modelDisallowsAssistantPrefill,
+  stripUnsupportedAssistantPrefill,
+} from './utils/message_inputs';
 import {
   _makeMessageChunkFromAnthropicEvent,
   getAnthropicUsageMetadata,
@@ -2637,6 +2641,61 @@ describe('Anthropic Reasoning with contentBlocks', () => {
   });
 });
 
+describe('Claude assistant prefill compatibility', () => {
+  test.each([
+    'claude-sonnet-4-6',
+    'claude-sonnet-4-6@20260217',
+    'claude-opus-4-7',
+    'claude-opus-4-10',
+    'global.anthropic.claude-opus-4-6-v1:0',
+    'anthropic/claude-sonnet-4.6',
+    'anthropic/claude-sonnet-4.12',
+  ])('detects %s as not supporting assistant prefill', (model) => {
+    expect(modelDisallowsAssistantPrefill(model)).toBe(true);
+  });
+
+  test.each([
+    'claude-sonnet-4-5-20250929',
+    'claude-opus-4-20250514',
+    'anthropic.claude-opus-4-20250514-v1:0',
+    'gpt-5.4',
+  ])('leaves %s prefill support unchanged', (model) => {
+    expect(modelDisallowsAssistantPrefill(model)).toBe(false);
+  });
+
+  test('strips trailing assistant messages for Claude 4.6+ requests', () => {
+    const request = {
+      model: 'claude-opus-4-6',
+      max_tokens: 100,
+      messages: [
+        { role: 'user' as const, content: 'What changed?' },
+        { role: 'assistant' as const, content: 'Draft prefill' },
+        { role: 'assistant' as const, content: 'Another prefill' },
+      ],
+    };
+
+    const sanitized = stripUnsupportedAssistantPrefill(request);
+
+    expect(sanitized).not.toBe(request);
+    expect(sanitized.messages).toEqual([
+      { role: 'user', content: 'What changed?' },
+    ]);
+  });
+
+  test('does not strip assistant messages for older Claude models', () => {
+    const request = {
+      model: 'claude-sonnet-4-5-20250929',
+      max_tokens: 100,
+      messages: [
+        { role: 'user' as const, content: 'Write JSON only.' },
+        { role: 'assistant' as const, content: '{' },
+      ],
+    };
+
+    expect(stripUnsupportedAssistantPrefill(request)).toBe(request);
+  });
+});
+
 const opus46Model = 'claude-opus-4-6';
 
 describe('Opus 4.6', () => {

package/src/llm/anthropic/utils/message_inputs.ts

@@ -49,6 +49,10 @@ type GoogleFunctionCallBlock = MessageContentComplex & {
 };
 
 const ANTHROPIC_EMPTY_TEXT_PLACEHOLDER = '_';
+const CLAUDE_4_RELEASE_DATE_MODEL_PATTERN =
+  /claude-(?:opus|sonnet|haiku)-4-\d{8}(?:[-.@]|$)/i;
+const CLAUDE_4_MINOR_MODEL_PATTERN =
+  /claude-(?:opus|sonnet|haiku)-4[-.](\d+)(?:[-.@]|$)/i;
 
 function _formatImage(imageUrl: string) {
   const parsed = parseBase64DataUrl({ dataUrl: imageUrl });
@@ -796,6 +800,48 @@ export function _convertMessagesToAnthropicPayload(
   } as AnthropicMessageCreateParams;
 }
 
+export function modelDisallowsAssistantPrefill(model?: string): boolean {
+  const modelId = model ?? '';
+  if (CLAUDE_4_RELEASE_DATE_MODEL_PATTERN.test(modelId)) {
+    return false;
+  }
+
+  const match = CLAUDE_4_MINOR_MODEL_PATTERN.exec(modelId);
+  if (!match) {
+    return false;
+  }
+  return Number(match[1]) >= 6;
+}
+
+export function stripUnsupportedAssistantPrefill<
+  T extends Pick<AnthropicMessageCreateParams, 'messages'> & { model?: string },
+>(request: T): T {
+  if (!modelDisallowsAssistantPrefill(request.model)) {
+    return request;
+  }
+
+  const messages = request.messages;
+  if (
+    messages.length <= 1 ||
+    messages[messages.length - 1]?.role !== 'assistant'
+  ) {
+    return request;
+  }
+
+  const nextMessages = [...messages];
+  while (
+    nextMessages.length > 1 &&
+    nextMessages[nextMessages.length - 1]?.role === 'assistant'
+  ) {
+    nextMessages.pop();
+  }
+
+  return {
+    ...request,
+    messages: nextMessages,
+  };
+}
+
 function mergeMessages(messages: AnthropicMessageCreateParams['messages']) {
   if (messages.length <= 1) {
     return messages;

package/src/run.ts

@@ -1,6 +1,5 @@
 // src/run.ts
 import './instrumentation';
-import { CallbackHandler } from '@langfuse/langchain';
 import { PromptTemplate } from '@langchain/core/prompts';
 import { RunnableLambda } from '@langchain/core/runnables';
 import { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';
@@ -31,7 +30,14 @@ import { initializeModel } from '@/llm/init';
 import { HandlerRegistry } from '@/events';
 import { executeHooks } from '@/hooks';
 import { isOpenAILike } from '@/utils/llm';
-import { isPresent } from '@/utils/misc';
+import {
+  createLegacyLangfuseHandler,
+  createLangfuseHandler,
+  disposeLangfuseHandler,
+  hasExplicitLangfuseConfig,
+  hasLangfuseEnvConfig,
+  isLangfuseCallbackHandler,
+} from '@/langfuse';
 import type { HookRegistry } from '@/hooks';
 
 export const defaultOmitOptions = new Set([
@@ -607,9 +613,8 @@ export class Run<_T extends t.BaseGraphState> {
       .concat(customHandler);
 
     if (
-      isPresent(process.env.LANGFUSE_SECRET_KEY) &&
-      isPresent(process.env.LANGFUSE_PUBLIC_KEY) &&
-      isPresent(process.env.LANGFUSE_BASE_URL)
+      hasLangfuseEnvConfig() &&
+      !hasExplicitLangfuseConfig(this.Graph.agentContexts.values())
     ) {
       const userId = config.configurable?.user_id;
       const sessionId = config.configurable?.thread_id;
@@ -621,7 +626,7 @@ export class Run<_T extends t.BaseGraphState> {
         parentMessageId: config.configurable?.requestBody?.parentMessageId,
         agentName: primaryContext?.name,
       };
-      const handler = new CallbackHandler({
+      const handler = createLegacyLangfuseHandler({
         userId,
         sessionId,
         traceMetadata,
@@ -1134,12 +1139,8 @@ export class Run<_T extends t.BaseGraphState> {
     titleMethod = TitleMethod.COMPLETION,
     titlePromptTemplate,
   }: t.RunTitleOptions): Promise<{ language?: string; title?: string }> {
-    if (
-      chainOptions != null &&
-      isPresent(process.env.LANGFUSE_SECRET_KEY) &&
-      isPresent(process.env.LANGFUSE_PUBLIC_KEY) &&
-      isPresent(process.env.LANGFUSE_BASE_URL)
-    ) {
+    let titleLangfuseHandler: unknown;
+    if (chainOptions != null) {
       const userId = chainOptions.configurable?.user_id;
       const sessionId = chainOptions.configurable?.thread_id;
       const titleContext = this.Graph?.agentContexts.get(
@@ -1149,14 +1150,31 @@ export class Run<_T extends t.BaseGraphState> {
         messageId: 'title-' + this.id,
         agentName: titleContext?.name,
       };
-      const handler = new CallbackHandler({
-        userId,
-        sessionId,
-        traceMetadata,
-      });
-      chainOptions.callbacks = (
-        (chainOptions.callbacks as t.ProvidedCallbacks) ?? []
-      ).concat([handler]);
+      const hasExplicitLangfuse =
+        this.Graph != null &&
+        hasExplicitLangfuseConfig(this.Graph.agentContexts.values());
+      if (titleContext?.langfuse != null) {
+        titleLangfuseHandler = createLangfuseHandler({
+          langfuse: titleContext.langfuse,
+          userId,
+          sessionId,
+          traceMetadata,
+        });
+      } else if (hasLangfuseEnvConfig() && !hasExplicitLangfuse) {
+        titleLangfuseHandler = createLegacyLangfuseHandler({
+          userId,
+          sessionId,
+          traceMetadata,
+        });
+      }
+
+      if (titleLangfuseHandler != null) {
+        chainOptions.callbacks = (
+          (chainOptions.callbacks as t.ProvidedCallbacks) ?? []
+        ).concat([
+          titleLangfuseHandler as NonNullable<t.ProvidedCallbacks>[number],
+        ]);
+      }
     }
 
     const convoTemplate = PromptTemplate.fromTemplate(
@@ -1221,24 +1239,28 @@ export class Run<_T extends t.BaseGraphState> {
     });
 
     try {
-      return await fullChain.invoke(
-        { input: inputText, output: response },
-        invokeConfig
-      );
-    } catch (_e) {
-      // Fallback: strip callbacks to avoid EventStream tracer errors in certain environments
-      // But preserve langfuse handler if it exists
-      const langfuseHandler = (
-        invokeConfig.callbacks as t.ProvidedCallbacks
-      )?.find((cb) => cb instanceof CallbackHandler);
-      const { callbacks: _cb, ...rest } = invokeConfig;
-      const safeConfig = Object.assign({}, rest, {
-        callbacks: langfuseHandler ? [langfuseHandler] : [],
-      });
-      return await fullChain.invoke(
-        { input: inputText, output: response },
-        safeConfig as Partial<RunnableConfig>
-      );
+      try {
+        return await fullChain.invoke(
+          { input: inputText, output: response },
+          invokeConfig
+        );
+      } catch (_e) {
+        // Fallback: strip callbacks to avoid EventStream tracer errors in certain environments
+        // but preserve Langfuse tracing if it exists.
+        const langfuseHandler = (
+          invokeConfig.callbacks as t.ProvidedCallbacks
+        )?.find(isLangfuseCallbackHandler);
+        const { callbacks: _cb, ...rest } = invokeConfig;
+        const safeConfig = Object.assign({}, rest, {
+          callbacks: langfuseHandler ? [langfuseHandler] : [],
+        });
+        return await fullChain.invoke(
+          { input: inputText, output: response },
+          safeConfig as Partial<RunnableConfig>
+        );
+      }
+    } finally {
+      await disposeLangfuseHandler(titleLangfuseHandler);
     }
   }
 }