@librechat/agents 3.4.4 → 3.4.6

package/src/specs/activity-label-observability.live.test.ts

@@ -0,0 +1,262 @@
+/**
+ * Live provider + Langfuse export verification for activity-label traces.
+ *
+ * Run with:
+ * RUN_ACTIVITY_LABEL_LANGFUSE_LIVE=1 OPENAI_API_KEY=... \
+ * LANGFUSE_PUBLIC_KEY=... LANGFUSE_SECRET_KEY=... LANGFUSE_BASE_URL=... \
+ * LANGFUSE_FORCE_FLUSH_ON_DISPOSE=true \
+ * npm test -- activity-label-observability.live.test.ts --runInBand
+ */
+import { config as dotenvConfig } from 'dotenv';
+dotenvConfig();
+
+import { HumanMessage } from '@langchain/core/messages';
+import { describe, expect, it, jest } from '@jest/globals';
+import { Providers } from '@/common';
+import { Run } from '@/run';
+
+type LangfuseMetadata = {
+  sourceRunId?: string;
+  responseId?: string;
+  activityIndex?: string | number;
+  phaseIndex?: string | number;
+  activityCount?: string | number;
+};
+
+type LangfuseObservation = {
+  id: string;
+  parentObservationId?: string | null;
+  type: string;
+  name: string;
+  model?: string | null;
+  usage?: { input?: number; output?: number; total?: number };
+};
+
+type LangfuseTrace = {
+  id: string;
+  name: string;
+  tags?: string[];
+  metadata?: LangfuseMetadata;
+  observations?: LangfuseObservation[];
+};
+
+type LangfuseTraceList = {
+  data: LangfuseTrace[];
+};
+
+const shouldRunLive =
+  process.env.RUN_ACTIVITY_LABEL_LANGFUSE_LIVE === '1' &&
+  (process.env.OPENAI_API_KEY ?? '') !== '' &&
+  (process.env.LANGFUSE_PUBLIC_KEY ?? '') !== '' &&
+  (process.env.LANGFUSE_SECRET_KEY ?? '') !== '' &&
+  (process.env.LANGFUSE_BASE_URL ?? '') !== '';
+
+const describeIfLive = shouldRunLive ? describe : describe.skip;
+
+function langfuseHeaders(): HeadersInit {
+  const credentials = Buffer.from(
+    `${process.env.LANGFUSE_PUBLIC_KEY}:${process.env.LANGFUSE_SECRET_KEY}`
+  ).toString('base64');
+  return { Authorization: `Basic ${credentials}` };
+}
+
+async function requestLangfuse(path: string): Promise<Response> {
+  const baseUrl = process.env.LANGFUSE_BASE_URL?.replace(/\/$/, '');
+  return fetch(`${baseUrl}${path}`, {
+    headers: langfuseHeaders(),
+  });
+}
+
+async function getJson<T>(path: string): Promise<T> {
+  const response = await requestLangfuse(path);
+  if (!response.ok) {
+    throw new Error(`Langfuse request failed: ${response.status}`);
+  }
+  return (await response.json()) as T;
+}
+
+async function getJsonIfPresent<T>(path: string): Promise<T | undefined> {
+  const response = await requestLangfuse(path);
+  if (response.status === 404) {
+    return undefined;
+  }
+  if (!response.ok) {
+    throw new Error(`Langfuse request failed: ${response.status}`);
+  }
+  return (await response.json()) as T;
+}
+
+async function findExportedTraces(
+  sourceRunId: string,
+  fromTimestamp: string
+): Promise<{ label: LangfuseTrace; phase: LangfuseTrace }> {
+  const query = new URLSearchParams({
+    limit: '100',
+    fromTimestamp,
+  });
+  for (let attempt = 0; attempt < 20; attempt += 1) {
+    const list = await getJson<LangfuseTraceList>(
+      `/api/public/traces?${query.toString()}`
+    );
+    const candidates = list.data.filter(
+      (trace) => trace.metadata?.sourceRunId === sourceRunId
+    );
+    const labelSummary = candidates.find(
+      (trace) => trace.tags?.includes('activity-label') === true
+    );
+    const phaseSummary = candidates.find(
+      (trace) => trace.tags?.includes('activity-phase') === true
+    );
+    if (labelSummary != null && phaseSummary != null) {
+      const [label, phase] = await Promise.all([
+        getJsonIfPresent<LangfuseTrace>(
+          `/api/public/traces/${labelSummary.id}`
+        ),
+        getJsonIfPresent<LangfuseTrace>(
+          `/api/public/traces/${phaseSummary.id}`
+        ),
+      ]);
+      if (label != null && phase != null) {
+        const labelReady =
+          label.observations?.some(
+            (observation) => observation.type === 'GENERATION'
+          ) === true;
+        const phaseRootReady =
+          phase.observations?.some(
+            (observation) =>
+              observation.parentObservationId == null &&
+              observation.type === 'CHAIN'
+          ) === true;
+        const phaseGenerationReady =
+          phase.observations?.some(
+            (observation) => observation.type === 'GENERATION'
+          ) === true;
+        if (labelReady && phaseRootReady && phaseGenerationReady) {
+          return { label, phase };
+        }
+      }
+    }
+    await new Promise((resolve) => setTimeout(resolve, 3000));
+  }
+  throw new Error(`Timed out waiting for activity traces from ${sourceRunId}`);
+}
+
+describeIfLive('activity label Langfuse export (live)', () => {
+  jest.setTimeout(120_000);
+
+  it('exports stable, correlated traces with correct observation types', async () => {
+    const startedAt = new Date(Date.now() - 5000).toISOString();
+    const sourceRunId = `activity-label-live-${Date.now()}`;
+    const sessionId = `${sourceRunId}-session`;
+    const model = process.env.ACTIVITY_LABEL_LIVE_MODEL ?? 'gpt-4.1-mini';
+    const run = await Run.create({
+      runId: sourceRunId,
+      graphConfig: {
+        type: 'standard',
+        agents: [
+          {
+            agentId: 'activity-label-live-agent',
+            name: 'Volatile Agent Display Name',
+            provider: Providers.OPENAI,
+            clientOptions: {
+              apiKey: process.env.OPENAI_API_KEY,
+              model,
+            },
+            tools: [],
+          },
+        ],
+      },
+    });
+    if (run.Graph != null) {
+      run.Graph.messages = [
+        new HumanMessage('Verify activity-label Langfuse trace semantics'),
+      ];
+    }
+    const chainOptions = {
+      configurable: {
+        thread_id: sessionId,
+        user_id: 'activity-label-live-user',
+        requestBody: { parentMessageId: 'activity-label-live-parent' },
+      },
+    };
+
+    await run.generateActivityLabel({
+      provider: Providers.OPENAI,
+      agentId: 'activity-label-live-agent',
+      clientOptions: {
+        apiKey: process.env.OPENAI_API_KEY,
+        model,
+      },
+      entries: [
+        {
+          toolName: 'inspect_runtime',
+          toolInput: { target: 'trace-shape' },
+          toolOutput: { status: 'verified' },
+          status: 'success',
+        },
+      ],
+      chainOptions,
+    });
+    await run.generateActivityPhaseLabel({
+      provider: Providers.OPENAI,
+      clientOptions: {
+        apiKey: process.env.OPENAI_API_KEY,
+        model,
+      },
+      activities: [
+        { label: 'Inspected the activity-label trace shape' },
+        { label: 'Verified stable correlation metadata' },
+      ],
+      sourceRunId,
+      responseId: sourceRunId,
+      phaseIndex: 0,
+      chainOptions,
+    });
+
+    const { label, phase } = await findExportedTraces(sourceRunId, startedAt);
+    expect(label).toMatchObject({
+      name: 'LibreChat Activity Label',
+      metadata: {
+        sourceRunId,
+        responseId: sourceRunId,
+        activityIndex: 0,
+      },
+    });
+    expect(phase).toMatchObject({
+      name: 'LibreChat Activity Phase',
+      metadata: {
+        sourceRunId,
+        responseId: sourceRunId,
+        phaseIndex: 0,
+        activityCount: 2,
+      },
+    });
+
+    const labelGeneration = label.observations?.find(
+      (observation) => observation.type === 'GENERATION'
+    );
+    expect(labelGeneration).toMatchObject({
+      parentObservationId: null,
+      name: 'llm',
+      model: expect.stringContaining(model),
+    });
+    expect(labelGeneration?.usage?.total).toBeGreaterThan(0);
+
+    const phaseRoot = phase.observations?.find(
+      (observation) => observation.parentObservationId == null
+    );
+    expect(phaseRoot).toMatchObject({
+      type: 'CHAIN',
+      name: 'summarize-activity-phase',
+    });
+    const phaseGeneration = phase.observations?.find(
+      (observation) => observation.type === 'GENERATION'
+    );
+    expect(phaseGeneration).toMatchObject({
+      parentObservationId: phaseRoot?.id,
+      name: 'llm',
+      model: expect.stringContaining(model),
+    });
+    expect(phaseGeneration?.usage?.total).toBeGreaterThan(0);
+  });
+});

package/src/specs/activity-label-observability.test.ts

@@ -0,0 +1,167 @@
+import { CallbackHandler } from '@langfuse/langchain';
+import { propagateAttributes } from '@langfuse/tracing';
+import { AIMessage, HumanMessage } from '@langchain/core/messages';
+import { Providers } from '@/common';
+import { Run } from '@/run';
+
+const invoke = jest.fn();
+
+jest.mock('@/llm/init', () => ({
+  initializeModel: jest.fn(() => ({ invoke })),
+}));
+
+jest.mock('@langfuse/langchain', () => ({
+  CallbackHandler: jest.fn().mockImplementation(() => ({})),
+}));
+
+jest.mock('@langfuse/tracing', () => ({
+  ...jest.requireActual('@langfuse/tracing'),
+  propagateAttributes: jest.fn((_params, action: () => unknown) => action()),
+}));
+
+const MockedCallbackHandler = CallbackHandler as jest.MockedClass<
+  typeof CallbackHandler
+>;
+const MockedPropagateAttributes = propagateAttributes as jest.MockedFunction<
+  typeof propagateAttributes
+>;
+
+async function createRun(): Promise<Run<never>> {
+  const run = await Run.create({
+    runId: 'response-1',
+    graphConfig: {
+      type: 'standard',
+      agents: [
+        {
+          agentId: 'agent-1',
+          name: 'Changing Model Name',
+          provider: Providers.OPENAI,
+          clientOptions: { model: 'gpt-4.1-mini' },
+          tools: [],
+        },
+      ],
+    },
+  });
+  if (run.Graph != null) {
+    run.Graph.messages = [new HumanMessage('Why did the run fail?')];
+  }
+  return run;
+}
+
+describe('activity label observability', () => {
+  const originalEnv = process.env;
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    invoke.mockResolvedValue(new AIMessage('Resolved the failing run'));
+    process.env = {
+      ...originalEnv,
+      LANGFUSE_SECRET_KEY: 'sk-test',
+      LANGFUSE_PUBLIC_KEY: 'pk-test',
+      LANGFUSE_BASE_URL: 'https://langfuse.test',
+    };
+  });
+
+  afterEach(() => {
+    process.env = originalEnv;
+  });
+
+  it('uses a stable trace name and source-run metadata for batch labels', async () => {
+    const run = await createRun();
+
+    await run.generateActivityLabel({
+      provider: Providers.OPENAI,
+      entries: [
+        {
+          toolName: 'inspect_status',
+          toolInput: { id: 'one' },
+          toolOutput: { fixed: true },
+          status: 'success',
+        },
+      ],
+      chainOptions: {
+        configurable: {
+          thread_id: 'thread-1',
+          user_id: 'user-1',
+          requestBody: { parentMessageId: 'parent-1' },
+        },
+      },
+    });
+
+    expect(MockedCallbackHandler).toHaveBeenCalledTimes(1);
+    expect(MockedCallbackHandler.mock.calls[0][0]).toMatchObject({
+      traceMetadata: {
+        messageId: 'activity-label-response-1',
+        parentMessageId: 'parent-1',
+        agentId: 'agent-1',
+        agentName: 'Changing Model Name',
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        activityIndex: '0',
+      },
+      tags: ['librechat', 'activity-label'],
+    });
+    expect(MockedPropagateAttributes.mock.calls[0][0]).toMatchObject({
+      traceName: 'LibreChat Activity Label',
+      metadata: {
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        activityIndex: '0',
+      },
+    });
+    expect(invoke.mock.calls[0][1]).toMatchObject({
+      runName: 'LibreChat Activity Label',
+      tags: ['librechat', 'activity-label'],
+      metadata: {
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        activityIndex: 0,
+        parentMessageId: 'parent-1',
+        agentId: 'agent-1',
+        agentName: 'Changing Model Name',
+      },
+    });
+  });
+
+  it('uses a stable trace name for phase summaries', async () => {
+    const run = await createRun();
+
+    await run.generateActivityPhaseLabel({
+      provider: Providers.OPENAI,
+      activities: [
+        { label: 'Inspected session refresh behavior' },
+        { label: 'Fixed refresh token validation' },
+      ],
+      sourceRunId: 'response-1',
+      responseId: 'response-1',
+      phaseIndex: 0,
+      chainOptions: {
+        configurable: {
+          thread_id: 'thread-1',
+          user_id: 'user-1',
+          requestBody: { parentMessageId: 'parent-1' },
+        },
+      },
+    });
+
+    expect(MockedCallbackHandler).toHaveBeenCalledTimes(1);
+    expect(MockedCallbackHandler.mock.calls[0][0]).toMatchObject({
+      tags: ['librechat', 'activity-phase', 'agent-run-summary', 'agent'],
+      traceMetadata: {
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        phaseIndex: '0',
+        activityCount: '2',
+      },
+    });
+    expect(MockedPropagateAttributes.mock.calls[0][0]).toMatchObject({
+      traceName: 'LibreChat Activity Phase',
+      metadata: {
+        sourceRunId: 'response-1',
+        responseId: 'response-1',
+        phaseIndex: '0',
+        activityCount: '2',
+      },
+    });
+  });
+});

package/src/specs/ask-user-questions.live.test.ts

@@ -0,0 +1,185 @@
+/**
+ * Live proof that Anthropic can produce one tool call containing several
+ * questions, pause once, and continue from one keyed batch resolution.
+ *
+ * Run with:
+ * RUN_ASK_USER_QUESTIONS_LIVE_TESTS=1 ANTHROPIC_API_KEY=... npm test -- ask-user-questions.live.test.ts --runInBand
+ */
+import { config as dotenvConfig } from 'dotenv';
+dotenvConfig(
+  process.env.DOTENV_CONFIG_PATH != null
+    ? { path: process.env.DOTENV_CONFIG_PATH }
+    : undefined
+);
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { AIMessage, HumanMessage } from '@langchain/core/messages';
+import { MemorySaver } from '@langchain/langgraph';
+import { describe, expect, it, jest } from '@jest/globals';
+import type { BaseMessage } from '@langchain/core/messages';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type * as t from '@/types';
+import { Providers } from '@/common';
+import { askUserQuestions } from '@/hitl';
+import { Run } from '@/run';
+
+const shouldRunLive =
+  process.env.RUN_ASK_USER_QUESTIONS_LIVE_TESTS === '1' &&
+  process.env.ANTHROPIC_API_KEY != null &&
+  process.env.ANTHROPIC_API_KEY !== '';
+const describeIfLive = shouldRunLive ? describe : describe.skip;
+const modelName =
+  process.env.ANTHROPIC_BATCH_QUESTIONS_LIVE_MODEL ?? 'claude-sonnet-5';
+
+const questionSchema = z.object({
+  id: z.enum(['metric', 'window']),
+  header: z.string().max(20),
+  question: z.string(),
+  options: z
+    .array(z.object({ label: z.string().max(120), value: z.string() }))
+    .min(2)
+    .max(3),
+  multiSelect: z.boolean(),
+});
+const askUserQuestionsSchema = z.object({
+  questions: z.array(questionSchema).length(2),
+});
+type AskUserQuestionsInput = z.infer<typeof askUserQuestionsSchema>;
+
+const askTool = tool(
+  async (input: AskUserQuestionsInput, config) => {
+    const resolution = askUserQuestions(input, {
+      toolCallId: config.toolCall?.id,
+    });
+    return JSON.stringify(resolution);
+  },
+  {
+    name: 'ask_user_question',
+    description:
+      'Ask the user one to four related questions in one interaction. Put every question in this single tool call.',
+    schema: askUserQuestionsSchema,
+  }
+);
+
+type LiveStreamConfig = Partial<RunnableConfig> & {
+  version: 'v1' | 'v2';
+  streamMode: string;
+};
+
+function streamConfig(threadId: string): LiveStreamConfig {
+  return {
+    configurable: { thread_id: threadId },
+    streamMode: 'values',
+    version: 'v2',
+  };
+}
+
+function messageText(message: BaseMessage): string {
+  if (typeof message.content === 'string') {
+    return message.content;
+  }
+  if (!Array.isArray(message.content)) {
+    return '';
+  }
+  return message.content
+    .map((part) =>
+      typeof part === 'object' &&
+      'text' in part &&
+      typeof part.text === 'string'
+        ? part.text
+        : ''
+    )
+    .join('');
+}
+
+describeIfLive('askUserQuestions live Anthropic integration', () => {
+  jest.setTimeout(120_000);
+
+  it('uses one batched call and continues after one composite answer', async () => {
+    const nonce = `batch-questions-${Date.now()}`;
+    const saver = new MemorySaver();
+    const run = await Run.create<t.IState>({
+      runId: `${nonce}-run`,
+      graphConfig: {
+        type: 'standard',
+        agents: [
+          {
+            agentId: 'clarifier',
+            provider: Providers.ANTHROPIC,
+            clientOptions: {
+              modelName,
+              apiKey: process.env.ANTHROPIC_API_KEY,
+              maxTokens: 512,
+              streaming: true,
+            },
+            instructions: `You are testing a batched clarification tool.
+On the first turn, call ask_user_question exactly once. In that one call, ask exactly two questions:
+- id "metric": whether to analyze "workload" or "website"
+- id "window": whether to analyze "24h" or "7d"
+Do not emit two tool calls and do not answer in prose before the tool result.
+After the tool returns, reply exactly: LIVE_BATCH_OK metric=<metric>; window=<window>`,
+            maxContextTokens: 8000,
+            graphTools: [askTool],
+          },
+        ],
+        compileOptions: { checkpointer: saver },
+      },
+      returnContent: true,
+      skipCleanup: true,
+      interruptingToolNames: ['ask_user_question'],
+    });
+    const config = streamConfig(`${nonce}-thread`);
+
+    await run.processStream(
+      {
+        messages: [
+          new HumanMessage(
+            'Clarify both dimensions before doing any analysis.'
+          ),
+        ],
+      },
+      config
+    );
+
+    const pending = run.getInterrupt();
+    expect(pending?.payload.type).toBe('ask_user_question');
+    if (pending?.payload.type !== 'ask_user_question') {
+      throw new Error('expected ask_user_question interrupt');
+    }
+    expect(pending.payload.questions).toHaveLength(2);
+    expect(pending.payload.questions?.map(({ id }) => id)).toEqual([
+      'metric',
+      'window',
+    ]);
+
+    await run.resume<t.AskUserQuestionsResolution>(
+      { answers: { metric: 'workload', window: '7d' } },
+      config
+    );
+
+    expect(run.getInterrupt()).toBeUndefined();
+    const messages = run.getRunMessages() ?? [];
+    const askCalls = messages.flatMap((message) => {
+      if (message.getType() !== 'ai') {
+        return [];
+      }
+      return ((message as AIMessage).tool_calls ?? []).filter(
+        ({ name }) => name === 'ask_user_question'
+      );
+    });
+    expect(askCalls).toHaveLength(1);
+    expect(askCalls[0].args).toMatchObject({
+      questions: expect.arrayContaining([
+        expect.objectContaining({ id: 'metric' }),
+        expect.objectContaining({ id: 'window' }),
+      ]),
+    });
+
+    const finalText = messages
+      .filter((message) => message.getType() === 'ai')
+      .map(messageText)
+      .join('\n');
+    expect(finalText).toContain('LIVE_BATCH_OK metric=workload; window=7d');
+  });
+});