@librechat/agents 3.4.3 → 3.4.5

package/src/specs/activity-phase-label.test.ts

@@ -0,0 +1,306 @@
+import { AIMessage, HumanMessage } from '@langchain/core/messages';
+import { LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT } from '@/langfuseToolOutputTracing';
+import { Providers } from '@/common';
+import { Run } from '@/run';
+
+const invoke = jest.fn();
+
+jest.mock('@/llm/init', () => ({
+  initializeModel: jest.fn(() => ({ invoke })),
+}));
+
+async function createRun(): Promise<Run<never>> {
+  return Run.create({
+    runId: 'phase-run',
+    graphConfig: {
+      type: 'standard',
+      agents: [
+        {
+          agentId: 'agent-1',
+          name: 'Phase Agent',
+          provider: Providers.OPENAI,
+          clientOptions: { model: 'gpt-4.1-mini' },
+          tools: [],
+        },
+      ],
+    },
+  });
+}
+
+describe('generateActivityPhaseLabel', () => {
+  beforeEach(() => {
+    invoke.mockReset();
+    invoke.mockResolvedValue(
+      new AIMessage('"Fixed session refresh handling and verified auth tests."')
+    );
+  });
+
+  it('does not spend a model call on one logical activity', async () => {
+    const run = await createRun();
+
+    await expect(
+      run.generateActivityPhaseLabel({
+        provider: Providers.OPENAI,
+        activities: [{ label: 'Inspected session refresh middleware' }],
+      })
+    ).resolves.toEqual({});
+    expect(invoke).not.toHaveBeenCalled();
+  });
+
+  it('summarizes two activities and normalizes the persisted row', async () => {
+    const run = await createRun();
+    if (run.Graph != null) {
+      run.Graph.messages = [
+        new HumanMessage({
+          content: [
+            {
+              type: 'input_text',
+              text: 'Why is session refresh failing?',
+            } as never,
+          ],
+        }),
+        new HumanMessage({
+          content: 'Internal routing instructions',
+          additional_kwargs: {
+            role: 'user',
+            isMeta: true,
+            source: 'routing',
+          },
+        }),
+      ];
+    }
+    const handleChainStart = jest.fn();
+    const handleChainEnd = jest.fn();
+
+    await expect(
+      run.generateActivityPhaseLabel({
+        provider: Providers.OPENAI,
+        activities: [
+          { label: 'Inspected session refresh middleware' },
+          { label: 'Fixed refresh token validation' },
+        ],
+        assistantContext: ['I am checking the auth path.'],
+        closingTextPhase: 'final_answer',
+        chainOptions: {
+          callbacks: [{ handleChainStart, handleChainEnd }],
+          configurable: {
+            requestBody: { parentMessageId: 'parent-message-1' },
+          },
+        },
+      })
+    ).resolves.toEqual({
+      label: 'Fixed session refresh handling and verified auth tests',
+    });
+    expect(invoke).toHaveBeenCalledTimes(1);
+    const messages = invoke.mock.calls[0][0] as AIMessage[];
+    expect(String(messages[1].content)).toContain(
+      'Inspected session refresh middleware'
+    );
+    expect(String(messages[1].content)).toContain(
+      'Fixed refresh token validation'
+    );
+    expect(String(messages[1].content)).not.toContain(
+      'Why is session refresh failing?'
+    );
+    const modelConfig = invoke.mock.calls[0][1] as {
+      callbacks?: { getParentRunId?: () => string | undefined };
+      tags?: string[];
+      metadata?: Record<string, unknown>;
+    };
+    expect(modelConfig.callbacks?.getParentRunId?.()).toEqual(
+      expect.any(String)
+    );
+    expect(modelConfig.tags).toEqual(
+      expect.arrayContaining(['activity-phase', 'agent'])
+    );
+    expect(modelConfig.metadata).toEqual(
+      expect.objectContaining({
+        agentId: 'agent-1',
+        agentName: 'Phase Agent',
+        parentMessageId: 'parent-message-1',
+      })
+    );
+    expect(handleChainStart.mock.calls[0]?.[1]).toEqual(
+      expect.objectContaining({
+        messages: expect.arrayContaining([
+          expect.objectContaining({
+            content: 'Why is session refresh failing?',
+          }),
+        ]),
+      })
+    );
+    expect(JSON.stringify(handleChainStart.mock.calls[0]?.[1])).not.toContain(
+      'Internal routing instructions'
+    );
+    expect(handleChainEnd.mock.calls[0]?.[0]).toEqual(
+      expect.objectContaining({
+        messages: expect.arrayContaining([
+          expect.objectContaining({
+            content: 'Fixed session refresh handling and verified auth tests',
+          }),
+        ]),
+      })
+    );
+  });
+
+  it('does not call the model when retained activities have no evidence', async () => {
+    const run = await createRun();
+
+    await expect(
+      run.generateActivityPhaseLabel({
+        provider: Providers.OPENAI,
+        activities: [
+          ...Array.from({ length: 12 }, () => ({ status: 'success' as const })),
+          { label: 'Evidence beyond the prompt cap' },
+        ],
+      })
+    ).resolves.toEqual({});
+    expect(invoke).not.toHaveBeenCalled();
+  });
+
+  it('preserves the phase parent when retrying without a failing callback', async () => {
+    invoke.mockRejectedValueOnce(new Error('event stream callback failed'));
+    const run = await createRun();
+
+    await expect(
+      run.generateActivityPhaseLabel({
+        provider: Providers.OPENAI,
+        activities: [
+          { label: 'Inspected session refresh middleware' },
+          { label: 'Fixed refresh token validation' },
+        ],
+        chainOptions: {
+          callbacks: [{ handleChainStart: jest.fn() }],
+        },
+      })
+    ).resolves.toEqual({
+      label: 'Fixed session refresh handling and verified auth tests',
+    });
+
+    expect(invoke).toHaveBeenCalledTimes(2);
+    const firstCallbacks = invoke.mock.calls[0][1].callbacks as {
+      getParentRunId: () => string | undefined;
+      handlers: unknown[];
+    };
+    const retryCallbacks = invoke.mock.calls[1][1].callbacks as {
+      getParentRunId: () => string | undefined;
+      handlers: unknown[];
+    };
+    expect(firstCallbacks.getParentRunId()).toEqual(expect.any(String));
+    expect(retryCallbacks.getParentRunId()).toBe(
+      firstCallbacks.getParentRunId()
+    );
+    expect(retryCallbacks.handlers).toHaveLength(0);
+  });
+
+  it('applies every agent redaction policy when any activity is unattributed', async () => {
+    const run = await Run.create({
+      runId: 'mixed-attribution-phase-run',
+      graphConfig: {
+        type: 'multi-agent',
+        agents: [
+          {
+            agentId: 'agent-1',
+            provider: Providers.OPENAI,
+            clientOptions: { model: 'gpt-4.1-mini' },
+            tools: [],
+          },
+          {
+            agentId: 'agent-2',
+            provider: Providers.OPENAI,
+            clientOptions: { model: 'gpt-4.1-mini' },
+            tools: [],
+            langfuse: {
+              toolOutputTracing: { redactedToolNames: ['secret_tool'] },
+            },
+          },
+        ],
+        edges: [],
+      },
+    });
+
+    await run.generateActivityPhaseLabel({
+      provider: Providers.OPENAI,
+      activities: [
+        { agentId: 'agent-1', label: 'Inspected public session behavior' },
+        {
+          entries: [
+            {
+              toolName: 'secret_tool',
+              toolInput: { key: 'public-key' },
+              toolOutput: 'STRICT_AGENT_SECRET',
+              status: 'success',
+            },
+          ],
+        },
+      ],
+    });
+
+    const messages = invoke.mock.calls[0][0] as AIMessage[];
+    expect(String(messages[1].content)).not.toContain('STRICT_AGENT_SECRET');
+    expect(String(messages[1].content)).toContain(
+      LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT
+    );
+  });
+
+  it('applies every policy when omitted activities have no complete agent list', async () => {
+    const run = await Run.create({
+      runId: 'omitted-attribution-phase-run',
+      graphConfig: {
+        type: 'multi-agent',
+        agents: [
+          {
+            agentId: 'agent-1',
+            provider: Providers.OPENAI,
+            clientOptions: { model: 'gpt-4.1-mini' },
+            tools: [],
+          },
+          {
+            agentId: 'agent-2',
+            provider: Providers.OPENAI,
+            clientOptions: { model: 'gpt-4.1-mini' },
+            tools: [],
+            langfuse: {
+              toolOutputTracing: { redactedToolNames: ['secret_tool'] },
+            },
+          },
+        ],
+        edges: [],
+      },
+    });
+
+    await run.generateActivityPhaseLabel({
+      provider: Providers.OPENAI,
+      activities: [
+        {
+          agentId: 'agent-1',
+          entries: [
+            {
+              toolName: 'public_lookup',
+              toolInput: { id: 'one' },
+              toolOutput: 'public-one',
+              status: 'success',
+            },
+          ],
+        },
+        {
+          agentId: 'agent-1',
+          entries: [
+            {
+              toolName: 'public_lookup',
+              toolInput: { id: 'two' },
+              toolOutput: 'public-two',
+              status: 'success',
+            },
+          ],
+        },
+      ],
+      totalActivityCount: 3,
+      assistantContext: ['OMITTED_AGENT_SECRET'],
+    });
+
+    const messages = invoke.mock.calls[0][0] as AIMessage[];
+    expect(String(messages[1].content)).not.toContain('OMITTED_AGENT_SECRET');
+    expect(String(messages[1].content)).toContain('public-one');
+  });
+});

package/src/specs/ask-user-questions.live.test.ts

@@ -0,0 +1,185 @@
+/**
+ * Live proof that Anthropic can produce one tool call containing several
+ * questions, pause once, and continue from one keyed batch resolution.
+ *
+ * Run with:
+ * RUN_ASK_USER_QUESTIONS_LIVE_TESTS=1 ANTHROPIC_API_KEY=... npm test -- ask-user-questions.live.test.ts --runInBand
+ */
+import { config as dotenvConfig } from 'dotenv';
+dotenvConfig(
+  process.env.DOTENV_CONFIG_PATH != null
+    ? { path: process.env.DOTENV_CONFIG_PATH }
+    : undefined
+);
+
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { AIMessage, HumanMessage } from '@langchain/core/messages';
+import { MemorySaver } from '@langchain/langgraph';
+import { describe, expect, it, jest } from '@jest/globals';
+import type { BaseMessage } from '@langchain/core/messages';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type * as t from '@/types';
+import { Providers } from '@/common';
+import { askUserQuestions } from '@/hitl';
+import { Run } from '@/run';
+
+const shouldRunLive =
+  process.env.RUN_ASK_USER_QUESTIONS_LIVE_TESTS === '1' &&
+  process.env.ANTHROPIC_API_KEY != null &&
+  process.env.ANTHROPIC_API_KEY !== '';
+const describeIfLive = shouldRunLive ? describe : describe.skip;
+const modelName =
+  process.env.ANTHROPIC_BATCH_QUESTIONS_LIVE_MODEL ?? 'claude-sonnet-5';
+
+const questionSchema = z.object({
+  id: z.enum(['metric', 'window']),
+  header: z.string().max(20),
+  question: z.string(),
+  options: z
+    .array(z.object({ label: z.string().max(120), value: z.string() }))
+    .min(2)
+    .max(3),
+  multiSelect: z.boolean(),
+});
+const askUserQuestionsSchema = z.object({
+  questions: z.array(questionSchema).length(2),
+});
+type AskUserQuestionsInput = z.infer<typeof askUserQuestionsSchema>;
+
+const askTool = tool(
+  async (input: AskUserQuestionsInput, config) => {
+    const resolution = askUserQuestions(input, {
+      toolCallId: config.toolCall?.id,
+    });
+    return JSON.stringify(resolution);
+  },
+  {
+    name: 'ask_user_question',
+    description:
+      'Ask the user one to four related questions in one interaction. Put every question in this single tool call.',
+    schema: askUserQuestionsSchema,
+  }
+);
+
+type LiveStreamConfig = Partial<RunnableConfig> & {
+  version: 'v1' | 'v2';
+  streamMode: string;
+};
+
+function streamConfig(threadId: string): LiveStreamConfig {
+  return {
+    configurable: { thread_id: threadId },
+    streamMode: 'values',
+    version: 'v2',
+  };
+}
+
+function messageText(message: BaseMessage): string {
+  if (typeof message.content === 'string') {
+    return message.content;
+  }
+  if (!Array.isArray(message.content)) {
+    return '';
+  }
+  return message.content
+    .map((part) =>
+      typeof part === 'object' &&
+      'text' in part &&
+      typeof part.text === 'string'
+        ? part.text
+        : ''
+    )
+    .join('');
+}
+
+describeIfLive('askUserQuestions live Anthropic integration', () => {
+  jest.setTimeout(120_000);
+
+  it('uses one batched call and continues after one composite answer', async () => {
+    const nonce = `batch-questions-${Date.now()}`;
+    const saver = new MemorySaver();
+    const run = await Run.create<t.IState>({
+      runId: `${nonce}-run`,
+      graphConfig: {
+        type: 'standard',
+        agents: [
+          {
+            agentId: 'clarifier',
+            provider: Providers.ANTHROPIC,
+            clientOptions: {
+              modelName,
+              apiKey: process.env.ANTHROPIC_API_KEY,
+              maxTokens: 512,
+              streaming: true,
+            },
+            instructions: `You are testing a batched clarification tool.
+On the first turn, call ask_user_question exactly once. In that one call, ask exactly two questions:
+- id "metric": whether to analyze "workload" or "website"
+- id "window": whether to analyze "24h" or "7d"
+Do not emit two tool calls and do not answer in prose before the tool result.
+After the tool returns, reply exactly: LIVE_BATCH_OK metric=<metric>; window=<window>`,
+            maxContextTokens: 8000,
+            graphTools: [askTool],
+          },
+        ],
+        compileOptions: { checkpointer: saver },
+      },
+      returnContent: true,
+      skipCleanup: true,
+      interruptingToolNames: ['ask_user_question'],
+    });
+    const config = streamConfig(`${nonce}-thread`);
+
+    await run.processStream(
+      {
+        messages: [
+          new HumanMessage(
+            'Clarify both dimensions before doing any analysis.'
+          ),
+        ],
+      },
+      config
+    );
+
+    const pending = run.getInterrupt();
+    expect(pending?.payload.type).toBe('ask_user_question');
+    if (pending?.payload.type !== 'ask_user_question') {
+      throw new Error('expected ask_user_question interrupt');
+    }
+    expect(pending.payload.questions).toHaveLength(2);
+    expect(pending.payload.questions?.map(({ id }) => id)).toEqual([
+      'metric',
+      'window',
+    ]);
+
+    await run.resume<t.AskUserQuestionsResolution>(
+      { answers: { metric: 'workload', window: '7d' } },
+      config
+    );
+
+    expect(run.getInterrupt()).toBeUndefined();
+    const messages = run.getRunMessages() ?? [];
+    const askCalls = messages.flatMap((message) => {
+      if (message.getType() !== 'ai') {
+        return [];
+      }
+      return ((message as AIMessage).tool_calls ?? []).filter(
+        ({ name }) => name === 'ask_user_question'
+      );
+    });
+    expect(askCalls).toHaveLength(1);
+    expect(askCalls[0].args).toMatchObject({
+      questions: expect.arrayContaining([
+        expect.objectContaining({ id: 'metric' }),
+        expect.objectContaining({ id: 'window' }),
+      ]),
+    });
+
+    const finalText = messages
+      .filter((message) => message.getType() === 'ai')
+      .map(messageText)
+      .join('\n');
+    expect(finalText).toContain('LIVE_BATCH_OK metric=workload; window=7d');
+  });
+});