@librechat/agents 3.4.6 → 3.5.0

package/src/specs/reasoning.test.ts

@@ -1,201 +0,0 @@
-/* eslint-disable no-console */
-
-// src/scripts/cli.test.ts
-import { config } from 'dotenv';
-config();
-import {
-  HumanMessage,
-  BaseMessage,
-  MessageContentText,
-} from '@langchain/core/messages';
-import type { RunnableConfig } from '@langchain/core/runnables';
-import type * as t from '@/types';
-import { capitalizeFirstLetter } from './spec.utils';
-import { createContentAggregator } from '@/stream';
-import { GraphEvents, Providers } from '@/common';
-import { getLLMConfig } from '@/utils/llmConfig';
-import { getArgs } from '@/scripts/args';
-import { Run } from '@/run';
-
-const reasoningText = `<think>
-Okay, the user is Jo from New York. I should start by greeting them by name. Let's keep it friendly and open-ended. Maybe mention the weather in New York to make it personal. Then offer help with something specific like plans or questions. Need to keep it concise and welcoming. Check for any typos. Alright, that should work.
-</think>
-Hi Jo! 🌆 How's everything in New York today? Whether you need recommendations for the city, help with a task, or just want to chat, I'm here for it. What's on your mind? 😊`;
-
-const provider = 'Reasoning LLM';
-describe(`${capitalizeFirstLetter(provider)} Streaming Tests`, () => {
-  jest.setTimeout(30000);
-  let run: Run<t.IState>;
-  let contentParts: t.MessageContentComplex[];
-  let conversationHistory: BaseMessage[];
-  let aggregateContent: t.ContentAggregator;
-  let runSteps: Set<string>;
-
-  const config: Partial<RunnableConfig> & {
-    version: 'v1' | 'v2';
-    run_id?: string;
-    streamMode: string;
-  } = {
-    configurable: {
-      thread_id: 'conversation-num-1',
-    },
-    streamMode: 'values',
-    version: 'v2' as const,
-    callbacks: [
-      {
-        async handleCustomEvent(event, data, metadata): Promise<void> {
-          if (event !== GraphEvents.ON_MESSAGE_DELTA) {
-            return;
-          }
-          const messageDeltaData = data as t.MessageDeltaEvent;
-
-          // Wait until we see the run step (with timeout for safety)
-          const maxAttempts = 50; // 5 seconds total
-          let attempts = 0;
-          while (!runSteps.has(messageDeltaData.id) && attempts < maxAttempts) {
-            await new Promise((resolve) => setTimeout(resolve, 100));
-            attempts++;
-          }
-
-          if (!runSteps.has(messageDeltaData.id)) {
-            console.warn(
-              `Timeout waiting for run step: ${messageDeltaData.id}`
-            );
-          }
-
-          onMessageDeltaSpy(event, data, metadata, run.Graph);
-          aggregateContent({ event, data: messageDeltaData });
-        },
-      },
-    ],
-  };
-
-  beforeEach(async () => {
-    conversationHistory = [];
-    const { contentParts: parts, aggregateContent: ac } =
-      createContentAggregator();
-    aggregateContent = ac;
-    runSteps = new Set();
-    contentParts = parts as t.MessageContentComplex[];
-  });
-
-  afterEach(() => {
-    runSteps.clear();
-  });
-
-  const onReasoningDeltaSpy = jest.fn();
-  const onMessageDeltaSpy = jest.fn();
-  const onRunStepSpy = jest.fn();
-
-  afterAll(() => {
-    onReasoningDeltaSpy.mockReset();
-    onMessageDeltaSpy.mockReset();
-    onRunStepSpy.mockReset();
-  });
-
-  const setupCustomHandlers = (): Record<
-    string | GraphEvents,
-    t.EventHandler
-  > => ({
-    [GraphEvents.ON_RUN_STEP_COMPLETED]: {
-      handle: (
-        event: GraphEvents.ON_RUN_STEP_COMPLETED,
-        data: t.StreamEventData
-      ): void => {
-        aggregateContent({
-          event,
-          data: data as unknown as { result: t.ToolEndEvent },
-        });
-      },
-    },
-    [GraphEvents.ON_RUN_STEP]: {
-      handle: (
-        event: GraphEvents.ON_RUN_STEP,
-        data: t.StreamEventData,
-        metadata,
-        graph
-      ): void => {
-        const runStepData = data as t.RunStep;
-        runSteps.add(runStepData.id);
-
-        onRunStepSpy(event, runStepData, metadata, graph);
-        aggregateContent({ event, data: runStepData });
-      },
-    },
-    [GraphEvents.ON_RUN_STEP_DELTA]: {
-      handle: (
-        event: GraphEvents.ON_RUN_STEP_DELTA,
-        data: t.StreamEventData
-      ): void => {
-        aggregateContent({ event, data: data as t.RunStepDeltaEvent });
-      },
-    },
-    [GraphEvents.ON_REASONING_DELTA]: {
-      handle: (
-        event: GraphEvents.ON_REASONING_DELTA,
-        data: t.StreamEventData,
-        metadata,
-        graph
-      ): void => {
-        onReasoningDeltaSpy(event, data, metadata, graph);
-        aggregateContent({ event, data: data as t.ReasoningDeltaEvent });
-      },
-    },
-  });
-
-  test(`${capitalizeFirstLetter(provider)}: should process a simple reasoning message`, async () => {
-    const { userName, location } = await getArgs();
-    const llmConfig = getLLMConfig(Providers.OPENAI);
-    const customHandlers = setupCustomHandlers();
-
-    run = await Run.create<t.IState>({
-      runId: 'test-run-id',
-      graphConfig: {
-        type: 'standard',
-        llmConfig,
-        instructions:
-          'You are a friendly AI assistant. Always address the user by their name.',
-        additional_instructions: `The user's name is ${userName} and they are located in ${location}.`,
-      },
-      returnContent: true,
-      skipCleanup: true,
-      customHandlers,
-    });
-
-    run.Graph?.overrideTestModel([reasoningText], 2);
-
-    const userMessage = 'hi';
-    conversationHistory.push(new HumanMessage(userMessage));
-
-    const inputs = {
-      messages: conversationHistory,
-    };
-
-    await run.processStream(inputs, config);
-    expect(contentParts).toBeDefined();
-    expect(contentParts.length).toBe(2);
-    const reasoningContent = reasoningText.match(/<think>(.*)<\/think>/s)?.[0];
-    const content = reasoningText.split(/<\/think>/)[1];
-    expect((contentParts[0] as t.ReasoningContentText).think).toBe(
-      reasoningContent
-    );
-    expect((contentParts[1] as MessageContentText).text).toBe(content);
-
-    const finalMessages = run.getRunMessages();
-    expect(finalMessages).toBeDefined();
-    conversationHistory.push(...(finalMessages ?? []));
-    expect(conversationHistory.length).toBeGreaterThan(1);
-
-    expect(onMessageDeltaSpy).toHaveBeenCalled();
-    expect(onMessageDeltaSpy.mock.calls.length).toBeGreaterThan(1);
-    expect(onMessageDeltaSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
-
-    expect(onReasoningDeltaSpy).toHaveBeenCalled();
-    expect(onReasoningDeltaSpy.mock.calls.length).toBeGreaterThan(1);
-    expect(onReasoningDeltaSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
-
-    expect(onRunStepSpy).toHaveBeenCalled();
-    expect(onRunStepSpy.mock.calls.length).toBeGreaterThan(0);
-    expect(onRunStepSpy.mock.calls[0][3]).toBeDefined(); // Graph exists
-  });
-});

package/src/specs/run-step-timestamps.test.ts

@@ -1,414 +0,0 @@
-import { z } from 'zod';
-import { tool } from '@langchain/core/tools';
-import { HumanMessage } from '@langchain/core/messages';
-import { MemorySaver, Command } from '@langchain/langgraph';
-import type { ToolCall } from '@langchain/core/messages/tool';
-import type * as t from '@/types';
-import { GraphEvents, StepTypes, Providers } from '@/common';
-import { FakeChatModel } from '@/llm/fake';
-import { askUserQuestion } from '@/hitl';
-import { Run } from '@/run';
-
-const askTool = tool(
-  async (input) => {
-    const { answer } = askUserQuestion(input as { question: string });
-    return answer;
-  },
-  {
-    name: 'ask_user_question',
-    description:
-      'Ask the user a clarifying question and wait for their answer.',
-    schema: z.object({ question: z.string() }),
-  }
-);
-
-const llmConfig: t.LLMConfig = {
-  provider: Providers.OPENAI,
-  streaming: true,
-  streamUsage: false,
-};
-
-const echoTool = tool(
-  async ({ input }: { input: string }) => `echo: ${input}`,
-  {
-    name: 'echoTool',
-    description: 'Echoes the input back',
-    schema: z.object({ input: z.string() }),
-  }
-);
-
-const toolCalls: ToolCall[] = [
-  {
-    name: 'echoTool',
-    args: { input: 'ping' },
-    id: 'call_timestamps_1',
-    type: 'tool_call',
-  },
-];
-
-const createStreamConfig = (
-  threadId: string,
-  signal?: AbortSignal
-): t.RunStreamConfig => ({
-  configurable: { thread_id: threadId },
-  version: 'v2',
-  ...(signal != null ? { signal } : {}),
-});
-
-type RecordedEvent = {
-  event: string;
-  data: unknown;
-  /** Lifecycle fields captured at receipt — dispatched RunStep objects are
-   *  shared references that later closes mutate in place. */
-  statusAtReceipt?: t.RunStepStatus;
-};
-
-function createRecorder(): {
-  sequence: RecordedEvent[];
-  handlers: Record<string, t.EventHandler>;
-  } {
-  const sequence: RecordedEvent[] = [];
-  const record = (event: string): t.EventHandler => ({
-    handle: (_event, data): void => {
-      sequence.push({
-        event,
-        data,
-        ...(event === GraphEvents.ON_RUN_STEP
-          ? { statusAtReceipt: (data as t.RunStep).status }
-          : {}),
-      });
-    },
-  });
-  return {
-    sequence,
-    handlers: {
-      [GraphEvents.ON_RUN_STEP]: record(GraphEvents.ON_RUN_STEP),
-      [GraphEvents.ON_RUN_STEP_COMPLETED]: record(
-        GraphEvents.ON_RUN_STEP_COMPLETED
-      ),
-      [GraphEvents.ON_RUN_STEP_CLOSED]: record(GraphEvents.ON_RUN_STEP_CLOSED),
-    },
-  };
-}
-
-function getSteps(sequence: RecordedEvent[]): t.RunStep[] {
-  return sequence
-    .filter((entry) => entry.event === GraphEvents.ON_RUN_STEP)
-    .map((entry) => entry.data as t.RunStep);
-}
-
-function getClosed(sequence: RecordedEvent[]): t.RunStepClosedEvent[] {
-  return sequence
-    .filter((entry) => entry.event === GraphEvents.ON_RUN_STEP_CLOSED)
-    .map((entry) => entry.data as t.RunStepClosedEvent);
-}
-
-describe('run step timestamps', () => {
-  jest.setTimeout(20000);
-
-  it('stamps starts and closes every step exactly once on a natural finish', async () => {
-    const { sequence, handlers } = createRecorder();
-    const run = await Run.create<t.IState>({
-      runId: 'test-run-step-timestamps',
-      graphConfig: {
-        type: 'standard',
-        llmConfig,
-        tools: [echoTool],
-        instructions: 'You are a helpful assistant.',
-      },
-      returnContent: true,
-      skipCleanup: true,
-      customHandlers: handlers,
-    });
-
-    run.Graph?.overrideTestModel(
-      ['Let me call the tool', 'The tool answered'],
-      2,
-      toolCalls
-    );
-    const started = Date.now();
-    await run.processStream(
-      { messages: [new HumanMessage('hello')] },
-      createStreamConfig('run-step-timestamps')
-    );
-
-    const steps = getSteps(sequence);
-    const closed = getClosed(sequence);
-    expect(steps.length).toBeGreaterThanOrEqual(3);
-    const stepReceipts = sequence.filter(
-      (entry) => entry.event === GraphEvents.ON_RUN_STEP
-    );
-    for (const receipt of stepReceipts) {
-      expect(receipt.statusAtReceipt).toBe('in_progress');
-    }
-    for (const step of steps) {
-      expect(typeof step.created_at).toBe('number');
-      expect(step.created_at as number).toBeGreaterThanOrEqual(started);
-    }
-
-    const stepIds = steps.map((step) => step.id);
-    expect(closed.map((event) => event.id).sort()).toEqual([...stepIds].sort());
-    for (const event of closed) {
-      expect(event.status).toBe('completed');
-      expect(event.closed_at).toBeGreaterThanOrEqual(
-        event.created_at as number
-      );
-    }
-
-    const toolStep = steps.find((step) => step.type === StepTypes.TOOL_CALLS);
-    expect(toolStep).toBeDefined();
-    const toolStepId = (toolStep as t.RunStep).id;
-    const toolCompletedIndex = sequence.findIndex(
-      (entry) =>
-        entry.event === GraphEvents.ON_RUN_STEP_COMPLETED &&
-        (entry.data as { result?: { id?: string } }).result?.id === toolStepId
-    );
-    const toolClosedIndex = sequence.findIndex(
-      (entry) =>
-        entry.event === GraphEvents.ON_RUN_STEP_CLOSED &&
-        (entry.data as t.RunStepClosedEvent).id === toolStepId
-    );
-    expect(toolCompletedIndex).toBeGreaterThanOrEqual(0);
-    expect(toolClosedIndex).toBeGreaterThan(toolCompletedIndex);
-
-    const firstMessageStepId = steps[0].id;
-    const toolStepIndex = sequence.findIndex(
-      (entry) =>
-        entry.event === GraphEvents.ON_RUN_STEP &&
-        (entry.data as t.RunStep).id === toolStepId
-    );
-    const firstMessageClosedIndex = sequence.findIndex(
-      (entry) =>
-        entry.event === GraphEvents.ON_RUN_STEP_CLOSED &&
-        (entry.data as t.RunStepClosedEvent).id === firstMessageStepId
-    );
-    expect(firstMessageClosedIndex).toBeGreaterThanOrEqual(0);
-    expect(firstMessageClosedIndex).toBeLessThan(toolStepIndex);
-
-    const completedEvent = sequence
-      .filter((entry) => entry.event === GraphEvents.ON_RUN_STEP_COMPLETED)
-      .map((entry) => (entry.data as { result: t.ToolCompleteEvent }).result)
-      .find((result) => result.id === toolStepId);
-    expect(typeof completedEvent?.completed_at).toBe('number');
-
-    for (const step of run.Graph?.contentData ?? []) {
-      expect(step.status).toBe('completed');
-      expect(typeof step.completed_at).toBe('number');
-    }
-  });
-
-  it('changes nothing for hosts that only register the legacy handlers', async () => {
-    const received: string[] = [];
-    const legacyHandler = (event: string): t.EventHandler => ({
-      handle: (): void => {
-        received.push(event);
-      },
-    });
-    const run = await Run.create<t.IState>({
-      runId: 'test-run-step-timestamps-legacy',
-      graphConfig: {
-        type: 'standard',
-        llmConfig,
-        tools: [echoTool],
-        instructions: 'You are a helpful assistant.',
-      },
-      returnContent: true,
-      skipCleanup: true,
-      customHandlers: {
-        [GraphEvents.ON_RUN_STEP]: legacyHandler(GraphEvents.ON_RUN_STEP),
-        [GraphEvents.ON_RUN_STEP_COMPLETED]: legacyHandler(
-          GraphEvents.ON_RUN_STEP_COMPLETED
-        ),
-        [GraphEvents.ON_MESSAGE_DELTA]: legacyHandler(
-          GraphEvents.ON_MESSAGE_DELTA
-        ),
-      },
-    });
-
-    run.Graph?.overrideTestModel(
-      ['Let me call the tool', 'The tool answered'],
-      2,
-      toolCalls
-    );
-    await run.processStream(
-      { messages: [new HumanMessage('hello')] },
-      createStreamConfig('run-step-timestamps-legacy')
-    );
-
-    expect(new Set(received)).toEqual(
-      new Set([
-        GraphEvents.ON_RUN_STEP,
-        GraphEvents.ON_RUN_STEP_COMPLETED,
-        GraphEvents.ON_MESSAGE_DELTA,
-      ])
-    );
-    for (const step of run.Graph?.contentData ?? []) {
-      expect(step.status).toBe('completed');
-    }
-  });
-
-  it('sweeps unfinished steps as cancelled when the caller aborts mid-stream', async () => {
-    const { sequence, handlers } = createRecorder();
-    const controller = new AbortController();
-    const run = await Run.create<t.IState>({
-      runId: 'test-run-step-timestamps-abort',
-      graphConfig: {
-        type: 'standard',
-        llmConfig,
-        instructions: 'You are a helpful assistant.',
-      },
-      returnContent: true,
-      skipCleanup: true,
-      customHandlers: {
-        ...handlers,
-        [GraphEvents.ON_MESSAGE_DELTA]: {
-          handle: (): void => {
-            controller.abort();
-          },
-        },
-      },
-    });
-
-    run.Graph?.overrideTestModel(
-      ['This response streams slowly enough to abort in flight'],
-      25
-    );
-    await expect(
-      run.processStream(
-        { messages: [new HumanMessage('hello')] },
-        createStreamConfig('run-step-timestamps-abort', controller.signal)
-      )
-    ).rejects.toThrow();
-
-    const closed = getClosed(sequence);
-    expect(closed.length).toBeGreaterThanOrEqual(1);
-    expect(closed.every((event) => event.status === 'cancelled')).toBe(true);
-    for (const step of run.Graph?.contentData ?? []) {
-      expect(step.status).toBe('cancelled');
-      expect(typeof step.cancelled_at).toBe('number');
-      expect(step.cancelled_at as number).toBeGreaterThanOrEqual(
-        step.created_at as number
-      );
-    }
-  });
-
-  it('sweeps as failed when an AbortError surfaces with no signal aborted', async () => {
-    const { sequence, handlers } = createRecorder();
-    const run = await Run.create<t.IState>({
-      runId: 'test-run-step-timestamps-uncorroborated-abort',
-      graphConfig: {
-        type: 'standard',
-        llmConfig,
-        instructions: 'You are a helpful assistant.',
-      },
-      returnContent: true,
-      skipCleanup: true,
-      customHandlers: {
-        ...handlers,
-        [GraphEvents.ON_MESSAGE_DELTA]: {
-          handle: (): void => {
-            /** A provider/host rejection that merely borrows the name — no
-             *  caller or construction signal was ever aborted. */
-            const error = new Error('provider stream aborted internally');
-            error.name = 'AbortError';
-            throw error;
-          },
-        },
-      },
-    });
-
-    run.Graph?.overrideTestModel(['some streamed content'], 2);
-    await expect(
-      run.processStream(
-        { messages: [new HumanMessage('hello')] },
-        createStreamConfig('run-step-timestamps-uncorroborated-abort')
-      )
-    ).rejects.toThrow();
-
-    const closed = getClosed(sequence);
-    expect(closed.length).toBeGreaterThanOrEqual(1);
-    expect(closed.every((event) => event.status === 'failed')).toBe(true);
-    for (const step of run.Graph?.contentData ?? []) {
-      expect(step.status).toBe('failed');
-      expect(typeof step.failed_at).toBe('number');
-      expect(step.cancelled_at).toBeUndefined();
-    }
-  });
-
-  it('keeps steps open across a HITL interrupt and closes them on resume with the original created_at', async () => {
-    jest.setTimeout(30000);
-    const { sequence, handlers } = createRecorder();
-    const saver = new MemorySaver();
-    const run = await Run.create<t.IState>({
-      runId: 'test-run-step-timestamps-hitl',
-      graphConfig: {
-        type: 'standard',
-        agents: [
-          {
-            agentId: 'timestamps-hitl-agent',
-            provider: Providers.OPENAI,
-            clientOptions: {
-              model: 'gpt-4o-mini',
-              streaming: true,
-              streamUsage: false,
-            },
-            instructions: 'You are a helpful assistant.',
-            maxContextTokens: 8000,
-            graphTools: [askTool],
-          },
-        ],
-        compileOptions: { checkpointer: saver },
-      },
-      returnContent: true,
-      skipCleanup: true,
-      customHandlers: handlers,
-    });
-    run.Graph!.overrideModel = new FakeChatModel({
-      responses: ['asking the user', 'done after resume'],
-      toolCalls: [
-        {
-          name: 'ask_user_question',
-          args: { question: 'pick one' },
-          id: 'call_ask_timestamps',
-          type: 'tool_call',
-        },
-      ],
-    });
-
-    const streamConfig = createStreamConfig('run-step-timestamps-hitl');
-    await run.processStream(
-      { messages: [new HumanMessage('go')] },
-      streamConfig
-    );
-
-    expect(run.getInterrupt()).toBeDefined();
-    const toolStep = run.Graph?.contentData.find(
-      (step) => step.type === StepTypes.TOOL_CALLS
-    ) as t.RunStep;
-    expect(toolStep).toBeDefined();
-    expect(toolStep.status).toBe('in_progress');
-    const createdAt = toolStep.created_at;
-    expect(typeof createdAt).toBe('number');
-    expect(getClosed(sequence).map((event) => event.id)).not.toContain(
-      toolStep.id
-    );
-
-    await run.processStream(
-      new Command({ resume: { answer: 'blue' } }) as unknown as t.IState,
-      streamConfig
-    );
-
-    expect(toolStep.status).toBe('completed');
-    expect(toolStep.created_at).toBe(createdAt);
-    const closedForTool = getClosed(sequence).filter(
-      (event) => event.id === toolStep.id
-    );
-    expect(closedForTool).toHaveLength(1);
-    expect(closedForTool[0].closed_at).toBeGreaterThanOrEqual(
-      createdAt as number
-    );
-    for (const step of run.Graph?.contentData ?? []) {
-      expect(step.status).toBe('completed');
-    }
-  });
-});

package/src/specs/spec.utils.ts

@@ -1,15 +0,0 @@
-export function capitalizeFirstLetter(string: string): string {
-  return string.charAt(0).toUpperCase() + string.slice(1);
-}
-
-export function hasEnv(key: string): boolean {
-  return (process.env[key]?.trim() ?? '') !== '';
-}
-
-export function hasEveryEnv(keys: readonly string[]): boolean {
-  return keys.every(hasEnv);
-}
-
-export function hasAnyEnv(keys: readonly string[]): boolean {
-  return keys.some(hasEnv);
-}