@librechat/agents 3.4.5 → 3.4.7

package/src/specs/run-step-timestamps.test.ts

@@ -0,0 +1,414 @@
+import { z } from 'zod';
+import { tool } from '@langchain/core/tools';
+import { HumanMessage } from '@langchain/core/messages';
+import { MemorySaver, Command } from '@langchain/langgraph';
+import type { ToolCall } from '@langchain/core/messages/tool';
+import type * as t from '@/types';
+import { GraphEvents, StepTypes, Providers } from '@/common';
+import { FakeChatModel } from '@/llm/fake';
+import { askUserQuestion } from '@/hitl';
+import { Run } from '@/run';
+
+const askTool = tool(
+  async (input) => {
+    const { answer } = askUserQuestion(input as { question: string });
+    return answer;
+  },
+  {
+    name: 'ask_user_question',
+    description:
+      'Ask the user a clarifying question and wait for their answer.',
+    schema: z.object({ question: z.string() }),
+  }
+);
+
+const llmConfig: t.LLMConfig = {
+  provider: Providers.OPENAI,
+  streaming: true,
+  streamUsage: false,
+};
+
+const echoTool = tool(
+  async ({ input }: { input: string }) => `echo: ${input}`,
+  {
+    name: 'echoTool',
+    description: 'Echoes the input back',
+    schema: z.object({ input: z.string() }),
+  }
+);
+
+const toolCalls: ToolCall[] = [
+  {
+    name: 'echoTool',
+    args: { input: 'ping' },
+    id: 'call_timestamps_1',
+    type: 'tool_call',
+  },
+];
+
+const createStreamConfig = (
+  threadId: string,
+  signal?: AbortSignal
+): t.RunStreamConfig => ({
+  configurable: { thread_id: threadId },
+  version: 'v2',
+  ...(signal != null ? { signal } : {}),
+});
+
+type RecordedEvent = {
+  event: string;
+  data: unknown;
+  /** Lifecycle fields captured at receipt — dispatched RunStep objects are
+   *  shared references that later closes mutate in place. */
+  statusAtReceipt?: t.RunStepStatus;
+};
+
+function createRecorder(): {
+  sequence: RecordedEvent[];
+  handlers: Record<string, t.EventHandler>;
+  } {
+  const sequence: RecordedEvent[] = [];
+  const record = (event: string): t.EventHandler => ({
+    handle: (_event, data): void => {
+      sequence.push({
+        event,
+        data,
+        ...(event === GraphEvents.ON_RUN_STEP
+          ? { statusAtReceipt: (data as t.RunStep).status }
+          : {}),
+      });
+    },
+  });
+  return {
+    sequence,
+    handlers: {
+      [GraphEvents.ON_RUN_STEP]: record(GraphEvents.ON_RUN_STEP),
+      [GraphEvents.ON_RUN_STEP_COMPLETED]: record(
+        GraphEvents.ON_RUN_STEP_COMPLETED
+      ),
+      [GraphEvents.ON_RUN_STEP_CLOSED]: record(GraphEvents.ON_RUN_STEP_CLOSED),
+    },
+  };
+}
+
+function getSteps(sequence: RecordedEvent[]): t.RunStep[] {
+  return sequence
+    .filter((entry) => entry.event === GraphEvents.ON_RUN_STEP)
+    .map((entry) => entry.data as t.RunStep);
+}
+
+function getClosed(sequence: RecordedEvent[]): t.RunStepClosedEvent[] {
+  return sequence
+    .filter((entry) => entry.event === GraphEvents.ON_RUN_STEP_CLOSED)
+    .map((entry) => entry.data as t.RunStepClosedEvent);
+}
+
+describe('run step timestamps', () => {
+  jest.setTimeout(20000);
+
+  it('stamps starts and closes every step exactly once on a natural finish', async () => {
+    const { sequence, handlers } = createRecorder();
+    const run = await Run.create<t.IState>({
+      runId: 'test-run-step-timestamps',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        tools: [echoTool],
+        instructions: 'You are a helpful assistant.',
+      },
+      returnContent: true,
+      skipCleanup: true,
+      customHandlers: handlers,
+    });
+
+    run.Graph?.overrideTestModel(
+      ['Let me call the tool', 'The tool answered'],
+      2,
+      toolCalls
+    );
+    const started = Date.now();
+    await run.processStream(
+      { messages: [new HumanMessage('hello')] },
+      createStreamConfig('run-step-timestamps')
+    );
+
+    const steps = getSteps(sequence);
+    const closed = getClosed(sequence);
+    expect(steps.length).toBeGreaterThanOrEqual(3);
+    const stepReceipts = sequence.filter(
+      (entry) => entry.event === GraphEvents.ON_RUN_STEP
+    );
+    for (const receipt of stepReceipts) {
+      expect(receipt.statusAtReceipt).toBe('in_progress');
+    }
+    for (const step of steps) {
+      expect(typeof step.created_at).toBe('number');
+      expect(step.created_at as number).toBeGreaterThanOrEqual(started);
+    }
+
+    const stepIds = steps.map((step) => step.id);
+    expect(closed.map((event) => event.id).sort()).toEqual([...stepIds].sort());
+    for (const event of closed) {
+      expect(event.status).toBe('completed');
+      expect(event.closed_at).toBeGreaterThanOrEqual(
+        event.created_at as number
+      );
+    }
+
+    const toolStep = steps.find((step) => step.type === StepTypes.TOOL_CALLS);
+    expect(toolStep).toBeDefined();
+    const toolStepId = (toolStep as t.RunStep).id;
+    const toolCompletedIndex = sequence.findIndex(
+      (entry) =>
+        entry.event === GraphEvents.ON_RUN_STEP_COMPLETED &&
+        (entry.data as { result?: { id?: string } }).result?.id === toolStepId
+    );
+    const toolClosedIndex = sequence.findIndex(
+      (entry) =>
+        entry.event === GraphEvents.ON_RUN_STEP_CLOSED &&
+        (entry.data as t.RunStepClosedEvent).id === toolStepId
+    );
+    expect(toolCompletedIndex).toBeGreaterThanOrEqual(0);
+    expect(toolClosedIndex).toBeGreaterThan(toolCompletedIndex);
+
+    const firstMessageStepId = steps[0].id;
+    const toolStepIndex = sequence.findIndex(
+      (entry) =>
+        entry.event === GraphEvents.ON_RUN_STEP &&
+        (entry.data as t.RunStep).id === toolStepId
+    );
+    const firstMessageClosedIndex = sequence.findIndex(
+      (entry) =>
+        entry.event === GraphEvents.ON_RUN_STEP_CLOSED &&
+        (entry.data as t.RunStepClosedEvent).id === firstMessageStepId
+    );
+    expect(firstMessageClosedIndex).toBeGreaterThanOrEqual(0);
+    expect(firstMessageClosedIndex).toBeLessThan(toolStepIndex);
+
+    const completedEvent = sequence
+      .filter((entry) => entry.event === GraphEvents.ON_RUN_STEP_COMPLETED)
+      .map((entry) => (entry.data as { result: t.ToolCompleteEvent }).result)
+      .find((result) => result.id === toolStepId);
+    expect(typeof completedEvent?.completed_at).toBe('number');
+
+    for (const step of run.Graph?.contentData ?? []) {
+      expect(step.status).toBe('completed');
+      expect(typeof step.completed_at).toBe('number');
+    }
+  });
+
+  it('changes nothing for hosts that only register the legacy handlers', async () => {
+    const received: string[] = [];
+    const legacyHandler = (event: string): t.EventHandler => ({
+      handle: (): void => {
+        received.push(event);
+      },
+    });
+    const run = await Run.create<t.IState>({
+      runId: 'test-run-step-timestamps-legacy',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        tools: [echoTool],
+        instructions: 'You are a helpful assistant.',
+      },
+      returnContent: true,
+      skipCleanup: true,
+      customHandlers: {
+        [GraphEvents.ON_RUN_STEP]: legacyHandler(GraphEvents.ON_RUN_STEP),
+        [GraphEvents.ON_RUN_STEP_COMPLETED]: legacyHandler(
+          GraphEvents.ON_RUN_STEP_COMPLETED
+        ),
+        [GraphEvents.ON_MESSAGE_DELTA]: legacyHandler(
+          GraphEvents.ON_MESSAGE_DELTA
+        ),
+      },
+    });
+
+    run.Graph?.overrideTestModel(
+      ['Let me call the tool', 'The tool answered'],
+      2,
+      toolCalls
+    );
+    await run.processStream(
+      { messages: [new HumanMessage('hello')] },
+      createStreamConfig('run-step-timestamps-legacy')
+    );
+
+    expect(new Set(received)).toEqual(
+      new Set([
+        GraphEvents.ON_RUN_STEP,
+        GraphEvents.ON_RUN_STEP_COMPLETED,
+        GraphEvents.ON_MESSAGE_DELTA,
+      ])
+    );
+    for (const step of run.Graph?.contentData ?? []) {
+      expect(step.status).toBe('completed');
+    }
+  });
+
+  it('sweeps unfinished steps as cancelled when the caller aborts mid-stream', async () => {
+    const { sequence, handlers } = createRecorder();
+    const controller = new AbortController();
+    const run = await Run.create<t.IState>({
+      runId: 'test-run-step-timestamps-abort',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        instructions: 'You are a helpful assistant.',
+      },
+      returnContent: true,
+      skipCleanup: true,
+      customHandlers: {
+        ...handlers,
+        [GraphEvents.ON_MESSAGE_DELTA]: {
+          handle: (): void => {
+            controller.abort();
+          },
+        },
+      },
+    });
+
+    run.Graph?.overrideTestModel(
+      ['This response streams slowly enough to abort in flight'],
+      25
+    );
+    await expect(
+      run.processStream(
+        { messages: [new HumanMessage('hello')] },
+        createStreamConfig('run-step-timestamps-abort', controller.signal)
+      )
+    ).rejects.toThrow();
+
+    const closed = getClosed(sequence);
+    expect(closed.length).toBeGreaterThanOrEqual(1);
+    expect(closed.every((event) => event.status === 'cancelled')).toBe(true);
+    for (const step of run.Graph?.contentData ?? []) {
+      expect(step.status).toBe('cancelled');
+      expect(typeof step.cancelled_at).toBe('number');
+      expect(step.cancelled_at as number).toBeGreaterThanOrEqual(
+        step.created_at as number
+      );
+    }
+  });
+
+  it('sweeps as failed when an AbortError surfaces with no signal aborted', async () => {
+    const { sequence, handlers } = createRecorder();
+    const run = await Run.create<t.IState>({
+      runId: 'test-run-step-timestamps-uncorroborated-abort',
+      graphConfig: {
+        type: 'standard',
+        llmConfig,
+        instructions: 'You are a helpful assistant.',
+      },
+      returnContent: true,
+      skipCleanup: true,
+      customHandlers: {
+        ...handlers,
+        [GraphEvents.ON_MESSAGE_DELTA]: {
+          handle: (): void => {
+            /** A provider/host rejection that merely borrows the name — no
+             *  caller or construction signal was ever aborted. */
+            const error = new Error('provider stream aborted internally');
+            error.name = 'AbortError';
+            throw error;
+          },
+        },
+      },
+    });
+
+    run.Graph?.overrideTestModel(['some streamed content'], 2);
+    await expect(
+      run.processStream(
+        { messages: [new HumanMessage('hello')] },
+        createStreamConfig('run-step-timestamps-uncorroborated-abort')
+      )
+    ).rejects.toThrow();
+
+    const closed = getClosed(sequence);
+    expect(closed.length).toBeGreaterThanOrEqual(1);
+    expect(closed.every((event) => event.status === 'failed')).toBe(true);
+    for (const step of run.Graph?.contentData ?? []) {
+      expect(step.status).toBe('failed');
+      expect(typeof step.failed_at).toBe('number');
+      expect(step.cancelled_at).toBeUndefined();
+    }
+  });
+
+  it('keeps steps open across a HITL interrupt and closes them on resume with the original created_at', async () => {
+    jest.setTimeout(30000);
+    const { sequence, handlers } = createRecorder();
+    const saver = new MemorySaver();
+    const run = await Run.create<t.IState>({
+      runId: 'test-run-step-timestamps-hitl',
+      graphConfig: {
+        type: 'standard',
+        agents: [
+          {
+            agentId: 'timestamps-hitl-agent',
+            provider: Providers.OPENAI,
+            clientOptions: {
+              model: 'gpt-4o-mini',
+              streaming: true,
+              streamUsage: false,
+            },
+            instructions: 'You are a helpful assistant.',
+            maxContextTokens: 8000,
+            graphTools: [askTool],
+          },
+        ],
+        compileOptions: { checkpointer: saver },
+      },
+      returnContent: true,
+      skipCleanup: true,
+      customHandlers: handlers,
+    });
+    run.Graph!.overrideModel = new FakeChatModel({
+      responses: ['asking the user', 'done after resume'],
+      toolCalls: [
+        {
+          name: 'ask_user_question',
+          args: { question: 'pick one' },
+          id: 'call_ask_timestamps',
+          type: 'tool_call',
+        },
+      ],
+    });
+
+    const streamConfig = createStreamConfig('run-step-timestamps-hitl');
+    await run.processStream(
+      { messages: [new HumanMessage('go')] },
+      streamConfig
+    );
+
+    expect(run.getInterrupt()).toBeDefined();
+    const toolStep = run.Graph?.contentData.find(
+      (step) => step.type === StepTypes.TOOL_CALLS
+    ) as t.RunStep;
+    expect(toolStep).toBeDefined();
+    expect(toolStep.status).toBe('in_progress');
+    const createdAt = toolStep.created_at;
+    expect(typeof createdAt).toBe('number');
+    expect(getClosed(sequence).map((event) => event.id)).not.toContain(
+      toolStep.id
+    );
+
+    await run.processStream(
+      new Command({ resume: { answer: 'blue' } }) as unknown as t.IState,
+      streamConfig
+    );
+
+    expect(toolStep.status).toBe('completed');
+    expect(toolStep.created_at).toBe(createdAt);
+    const closedForTool = getClosed(sequence).filter(
+      (event) => event.id === toolStep.id
+    );
+    expect(closedForTool).toHaveLength(1);
+    expect(closedForTool[0].closed_at).toBeGreaterThanOrEqual(
+      createdAt as number
+    );
+    for (const step of run.Graph?.contentData ?? []) {
+      expect(step.status).toBe('completed');
+    }
+  });
+});

package/src/stream.ts

@@ -924,6 +924,7 @@ async function dispatchEagerToolCompletions(args: {
               progress: 1,
               ...(outcome != null && { outcome }),
             } as t.ProcessedToolCall,
+            completed_at: Date.now(),
           },
         },
         graph.config
@@ -1836,9 +1837,7 @@ export class ChatModelStreamHandler implements t.EventHandler {
         contentGroups[0]
       ).phase;
       const phaseChanged =
-        currentPhase != null &&
-        nextPhase != null &&
-        currentPhase !== nextPhase;
+        currentPhase != null && nextPhase != null && currentPhase !== nextPhase;
       if (contentGroups.length > 1 || phaseChanged) {
         for (const contentGroup of contentGroups) {
           const currentStepId = await dispatchMessageCreationStep({
@@ -2299,20 +2298,44 @@ export function createContentAggregator(): t.ContentAggregatorResult {
       return;
     }
 
+    const incomingText =
+      ContentTypes.TEXT in contentPart && typeof contentPart.text === 'string'
+        ? contentPart.text
+        : undefined;
+    /**
+     * Anthropic emits a search turn's citations as their own `citations_delta`,
+     * which arrives here as a text part carrying `citations` and no `text`.
+     */
+    const { citations } = contentPart as {
+      citations?: t.MessageDeltaUpdate['citations'];
+    };
+    const incomingCitations = Array.isArray(citations) ? citations : undefined;
+
     if (
       partType.startsWith(ContentTypes.TEXT) &&
-      ContentTypes.TEXT in contentPart &&
-      typeof contentPart.text === 'string'
+      (incomingText !== undefined || incomingCitations !== undefined)
     ) {
       // TODO: update this!!
       const currentContent = contentParts[index] as t.MessageDeltaUpdate;
       const update: t.MessageDeltaUpdate = {
         type: ContentTypes.TEXT,
-        text: (currentContent.text || '') + contentPart.text,
+        text: (currentContent.text || '') + (incomingText ?? ''),
       };
 
       if (contentPart.tool_call_ids) {
         update.tool_call_ids = contentPart.tool_call_ids;
+      } else if (incomingText === undefined && currentContent.tool_call_ids) {
+        /** A citations-only delta must not drop ids already accumulated */
+        update.tool_call_ids = currentContent.tool_call_ids;
+      }
+
+      if (incomingCitations !== undefined) {
+        update.citations =
+          currentContent.citations !== undefined
+            ? [...currentContent.citations, ...incomingCitations]
+            : [...incomingCitations];
+      } else if (currentContent.citations !== undefined) {
+        update.citations = currentContent.citations;
       }
       contentParts[index] = update;
     } else if (

package/src/summarization/node.ts

@@ -908,6 +908,16 @@ interface CreateSummarizeNodeParams {
       result: t.StepCompleted,
       config?: RunnableConfig
     ) => Promise<void>;
+    /**
+     * Terminal close for a summary step that ends without a completion —
+     * an errored or empty summary would otherwise stay `in_progress` until
+     * the run-end sweep reported it as `completed`.
+     */
+    closeRunStep?: (
+      stepId: string,
+      status: Exclude<t.RunStepStatus, 'in_progress'>,
+      config?: RunnableConfig
+    ) => Promise<void>;
     /** The run's shared breaker signal, composed into every summarization
      * model attempt so a sibling branch tripping a stream limit also
      * cancels in-flight summaries. */
@@ -1342,6 +1352,7 @@ export function createSummarizeNode({
           runnableConfig
         );
       }
+      await graph.closeRunStep?.(stepId, 'failed', runnableConfig);
       return { summarizationRequest: undefined };
     }
 

package/src/tools/ToolNode.ts

@@ -2561,6 +2561,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
             index: turn,
             type: 'tool_call' as const,
             tool_call,
+            completed_at: Date.now(),
           },
         },
         config
@@ -3965,6 +3966,7 @@ export class ToolNode<T = any> extends RunnableCallable<T, T> {
             progress: 1,
             ...(outcome != null && { outcome }),
           } as t.ProcessedToolCall,
+          completed_at: Date.now(),
         },
       },
       config

package/src/tools/__tests__/SubagentExecutor.test.ts

@@ -1563,17 +1563,13 @@ describe('SubagentExecutor', () => {
     const selected = makeConfig('selected', {
       agentInputs: {
         ...makeChildInputs('selected-agent'),
-        initialSessions: new Map([
-          [Constants.EXECUTE_CODE, selectedSession],
-        ]),
+        initialSessions: new Map([[Constants.EXECUTE_CODE, selectedSession]]),
       },
     });
     const sibling = makeConfig('sibling', {
       agentInputs: {
         ...makeChildInputs('sibling-agent'),
-        initialSessions: new Map([
-          [Constants.EXECUTE_CODE, siblingSession],
-        ]),
+        initialSessions: new Map([[Constants.EXECUTE_CODE, siblingSession]]),
       },
     });
     const childSessions: ToolSessionMap = new Map();
@@ -1613,7 +1609,10 @@ describe('SubagentExecutor', () => {
   });
 
   it('combines member session seeds inside a selected graph child', async () => {
-    const makeSession = (storageSessionId: string, includeFileSession = true) => ({
+    const makeSession = (
+      storageSessionId: string,
+      includeFileSession = true
+    ) => ({
       session_id: storageSessionId,
       files: [
         {
@@ -1696,9 +1695,7 @@ describe('SubagentExecutor', () => {
     const hitlConfig = makeConfig('hitl-child', {
       agentInputs: {
         ...makeChildInputs('hitl-child'),
-        initialSessions: new Map([
-          [Constants.EXECUTE_CODE, seededSession],
-        ]),
+        initialSessions: new Map([[Constants.EXECUTE_CODE, seededSession]]),
       },
     });
     const childSessions: ToolSessionMap = new Map();
@@ -3625,6 +3622,51 @@ describe('sanitizeForwardedSubagentUpdateData', () => {
     expect(serialized).not.toContain('refresh-secret');
   });
 
+  it('forwards run step lifecycle stamps and closed events via allowlists', () => {
+    const sanitizedStep = sanitizeForwardedSubagentUpdateData(
+      GraphEvents.ON_RUN_STEP,
+      {
+        id: 'step_1',
+        type: StepTypes.MESSAGE_CREATION,
+        index: 0,
+        created_at: 1_000,
+        status: 'in_progress',
+        stepDetails: {
+          type: StepTypes.MESSAGE_CREATION,
+          message_creation: { message_id: 'message_1' },
+        },
+      }
+    ) as { created_at?: number; status?: string };
+    expect(sanitizedStep.created_at).toBe(1_000);
+    expect(sanitizedStep.status).toBe('in_progress');
+
+    const sanitizedClosed = sanitizeForwardedSubagentUpdateData(
+      GraphEvents.ON_RUN_STEP_CLOSED,
+      {
+        id: 'step_1',
+        index: 0,
+        type: StepTypes.MESSAGE_CREATION,
+        status: 'completed',
+        created_at: 1_000,
+        closed_at: 2_000,
+        runId: 'run_1',
+        agentId: 'researcher',
+        futureSecret: 'top-level-secret',
+      }
+    );
+    expect(sanitizedClosed).toEqual({
+      id: 'step_1',
+      index: 0,
+      type: StepTypes.MESSAGE_CREATION,
+      status: 'completed',
+      created_at: 1_000,
+      closed_at: 2_000,
+      runId: 'run_1',
+      agentId: 'researcher',
+    });
+    expect(JSON.stringify(sanitizedClosed)).not.toContain('top-level-secret');
+  });
+
   it('keeps completed tool output while stripping operational fields', () => {
     const output = 'x'.repeat(10_000);
     const sanitized = sanitizeForwardedSubagentUpdateData(

package/src/tools/__tests__/handlers.test.ts

@@ -16,6 +16,7 @@ type MockGraph = {
   getRunStep: jest.Mock;
   dispatchRunStep: jest.Mock;
   dispatchRunStepDelta: jest.Mock;
+  registerPendingToolCall: jest.Mock;
   toolCallStepIds: Map<string, string>;
   messageStepHasToolCalls: Map<string, boolean>;
   messageIdsByStepKey: Map<string, string>;
@@ -40,6 +41,7 @@ function createMockGraph(overrides?: Partial<MockGraph>): MockGraph {
     dispatchRunStepDelta: jest
       .fn<() => Promise<void>>()
       .mockResolvedValue(undefined),
+    registerPendingToolCall: jest.fn<() => void>(),
     toolCallStepIds: new Map(),
     messageStepHasToolCalls: new Map(),
     messageIdsByStepKey: new Map(),

package/src/tools/handlers.ts

@@ -181,6 +181,7 @@ export const handleToolCalls = async (
       const isEmpty = !details.tool_calls || details.tool_calls.length === 0;
       if (isEmpty && prevStepId !== reusedChunkStepId) {
         graph.toolCallStepIds.set(toolCallId, prevStepId);
+        graph.registerPendingToolCall(toolCallId, prevStepId);
         reusedChunkStepId = prevStepId;
         continue;
       }