@illuma-ai/agents 1.1.19 → 1.1.21

package/src/tools/__tests__/ToolApproval.test.ts

@@ -1,9 +1,13 @@
 /**
  * Tests for Human-in-the-Loop (HITL) tool approval in ToolNode.
  *
- * Tests the approval policy evaluation and the event-driven approval flow.
- * The HITL system dispatches ON_TOOL_APPROVAL_REQUIRED events that the host
- * (Ranger) handles by showing UI and resolving/rejecting the promise.
+ * Uses LangGraph's native interrupt()/Command({ resume }) pattern.
+ * Tests verify:
+ * - Policy evaluation (no config, scheduled, overrides, custom functions)
+ * - Interrupt/resume flow with MemorySaver checkpointer
+ * - Denial stops tool execution
+ * - Modified args are applied on resume
+ * - Multiple tool calls each get individual interrupts
  */
 import { tool } from '@langchain/core/tools';
 import { z } from 'zod';
@@ -13,6 +17,8 @@ import {
   Annotation,
   messagesStateReducer,
   START,
+  MemorySaver,
+  Command,
 } from '@langchain/langgraph';
 import type { BaseMessage } from '@langchain/core/messages';
 import type { RunnableConfig } from '@langchain/core/runnables';
@@ -68,22 +74,19 @@ const searchTool = tool(
 );
 
 /**
- * Helper to build a graph with HITL support for integration testing.
- * Returns the compiled graph and a way to intercept approval events.
+ * Helper to build a graph with HITL support using interrupt/resume pattern.
+ * Uses MemorySaver checkpointer for interrupt persistence.
+ * Returns the compiled graph and a notification collector.
  */
 function createTestGraph(
   tools: t.GenericTool[],
   toolApprovalConfig: t.ToolApprovalConfig,
-  modelResponses: string[][],
-  approvalHandler: (event: t.ToolApprovalEvent) => void
+  modelResponses: string[][]
 ): {
-  compiled: {
-    invoke: (
-      input: { messages: BaseMessage[] },
-      config?: Partial<RunnableConfig>
-    ) => Promise<{ messages: BaseMessage[] }>;
-  };
-  config: Partial<RunnableConfig> & { version: 'v1' | 'v2' };
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  compiled: any;
+  notifications: t.ToolApprovalNotification[];
+  config: Partial<RunnableConfig> & { version: 'v1' | 'v2'; configurable: { thread_id: string } };
 } {
   const StateAnnotation = Annotation.Root({
     messages: Annotation<BaseMessage[]>({
@@ -135,11 +138,18 @@ function createTestGraph(
     .addConditionalEdges('agent', routeMessage)
     .addEdge('tools', 'agent');
 
-  const compiled = workflow.compile();
+  // MemorySaver is required for interrupt()/Command({ resume }) to work
+  const checkpointer = new MemorySaver();
+  const compiled = workflow.compile({ checkpointer });
+
+  const notifications: t.ToolApprovalNotification[] = [];
 
-  // Create a config with custom event handler to intercept approval requests
-  const config: Partial<RunnableConfig> & { version: 'v1' | 'v2' } = {
+  // Collect notification events (data-only, no resolve/reject)
+  const config: Partial<RunnableConfig> & { version: 'v1' | 'v2'; configurable: { thread_id: string } } = {
     version: 'v2',
+    configurable: {
+      thread_id: `test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+    },
     callbacks: [
       {
         handleCustomEvent: async (
@@ -147,14 +157,14 @@ function createTestGraph(
           data: unknown
         ): Promise<void> => {
           if (eventName === GraphEvents.ON_TOOL_APPROVAL_REQUIRED) {
-            approvalHandler(data as t.ToolApprovalEvent);
+            notifications.push(data as t.ToolApprovalNotification);
           }
         },
       },
     ],
   };
 
-  return { compiled, config };
+  return { compiled, notifications, config };
 }
 
 // ============================================================================
@@ -265,7 +275,6 @@ describe('HITL Tool Approval - Policy Evaluation', () => {
   });
 
   test('scheduled context bypasses custom function policy', async () => {
-    // Even a custom function that always returns true should be skipped in scheduled context
     const alwaysRequire = (): boolean => true;
     const toolNode = new ToolNode({
       tools: [echoTool],
@@ -299,7 +308,6 @@ describe('HITL Tool Approval - Policy Evaluation', () => {
   });
 
   test('scheduled context bypasses per-tool overrides', async () => {
-    // Even with an explicit 'always' override for the tool, scheduled context should skip it
     const toolNode = new ToolNode({
       tools: [searchTool],
       toolApprovalConfig: {
@@ -333,58 +341,17 @@ describe('HITL Tool Approval - Policy Evaluation', () => {
       'Result for: scheduled override test'
     );
   });
-
-  test('interactive context still requires approval (scheduled does not)', async () => {
-    // Verify that 'interactive' context DOES require approval (contrasts with scheduled)
-    const approvalEvents: t.ToolApprovalEvent[] = [];
-    const { compiled, config } = createTestGraph(
-      [sendEmailTool],
-      {
-        defaultPolicy: 'always',
-        executionContext: 'interactive',
-      },
-      [
-        // First model call: request tool call
-        [
-          JSON.stringify({
-            name: 'send_email',
-            args: { to: 'user@example.com', subject: 'Interactive' },
-            id: 'call_int_1',
-            type: 'tool_call',
-          }),
-        ],
-        // Second model call: finish after tool result
-        ['Done.'],
-      ],
-      (event) => {
-        approvalEvents.push(event);
-        // Auto-approve for test to complete
-        event.resolve({ approved: true });
-      }
-    );
-
-    const _result = await compiled.invoke(
-      { messages: [new HumanMessage('send an email')] },
-      config
-    );
-
-    // Approval WAS required for interactive context
-    expect(approvalEvents.length).toBe(1);
-    expect(approvalEvents[0].toolName).toBe('send_email');
-  });
 });
 
 // ============================================================================
-// Integration Tests: Event-driven approval flow
+// Integration Tests: Interrupt/Resume approval flow
 // ============================================================================
 
-describe('HITL Tool Approval - Event-Driven Flow', () => {
+describe('HITL Tool Approval - Interrupt/Resume Flow', () => {
   jest.setTimeout(15000);
 
-  test('interactive mode with "always" policy dispatches approval event and waits', async () => {
-    const approvalEvents: t.ToolApprovalEvent[] = [];
-
-    const { compiled, config } = createTestGraph(
+  test('interactive mode with "always" policy interrupts graph and resumes on approval', async () => {
+    const { compiled, notifications, config } = createTestGraph(
       [echoTool as t.GenericTool],
       {
         defaultPolicy: 'always',
@@ -402,24 +369,26 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
         ],
         // Second model response (after tool executes): final text
         ['The echo returned the result.'],
-      ],
-      (event) => {
-        // Auto-approve after capturing the event
-        approvalEvents.push(event);
-        event.resolve({ approved: true });
-      }
+      ]
     );
 
-    const result = await compiled.invoke(
+    // First invoke: graph runs until interrupt() is hit
+    const interruptResult = await compiled.invoke(
       { messages: [new HumanMessage('Say hello')] },
       config
     );
 
-    // Verify the approval event was dispatched
-    expect(approvalEvents).toHaveLength(1);
-    expect(approvalEvents[0].type).toBe('tool_approval_required');
-    expect(approvalEvents[0].toolName).toBe('echo');
-    expect(approvalEvents[0].toolArgs).toEqual({ message: 'needs approval' });
+    // Verify the notification was dispatched
+    expect(notifications).toHaveLength(1);
+    expect(notifications[0].type).toBe('tool_approval_required');
+    expect(notifications[0].toolName).toBe('echo');
+    expect(notifications[0].toolArgs).toEqual({ message: 'needs approval' });
+
+    // Resume with approval — Command({ resume }) returns value to interrupt()
+    const result = await compiled.invoke(
+      new Command({ resume: { approved: true } as t.ToolApprovalResponse }),
+      config
+    );
 
     // Verify the tool executed after approval
     const toolMessages = result.messages.filter(
@@ -454,18 +423,21 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
         ],
         // Second model response (after denial): acknowledge
         ['I understand, the email was not sent.'],
-      ],
-      (event) => {
-        // Deny the tool call
-        event.resolve({ approved: false, feedback: 'Not now' });
-      }
+      ]
     );
 
-    const result = await compiled.invoke(
+    // First invoke: hits interrupt
+    await compiled.invoke(
       { messages: [new HumanMessage('Send an email')] },
       config
     );
 
+    // Resume with denial
+    const result = await compiled.invoke(
+      new Command({ resume: { approved: false, feedback: 'Not now' } as t.ToolApprovalResponse }),
+      config
+    );
+
     // Find the tool message - it should be a denial
     const toolMessages = result.messages.filter(
       (m: BaseMessage) => m._getType() === 'tool'
@@ -483,11 +455,45 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
     );
   });
 
-  test('custom function override evaluates args to determine approval', async () => {
-    // Only require approval for emails to external domains
-    const approvalEvents: t.ToolApprovalEvent[] = [];
+  test('interactive context triggers interrupt (scheduled does not)', async () => {
+    const { compiled, notifications, config } = createTestGraph(
+      [sendEmailTool as t.GenericTool],
+      {
+        defaultPolicy: 'always',
+        executionContext: 'interactive',
+      },
+      [
+        [
+          JSON.stringify({
+            name: 'send_email',
+            args: { to: 'user@example.com', subject: 'Interactive' },
+            id: 'call_int_1',
+            type: 'tool_call',
+          }),
+        ],
+        ['Done.'],
+      ]
+    );
 
-    const { compiled, config } = createTestGraph(
+    // Graph should interrupt (not complete)
+    await compiled.invoke(
+      { messages: [new HumanMessage('send an email')] },
+      config
+    );
+
+    // Notification WAS dispatched for interactive context
+    expect(notifications.length).toBe(1);
+    expect(notifications[0].toolName).toBe('send_email');
+
+    // Resume to complete the graph
+    await compiled.invoke(
+      new Command({ resume: { approved: true } as t.ToolApprovalResponse }),
+      config
+    );
+  });
+
+  test('custom function override evaluates args to determine approval', async () => {
+    const { compiled, notifications, config } = createTestGraph(
       [sendEmailTool as t.GenericTool],
       {
         defaultPolicy: 'never',
@@ -511,11 +517,7 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
         ],
         // Final text
         ['Email sent to internal address.'],
-      ],
-      (event) => {
-        approvalEvents.push(event);
-        event.resolve({ approved: true });
-      }
+      ]
     );
 
     const result = await compiled.invoke(
@@ -523,10 +525,10 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
       config
     );
 
-    // No approval events should have been dispatched (internal domain)
-    expect(approvalEvents).toHaveLength(0);
+    // No notifications should have been dispatched (internal domain bypasses approval)
+    expect(notifications).toHaveLength(0);
 
-    // Email should have been sent directly
+    // Email should have been sent directly without interrupt
     const toolMessages = result.messages.filter(
       (m: BaseMessage) => m._getType() === 'tool'
     );
@@ -536,10 +538,8 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
     );
   });
 
-  test('custom function override triggers approval for external email', async () => {
-    const approvalEvents: t.ToolApprovalEvent[] = [];
-
-    const { compiled, config } = createTestGraph(
+  test('custom function override triggers interrupt for external email', async () => {
+    const { compiled, notifications, config } = createTestGraph(
       [sendEmailTool as t.GenericTool],
       {
         defaultPolicy: 'never',
@@ -563,22 +563,25 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
         ],
         // After approval
         ['Email sent to external address.'],
-      ],
-      (event) => {
-        approvalEvents.push(event);
-        event.resolve({ approved: true });
-      }
+      ]
     );
 
-    const result = await compiled.invoke(
+    // First invoke: hits interrupt for external domain
+    await compiled.invoke(
       { messages: [new HumanMessage('Send external email')] },
       config
     );
 
-    // Approval event should have been dispatched (external domain)
-    expect(approvalEvents).toHaveLength(1);
-    expect(approvalEvents[0].toolName).toBe('send_email');
-    expect(approvalEvents[0].toolArgs.to).toBe('external@gmail.com');
+    // Notification should have been dispatched (external domain)
+    expect(notifications).toHaveLength(1);
+    expect(notifications[0].toolName).toBe('send_email');
+    expect(notifications[0].toolArgs.to).toBe('external@gmail.com');
+
+    // Resume with approval
+    const result = await compiled.invoke(
+      new Command({ resume: { approved: true } as t.ToolApprovalResponse }),
+      config
+    );
 
     // Email should have been sent after approval
     const lastMessage = result.messages[result.messages.length - 1];
@@ -604,21 +607,26 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
         ],
         // After tool execution with modified args
         ['Email sent with corrected details.'],
-      ],
-      (event) => {
-        // Approve with modified args
-        event.resolve({
+      ]
+    );
+
+    // First invoke: hits interrupt
+    await compiled.invoke(
+      { messages: [new HumanMessage('Send email')] },
+      config
+    );
+
+    // Resume with modified args
+    const result = await compiled.invoke(
+      new Command({
+        resume: {
           approved: true,
           modifiedArgs: {
             to: 'correct@example.com',
             subject: 'Corrected Subject',
           },
-        });
-      }
-    );
-
-    const result = await compiled.invoke(
-      { messages: [new HumanMessage('Send email')] },
+        } as t.ToolApprovalResponse,
+      }),
       config
     );
 
@@ -635,14 +643,12 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
     expect(sentMsg!.content.toString()).toContain('Corrected Subject');
   });
 
-  test('scheduled context auto-approves without dispatching events', async () => {
-    const approvalEvents: t.ToolApprovalEvent[] = [];
-
-    const { compiled, config } = createTestGraph(
+  test('scheduled context auto-approves without dispatching events or interrupting', async () => {
+    const { compiled, notifications, config } = createTestGraph(
       [sendEmailTool as t.GenericTool],
       {
         defaultPolicy: 'always',
-        executionContext: 'scheduled', // Scheduled = auto-approve
+        executionContext: 'scheduled', // Scheduled = auto-approve, no interrupt
       },
       [
         [
@@ -654,20 +660,17 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
           }),
         ],
         ['Scheduled email sent.'],
-      ],
-      (event) => {
-        approvalEvents.push(event);
-        event.resolve({ approved: true });
-      }
+      ]
     );
 
+    // Should complete without interrupting — no resume needed
     const result = await compiled.invoke(
       { messages: [new HumanMessage('Send scheduled email')] },
       config
     );
 
-    // No approval events should be dispatched for scheduled execution
-    expect(approvalEvents).toHaveLength(0);
+    // No notifications should be dispatched for scheduled execution
+    expect(notifications).toHaveLength(0);
 
     // Email should have been sent directly
     const toolMessages = result.messages.filter(
@@ -677,94 +680,8 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
     expect(toolMessages[0].content.toString()).toContain('user@example.com');
   });
 
-  test('approval promise rejection is handled gracefully (host error)', async () => {
-    const toolNode = new ToolNode({
-      tools: [echoTool] as t.GenericTool[],
-      toolApprovalConfig: {
-        defaultPolicy: 'always',
-        executionContext: 'interactive',
-      },
-    });
-
-    const StateAnnotation = Annotation.Root({
-      messages: Annotation<BaseMessage[]>({
-        reducer: messagesStateReducer,
-        default: () => [],
-      }),
-    });
-
-    let callCount = 0;
-    const callModel = async (_state: {
-      messages: BaseMessage[];
-    }): Promise<{ messages: BaseMessage[] }> => {
-      callCount++;
-      if (callCount === 1) {
-        return {
-          messages: [
-            new AIMessage({
-              content: '',
-              tool_calls: [
-                {
-                  name: 'echo',
-                  args: { message: 'test' },
-                  id: 'call_reject_1',
-                  type: 'tool_call' as const,
-                },
-              ],
-            }),
-          ],
-        };
-      }
-      return { messages: [new AIMessage({ content: 'Recovered.' })] };
-    };
-
-    const routeMessage = (state: typeof StateAnnotation.State): string =>
-      toolsCondition(state, 'tools');
-
-    const workflow = new StateGraph(StateAnnotation)
-      .addNode('agent', callModel)
-      .addNode('tools', toolNode)
-      .addEdge(START, 'agent')
-      .addConditionalEdges('agent', routeMessage)
-      .addEdge('tools', 'agent');
-
-    const compiled = workflow.compile();
-    const config: Partial<RunnableConfig> & { version: 'v1' | 'v2' } = {
-      version: 'v2',
-      callbacks: [
-        {
-          handleCustomEvent: async (
-            eventName: string,
-            data: unknown
-          ): Promise<void> => {
-            if (eventName === GraphEvents.ON_TOOL_APPROVAL_REQUIRED) {
-              const event = data as t.ToolApprovalEvent;
-              // Simulate host-side error (e.g., stream disconnected)
-              event.reject(new Error('Stream closed before approval'));
-            }
-          },
-        },
-      ],
-    };
-
-    // The graph should handle rejection without crashing
-    const result = await compiled.invoke(
-      { messages: [new HumanMessage('Say hello')] },
-      config
-    );
-
-    // Should have a tool message with error content
-    const toolMessages = result.messages.filter(
-      (m: BaseMessage) => m._getType() === 'tool'
-    );
-    expect(toolMessages.length).toBeGreaterThan(0);
-    // Agent should continue after rejection
-    const lastMessage = result.messages[result.messages.length - 1];
-    expect(lastMessage._getType()).toBe('ai');
-  });
-
-  test('multiple tool calls each get individual approval', async () => {
-    const approvalEvents: t.ToolApprovalEvent[] = [];
+  test('multiple tool calls each get individual interrupts', async () => {
+    const notifications: t.ToolApprovalNotification[] = [];
 
     const toolNode = new ToolNode({
       tools: [echoTool, sendEmailTool] as t.GenericTool[],
@@ -822,9 +739,14 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
       .addConditionalEdges('agent', routeMessage)
       .addEdge('tools', 'agent');
 
-    const compiled = workflow.compile();
-    const config: Partial<RunnableConfig> & { version: 'v1' | 'v2' } = {
+    const checkpointer = new MemorySaver();
+    const compiled = workflow.compile({ checkpointer });
+
+    const config: Partial<RunnableConfig> & { version: 'v1' | 'v2'; configurable: { thread_id: string } } = {
       version: 'v2',
+      configurable: {
+        thread_id: `test-multi-${Date.now()}`,
+      },
       callbacks: [
         {
           handleCustomEvent: async (
@@ -832,122 +754,37 @@ describe('HITL Tool Approval - Event-Driven Flow', () => {
             data: unknown
           ): Promise<void> => {
             if (eventName === GraphEvents.ON_TOOL_APPROVAL_REQUIRED) {
-              const event = data as t.ToolApprovalEvent;
-              approvalEvents.push(event);
-              event.resolve({ approved: true });
+              notifications.push(data as t.ToolApprovalNotification);
             }
           },
         },
       ],
     };
 
-    const result = await compiled.invoke(
+    // First invoke: hits first interrupt
+    await compiled.invoke(
       { messages: [new HumanMessage('Do both')] },
       config
     );
 
-    // Both tools should have triggered approval events
-    expect(approvalEvents).toHaveLength(2);
-    expect(approvalEvents.map((e) => e.toolName).sort()).toEqual([
-      'echo',
-      'send_email',
-    ]);
+    // First interrupt notification received
+    expect(notifications.length).toBeGreaterThanOrEqual(1);
 
-    // Both tools should have executed
-    const toolMessages = result.messages.filter(
-      (m: BaseMessage) => m._getType() === 'tool'
+    // Resume first tool approval
+    await compiled.invoke(
+      new Command({ resume: { approved: true } as t.ToolApprovalResponse }),
+      config
     );
-    expect(toolMessages).toHaveLength(2);
-  });
-});
 
-// ============================================================================
-// Type Tests: ToolApprovalConfig shape validation
-// ============================================================================
-
-describe('HITL Types', () => {
-  test('ToolApprovalConfig accepts valid configurations', () => {
-    const config1: t.ToolApprovalConfig = {
-      defaultPolicy: 'always',
-      executionContext: 'interactive',
-    };
-    expect(config1.defaultPolicy).toBe('always');
-
-    const config2: t.ToolApprovalConfig = {
-      defaultPolicy: 'never',
-      executionContext: 'scheduled',
-    };
-    expect(config2.executionContext).toBe('scheduled');
-
-    const config3: t.ToolApprovalConfig = {
-      defaultPolicy: 'always',
-      executionContext: 'interactive',
-      overrides: {
-        search: 'never',
-        send_email: 'always',
-        custom_tool: (_toolName, args) => (args.dangerous as boolean) === true,
-      },
-    };
-    expect(config3.overrides).toBeDefined();
-    expect(config3.overrides!.search).toBe('never');
-  });
-
-  test('ToolApprovalRequest has correct shape', () => {
-    const request: t.ToolApprovalRequest = {
-      type: 'tool_approval_required',
-      toolCallId: 'call_123',
-      toolName: 'send_email',
-      toolArgs: { to: 'user@example.com' },
-      agentId: 'agent-1',
-      description: 'Send email to user',
-    };
-    expect(request.type).toBe('tool_approval_required');
-    expect(request.toolName).toBe('send_email');
-  });
-
-  test('ToolApprovalResponse covers all response types', () => {
-    const approved: t.ToolApprovalResponse = { approved: true };
-    expect(approved.approved).toBe(true);
-
-    const denied: t.ToolApprovalResponse = {
-      approved: false,
-      feedback: 'Too risky',
-    };
-    expect(denied.feedback).toBe('Too risky');
-
-    const edited: t.ToolApprovalResponse = {
-      approved: true,
-      modifiedArgs: { to: 'corrected@example.com' },
-    };
-    expect(edited.modifiedArgs).toBeDefined();
-  });
-
-  test('ToolApprovalRule function receives toolName as first argument', () => {
-    // Verify the function-based policy can use toolName for classification
-    const actionPrefixes = ['send', 'create', 'delete', 'post'];
-    const policy: t.ToolApprovalRule = (
-      toolName: string,
-      _args: Record<string, unknown>
-    ) => {
-      const lower = toolName.toLowerCase();
-      return actionPrefixes.some((prefix) => lower.includes(prefix));
-    };
-
-    // Action tools should require approval
-    expect(policy('send_email', { to: 'a@b.com' })).toBe(true);
-    expect(policy('create_document', {})).toBe(true);
-    expect(policy('delete_file', {})).toBe(true);
-    expect(policy('post_message', {})).toBe(true);
-
-    // Read-only tools should not
-    expect(policy('search', { query: 'test' })).toBe(false);
-    expect(policy('list_emails', {})).toBe(false);
-    expect(policy('get_calendar', {})).toBe(false);
+    // Second tool also requires approval — resume again
+    await compiled.invoke(
+      new Command({ resume: { approved: true } as t.ToolApprovalResponse }),
+      config
+    );
 
-    const config: t.ToolApprovalConfig = {
-      defaultPolicy: policy,
-      executionContext: 'interactive',
-    };
-    expect(typeof config.defaultPolicy).toBe('function');
+    // Notifications fire on every re-execution (interrupt() replays the node),
+    // so we check unique tool names rather than total count.
+    const uniqueTools = [...new Set(notifications.map((e) => e.toolName))].sort();
+    expect(uniqueTools).toEqual(['echo', 'send_email']);
   });
 });