@illuma-ai/agents 1.0.82 → 1.0.84

package/src/tools/__tests__/ToolApproval.test.ts

@@ -0,0 +1,699 @@
+/**
+ * Tests for Human-in-the-Loop (HITL) tool approval in ToolNode.
+ *
+ * Tests the approval policy evaluation and the event-driven approval flow.
+ * The HITL system dispatches ON_TOOL_APPROVAL_REQUIRED events that the host
+ * (Ranger) handles by showing UI and resolving/rejecting the promise.
+ */
+import { tool } from '@langchain/core/tools';
+import { z } from 'zod';
+import {
+  HumanMessage,
+  AIMessage,
+  ToolMessage,
+} from '@langchain/core/messages';
+import {
+  StateGraph,
+  Annotation,
+  messagesStateReducer,
+  START,
+  END,
+} from '@langchain/langgraph';
+import type { BaseMessage } from '@langchain/core/messages';
+import type { RunnableConfig } from '@langchain/core/runnables';
+import type * as t from '@/types';
+import { ToolNode, toolsCondition } from '@/tools/ToolNode';
+import { GraphEvents } from '@/common';
+
+// ============================================================================
+// Test Fixtures
+// ============================================================================
+
+/** Simple echo tool that returns its input */
+const echoTool = tool(
+  async ({ message }: { message: string }) => {
+    return `Echo: ${message}`;
+  },
+  {
+    name: 'echo',
+    description: 'Echoes the input message',
+    schema: z.object({
+      message: z.string().describe('The message to echo'),
+    }),
+  }
+);
+
+/** Simulated "send email" tool */
+const sendEmailTool = tool(
+  async ({ to, subject }: { to: string; subject: string }) => {
+    return JSON.stringify({ status: 'sent', to, subject });
+  },
+  {
+    name: 'send_email',
+    description: 'Sends an email',
+    schema: z.object({
+      to: z.string().describe('Recipient email'),
+      subject: z.string().describe('Email subject'),
+    }),
+  }
+);
+
+/** Simulated "search" tool - typically safe, no approval needed */
+const searchTool = tool(
+  async ({ query }: { query: string }) => {
+    return JSON.stringify({ results: [`Result for: ${query}`] });
+  },
+  {
+    name: 'search',
+    description: 'Searches for information',
+    schema: z.object({
+      query: z.string().describe('Search query'),
+    }),
+  }
+);
+
+/**
+ * Helper to build a graph with HITL support for integration testing.
+ * Returns the compiled graph and a way to intercept approval events.
+ */
+function createTestGraph(
+  tools: t.GenericTool[],
+  toolApprovalConfig: t.ToolApprovalConfig,
+  modelResponses: string[][],
+  approvalHandler: (event: t.ToolApprovalEvent) => void
+) {
+  const StateAnnotation = Annotation.Root({
+    messages: Annotation<BaseMessage[]>({
+      reducer: messagesStateReducer,
+      default: () => [],
+    }),
+  });
+
+  let responseIndex = 0;
+
+  /** Fake model node that returns pre-defined tool calls or text */
+  const callModel = async (state: { messages: BaseMessage[] }) => {
+    const responses = modelResponses[responseIndex] ?? ['Done.'];
+    responseIndex++;
+
+    // Check if the response is a tool call instruction (JSON format)
+    if (responses.length === 1 && responses[0].startsWith('{')) {
+      const parsed = JSON.parse(responses[0]);
+      return {
+        messages: [
+          new AIMessage({
+            content: '',
+            tool_calls: [parsed],
+          }),
+        ],
+      };
+    }
+
+    return {
+      messages: [new AIMessage({ content: responses.join(' ') })],
+    };
+  };
+
+  const toolNode = new ToolNode<typeof StateAnnotation.State>({
+    tools,
+    toolApprovalConfig,
+  });
+
+  const routeMessage = (state: typeof StateAnnotation.State) => {
+    return toolsCondition(state, 'tools');
+  };
+
+  const workflow = new StateGraph(StateAnnotation)
+    .addNode('agent', callModel)
+    .addNode('tools', toolNode)
+    .addEdge(START, 'agent')
+    .addConditionalEdges('agent', routeMessage)
+    .addEdge('tools', 'agent');
+
+  const compiled = workflow.compile();
+
+  // Create a config with custom event handler to intercept approval requests
+  const config: Partial<RunnableConfig> & { version: 'v1' | 'v2' } = {
+    version: 'v2',
+    callbacks: [
+      {
+        handleCustomEvent: async (
+          eventName: string,
+          data: unknown
+        ): Promise<void> => {
+          if (eventName === GraphEvents.ON_TOOL_APPROVAL_REQUIRED) {
+            approvalHandler(data as t.ToolApprovalEvent);
+          }
+        },
+      },
+    ],
+  };
+
+  return { compiled, config };
+}
+
+// ============================================================================
+// Unit Tests: ToolApprovalConfig policy evaluation
+// ============================================================================
+
+describe('HITL Tool Approval - Policy Evaluation', () => {
+  test('no approval config means no interruption', async () => {
+    // ToolNode without toolApprovalConfig should execute tools directly
+    const toolNode = new ToolNode({
+      tools: [echoTool],
+      // No toolApprovalConfig
+    });
+
+    const input = {
+      messages: [
+        new AIMessage({
+          content: '',
+          tool_calls: [
+            {
+              name: 'echo',
+              args: { message: 'hello' },
+              id: 'call_1',
+              type: 'tool_call' as const,
+            },
+          ],
+        }),
+      ],
+    };
+
+    // Should execute without interruption
+    const result = await toolNode.invoke(input);
+    expect(result.messages).toHaveLength(1);
+    expect(result.messages[0]).toBeInstanceOf(ToolMessage);
+    expect((result.messages[0] as ToolMessage).content).toContain('Echo: hello');
+  });
+
+  test('scheduled execution context bypasses all approvals', async () => {
+    // Even with defaultPolicy: 'always', scheduled should not interrupt
+    const toolNode = new ToolNode({
+      tools: [echoTool],
+      toolApprovalConfig: {
+        defaultPolicy: 'always',
+        executionContext: 'scheduled',
+      },
+    });
+
+    const input = {
+      messages: [
+        new AIMessage({
+          content: '',
+          tool_calls: [
+            {
+              name: 'echo',
+              args: { message: 'scheduled hello' },
+              id: 'call_2',
+              type: 'tool_call' as const,
+            },
+          ],
+        }),
+      ],
+    };
+
+    // Should execute without interruption because context is 'scheduled'
+    const result = await toolNode.invoke(input);
+    expect(result.messages).toHaveLength(1);
+    expect((result.messages[0] as ToolMessage).content).toContain(
+      'Echo: scheduled hello'
+    );
+  });
+
+  test('tool with "never" override skips approval even when default is "always"', async () => {
+    const toolNode = new ToolNode({
+      tools: [searchTool],
+      toolApprovalConfig: {
+        defaultPolicy: 'always',
+        executionContext: 'interactive',
+        overrides: {
+          search: 'never',
+        },
+      },
+    });
+
+    const input = {
+      messages: [
+        new AIMessage({
+          content: '',
+          tool_calls: [
+            {
+              name: 'search',
+              args: { query: 'test query' },
+              id: 'call_3',
+              type: 'tool_call' as const,
+            },
+          ],
+        }),
+      ],
+    };
+
+    // Should execute without interruption because override is 'never'
+    const result = await toolNode.invoke(input);
+    expect(result.messages).toHaveLength(1);
+    expect((result.messages[0] as ToolMessage).content).toContain(
+      'Result for: test query'
+    );
+  });
+});
+
+// ============================================================================
+// Integration Tests: Event-driven approval flow
+// ============================================================================
+
+describe('HITL Tool Approval - Event-Driven Flow', () => {
+  jest.setTimeout(15000);
+
+  test('interactive mode with "always" policy dispatches approval event and waits', async () => {
+    const approvalEvents: t.ToolApprovalEvent[] = [];
+
+    const { compiled, config } = createTestGraph(
+      [echoTool as t.GenericTool],
+      {
+        defaultPolicy: 'always',
+        executionContext: 'interactive',
+      },
+      [
+        // First model response: request tool call
+        [
+          JSON.stringify({
+            name: 'echo',
+            args: { message: 'needs approval' },
+            id: 'call_int_1',
+            type: 'tool_call',
+          }),
+        ],
+        // Second model response (after tool executes): final text
+        ['The echo returned the result.'],
+      ],
+      (event) => {
+        // Auto-approve after capturing the event
+        approvalEvents.push(event);
+        event.resolve({ approved: true });
+      }
+    );
+
+    const result = await compiled.invoke(
+      { messages: [new HumanMessage('Say hello')] },
+      config
+    );
+
+    // Verify the approval event was dispatched
+    expect(approvalEvents).toHaveLength(1);
+    expect(approvalEvents[0].type).toBe('tool_approval_required');
+    expect(approvalEvents[0].toolName).toBe('echo');
+    expect(approvalEvents[0].toolArgs).toEqual({ message: 'needs approval' });
+
+    // Verify the tool executed after approval
+    const toolMessages = result.messages.filter(
+      (m: BaseMessage) => m._getType() === 'tool'
+    );
+    expect(toolMessages.length).toBeGreaterThan(0);
+    expect(toolMessages[0].content.toString()).toContain('Echo: needs approval');
+
+    // Verify final model response
+    const lastMessage = result.messages[result.messages.length - 1];
+    expect(lastMessage.content).toContain('The echo returned the result');
+  });
+
+  test('denial stops tool execution and returns denial message', async () => {
+    const { compiled, config } = createTestGraph(
+      [sendEmailTool as t.GenericTool],
+      {
+        defaultPolicy: 'always',
+        executionContext: 'interactive',
+      },
+      [
+        // First model response: request send email
+        [
+          JSON.stringify({
+            name: 'send_email',
+            args: { to: 'user@example.com', subject: 'Test' },
+            id: 'call_deny_1',
+            type: 'tool_call',
+          }),
+        ],
+        // Second model response (after denial): acknowledge
+        ['I understand, the email was not sent.'],
+      ],
+      (event) => {
+        // Deny the tool call
+        event.resolve({ approved: false, feedback: 'Not now' });
+      }
+    );
+
+    const result = await compiled.invoke(
+      { messages: [new HumanMessage('Send an email')] },
+      config
+    );
+
+    // Find the tool message - it should be a denial
+    const toolMessages = result.messages.filter(
+      (m: BaseMessage) => m._getType() === 'tool'
+    );
+    expect(toolMessages.length).toBeGreaterThan(0);
+
+    const denialMsg = toolMessages.find((m: BaseMessage) =>
+      m.content.toString().includes('denied')
+    );
+    expect(denialMsg).toBeDefined();
+    expect(denialMsg!.content.toString()).toContain('Not now');
+  });
+
+  test('custom function override evaluates args to determine approval', async () => {
+    // Only require approval for emails to external domains
+    const approvalEvents: t.ToolApprovalEvent[] = [];
+
+    const { compiled, config } = createTestGraph(
+      [sendEmailTool as t.GenericTool],
+      {
+        defaultPolicy: 'never',
+        executionContext: 'interactive',
+        overrides: {
+          send_email: (args: Record<string, unknown>) => {
+            const to = (args.to as string) ?? '';
+            return !to.endsWith('@internal.com');
+          },
+        },
+      },
+      [
+        // Tool call to internal domain (should NOT require approval)
+        [
+          JSON.stringify({
+            name: 'send_email',
+            args: { to: 'colleague@internal.com', subject: 'Internal' },
+            id: 'call_func_1',
+            type: 'tool_call',
+          }),
+        ],
+        // Final text
+        ['Email sent to internal address.'],
+      ],
+      (event) => {
+        approvalEvents.push(event);
+        event.resolve({ approved: true });
+      }
+    );
+
+    const result = await compiled.invoke(
+      { messages: [new HumanMessage('Send internal email')] },
+      config
+    );
+
+    // No approval events should have been dispatched (internal domain)
+    expect(approvalEvents).toHaveLength(0);
+
+    // Email should have been sent directly
+    const toolMessages = result.messages.filter(
+      (m: BaseMessage) => m._getType() === 'tool'
+    );
+    expect(toolMessages.length).toBeGreaterThan(0);
+    expect(toolMessages[0].content.toString()).toContain('colleague@internal.com');
+  });
+
+  test('custom function override triggers approval for external email', async () => {
+    const approvalEvents: t.ToolApprovalEvent[] = [];
+
+    const { compiled, config } = createTestGraph(
+      [sendEmailTool as t.GenericTool],
+      {
+        defaultPolicy: 'never',
+        executionContext: 'interactive',
+        overrides: {
+          send_email: (args: Record<string, unknown>) => {
+            const to = (args.to as string) ?? '';
+            return !to.endsWith('@internal.com');
+          },
+        },
+      },
+      [
+        // Tool call to external domain (SHOULD require approval)
+        [
+          JSON.stringify({
+            name: 'send_email',
+            args: { to: 'external@gmail.com', subject: 'External' },
+            id: 'call_func_2',
+            type: 'tool_call',
+          }),
+        ],
+        // After approval
+        ['Email sent to external address.'],
+      ],
+      (event) => {
+        approvalEvents.push(event);
+        event.resolve({ approved: true });
+      }
+    );
+
+    const result = await compiled.invoke(
+      { messages: [new HumanMessage('Send external email')] },
+      config
+    );
+
+    // Approval event should have been dispatched (external domain)
+    expect(approvalEvents).toHaveLength(1);
+    expect(approvalEvents[0].toolName).toBe('send_email');
+    expect(approvalEvents[0].toolArgs.to).toBe('external@gmail.com');
+
+    // Email should have been sent after approval
+    const lastMessage = result.messages[result.messages.length - 1];
+    expect(lastMessage.content).toContain('Email sent to external address');
+  });
+
+  test('modified args are used when human edits the tool call', async () => {
+    const { compiled, config } = createTestGraph(
+      [sendEmailTool as t.GenericTool],
+      {
+        defaultPolicy: 'always',
+        executionContext: 'interactive',
+      },
+      [
+        // Model requests send email with original args
+        [
+          JSON.stringify({
+            name: 'send_email',
+            args: { to: 'wrong@example.com', subject: 'Original Subject' },
+            id: 'call_edit_1',
+            type: 'tool_call',
+          }),
+        ],
+        // After tool execution with modified args
+        ['Email sent with corrected details.'],
+      ],
+      (event) => {
+        // Approve with modified args
+        event.resolve({
+          approved: true,
+          modifiedArgs: {
+            to: 'correct@example.com',
+            subject: 'Corrected Subject',
+          },
+        });
+      }
+    );
+
+    const result = await compiled.invoke(
+      { messages: [new HumanMessage('Send email')] },
+      config
+    );
+
+    // Find the tool message to verify modified args were used
+    const toolMessages = result.messages.filter(
+      (m: BaseMessage) => m._getType() === 'tool'
+    );
+    const sentMsg = toolMessages.find(
+      (m: BaseMessage) =>
+        (m as ToolMessage).name === 'send_email' &&
+        m.content.toString().includes('correct@example.com')
+    );
+    expect(sentMsg).toBeDefined();
+    expect(sentMsg!.content.toString()).toContain('Corrected Subject');
+  });
+
+  test('scheduled context auto-approves without dispatching events', async () => {
+    const approvalEvents: t.ToolApprovalEvent[] = [];
+
+    const { compiled, config } = createTestGraph(
+      [sendEmailTool as t.GenericTool],
+      {
+        defaultPolicy: 'always',
+        executionContext: 'scheduled', // Scheduled = auto-approve
+      },
+      [
+        [
+          JSON.stringify({
+            name: 'send_email',
+            args: { to: 'user@example.com', subject: 'Scheduled Email' },
+            id: 'call_sched_1',
+            type: 'tool_call',
+          }),
+        ],
+        ['Scheduled email sent.'],
+      ],
+      (event) => {
+        approvalEvents.push(event);
+        event.resolve({ approved: true });
+      }
+    );
+
+    const result = await compiled.invoke(
+      { messages: [new HumanMessage('Send scheduled email')] },
+      config
+    );
+
+    // No approval events should be dispatched for scheduled execution
+    expect(approvalEvents).toHaveLength(0);
+
+    // Email should have been sent directly
+    const toolMessages = result.messages.filter(
+      (m: BaseMessage) => m._getType() === 'tool'
+    );
+    expect(toolMessages.length).toBeGreaterThan(0);
+    expect(toolMessages[0].content.toString()).toContain('user@example.com');
+  });
+
+  test('multiple tool calls each get individual approval', async () => {
+    const approvalEvents: t.ToolApprovalEvent[] = [];
+
+    const toolNode = new ToolNode({
+      tools: [echoTool, sendEmailTool] as t.GenericTool[],
+      toolApprovalConfig: {
+        defaultPolicy: 'always',
+        executionContext: 'interactive',
+      },
+    });
+
+    const StateAnnotation = Annotation.Root({
+      messages: Annotation<BaseMessage[]>({
+        reducer: messagesStateReducer,
+        default: () => [],
+      }),
+    });
+
+    let callCount = 0;
+    const callModel = async (_state: { messages: BaseMessage[] }) => {
+      callCount++;
+      if (callCount === 1) {
+        return {
+          messages: [
+            new AIMessage({
+              content: '',
+              tool_calls: [
+                { name: 'echo', args: { message: 'test' }, id: 'c1', type: 'tool_call' as const },
+                { name: 'send_email', args: { to: 'a@b.com', subject: 'Hi' }, id: 'c2', type: 'tool_call' as const },
+              ],
+            }),
+          ],
+        };
+      }
+      return { messages: [new AIMessage({ content: 'Both done.' })] };
+    };
+
+    const routeMessage = (state: typeof StateAnnotation.State) =>
+      toolsCondition(state, 'tools');
+
+    const workflow = new StateGraph(StateAnnotation)
+      .addNode('agent', callModel)
+      .addNode('tools', toolNode)
+      .addEdge(START, 'agent')
+      .addConditionalEdges('agent', routeMessage)
+      .addEdge('tools', 'agent');
+
+    const compiled = workflow.compile();
+    const config: Partial<RunnableConfig> & { version: 'v1' | 'v2' } = {
+      version: 'v2',
+      callbacks: [
+        {
+          handleCustomEvent: async (
+            eventName: string,
+            data: unknown
+          ): Promise<void> => {
+            if (eventName === GraphEvents.ON_TOOL_APPROVAL_REQUIRED) {
+              const event = data as t.ToolApprovalEvent;
+              approvalEvents.push(event);
+              event.resolve({ approved: true });
+            }
+          },
+        },
+      ],
+    };
+
+    const result = await compiled.invoke(
+      { messages: [new HumanMessage('Do both')] },
+      config
+    );
+
+    // Both tools should have triggered approval events
+    expect(approvalEvents).toHaveLength(2);
+    expect(approvalEvents.map((e) => e.toolName).sort()).toEqual(['echo', 'send_email']);
+
+    // Both tools should have executed
+    const toolMessages = result.messages.filter(
+      (m: BaseMessage) => m._getType() === 'tool'
+    );
+    expect(toolMessages).toHaveLength(2);
+  });
+});
+
+// ============================================================================
+// Type Tests: ToolApprovalConfig shape validation
+// ============================================================================
+
+describe('HITL Types', () => {
+  test('ToolApprovalConfig accepts valid configurations', () => {
+    const config1: t.ToolApprovalConfig = {
+      defaultPolicy: 'always',
+      executionContext: 'interactive',
+    };
+    expect(config1.defaultPolicy).toBe('always');
+
+    const config2: t.ToolApprovalConfig = {
+      defaultPolicy: 'never',
+      executionContext: 'scheduled',
+    };
+    expect(config2.executionContext).toBe('scheduled');
+
+    const config3: t.ToolApprovalConfig = {
+      defaultPolicy: 'always',
+      executionContext: 'interactive',
+      overrides: {
+        search: 'never',
+        send_email: 'always',
+        custom_tool: (args) => (args.dangerous as boolean) === true,
+      },
+    };
+    expect(config3.overrides).toBeDefined();
+    expect(config3.overrides!.search).toBe('never');
+  });
+
+  test('ToolApprovalRequest has correct shape', () => {
+    const request: t.ToolApprovalRequest = {
+      type: 'tool_approval_required',
+      toolCallId: 'call_123',
+      toolName: 'send_email',
+      toolArgs: { to: 'user@example.com' },
+      agentId: 'agent-1',
+      description: 'Send email to user',
+    };
+    expect(request.type).toBe('tool_approval_required');
+    expect(request.toolName).toBe('send_email');
+  });
+
+  test('ToolApprovalResponse covers all response types', () => {
+    const approved: t.ToolApprovalResponse = { approved: true };
+    expect(approved.approved).toBe(true);
+
+    const denied: t.ToolApprovalResponse = {
+      approved: false,
+      feedback: 'Too risky',
+    };
+    expect(denied.feedback).toBe('Too risky');
+
+    const edited: t.ToolApprovalResponse = {
+      approved: true,
+      modifiedArgs: { to: 'corrected@example.com' },
+    };
+    expect(edited.modifiedArgs).toBeDefined();
+  });
+});