@illuma-ai/agents 1.0.89 → 1.0.93

package/src/tools/__tests__/ToolApproval.test.js

@@ -0,0 +1,675 @@
+/**
+ * Tests for Human-in-the-Loop (HITL) tool approval in ToolNode.
+ *
+ * Tests the approval policy evaluation and the event-driven approval flow.
+ * The HITL system dispatches ON_TOOL_APPROVAL_REQUIRED events that the host
+ * (Ranger) handles by showing UI and resolving/rejecting the promise.
+ */
+import { tool } from '@langchain/core/tools';
+import { z } from 'zod';
+import { HumanMessage, AIMessage, ToolMessage } from '@langchain/core/messages';
+import { StateGraph, Annotation, messagesStateReducer, START, } from '@langchain/langgraph';
+import { ToolNode, toolsCondition } from '@/tools/ToolNode';
+import { GraphEvents } from '@/common';
+// ============================================================================
+// Test Fixtures
+// ============================================================================
+/** Simple echo tool that returns its input */
+const echoTool = tool(async ({ message }) => {
+    return `Echo: ${message}`;
+}, {
+    name: 'echo',
+    description: 'Echoes the input message',
+    schema: z.object({
+        message: z.string().describe('The message to echo'),
+    }),
+});
+/** Simulated "send email" tool */
+const sendEmailTool = tool(async ({ to, subject }) => {
+    return JSON.stringify({ status: 'sent', to, subject });
+}, {
+    name: 'send_email',
+    description: 'Sends an email',
+    schema: z.object({
+        to: z.string().describe('Recipient email'),
+        subject: z.string().describe('Email subject'),
+    }),
+});
+/** Simulated "search" tool - typically safe, no approval needed */
+const searchTool = tool(async ({ query }) => {
+    return JSON.stringify({ results: [`Result for: ${query}`] });
+}, {
+    name: 'search',
+    description: 'Searches for information',
+    schema: z.object({
+        query: z.string().describe('Search query'),
+    }),
+});
+/**
+ * Helper to build a graph with HITL support for integration testing.
+ * Returns the compiled graph and a way to intercept approval events.
+ */
+function createTestGraph(tools, toolApprovalConfig, modelResponses, approvalHandler) {
+    const StateAnnotation = Annotation.Root({
+        messages: Annotation({
+            reducer: messagesStateReducer,
+            default: () => [],
+        }),
+    });
+    let responseIndex = 0;
+    /** Fake model node that returns pre-defined tool calls or text */
+    const callModel = async (_state) => {
+        const responses = modelResponses[responseIndex] ?? ['Done.'];
+        responseIndex++;
+        // Check if the response is a tool call instruction (JSON format)
+        if (responses.length === 1 && responses[0].startsWith('{')) {
+            const parsed = JSON.parse(responses[0]);
+            return {
+                messages: [
+                    new AIMessage({
+                        content: '',
+                        tool_calls: [parsed],
+                    }),
+                ],
+            };
+        }
+        return {
+            messages: [new AIMessage({ content: responses.join(' ') })],
+        };
+    };
+    const toolNode = new ToolNode({
+        tools,
+        toolApprovalConfig,
+    });
+    const routeMessage = (state) => {
+        return toolsCondition(state, 'tools');
+    };
+    const workflow = new StateGraph(StateAnnotation)
+        .addNode('agent', callModel)
+        .addNode('tools', toolNode)
+        .addEdge(START, 'agent')
+        .addConditionalEdges('agent', routeMessage)
+        .addEdge('tools', 'agent');
+    const compiled = workflow.compile();
+    // Create a config with custom event handler to intercept approval requests
+    const config = {
+        version: 'v2',
+        callbacks: [
+            {
+                handleCustomEvent: async (eventName, data) => {
+                    if (eventName === GraphEvents.ON_TOOL_APPROVAL_REQUIRED) {
+                        approvalHandler(data);
+                    }
+                },
+            },
+        ],
+    };
+    return { compiled, config };
+}
+// ============================================================================
+// Unit Tests: ToolApprovalConfig policy evaluation
+// ============================================================================
+describe('HITL Tool Approval - Policy Evaluation', () => {
+    test('no approval config means no interruption', async () => {
+        // ToolNode without toolApprovalConfig should execute tools directly
+        const toolNode = new ToolNode({
+            tools: [echoTool],
+            // No toolApprovalConfig
+        });
+        const input = {
+            messages: [
+                new AIMessage({
+                    content: '',
+                    tool_calls: [
+                        {
+                            name: 'echo',
+                            args: { message: 'hello' },
+                            id: 'call_1',
+                            type: 'tool_call',
+                        },
+                    ],
+                }),
+            ],
+        };
+        // Should execute without interruption
+        const result = await toolNode.invoke(input);
+        expect(result.messages).toHaveLength(1);
+        expect(result.messages[0]).toBeInstanceOf(ToolMessage);
+        expect(result.messages[0].content).toContain('Echo: hello');
+    });
+    test('scheduled execution context bypasses all approvals', async () => {
+        // Even with defaultPolicy: 'always', scheduled should not interrupt
+        const toolNode = new ToolNode({
+            tools: [echoTool],
+            toolApprovalConfig: {
+                defaultPolicy: 'always',
+                executionContext: 'scheduled',
+            },
+        });
+        const input = {
+            messages: [
+                new AIMessage({
+                    content: '',
+                    tool_calls: [
+                        {
+                            name: 'echo',
+                            args: { message: 'scheduled hello' },
+                            id: 'call_2',
+                            type: 'tool_call',
+                        },
+                    ],
+                }),
+            ],
+        };
+        // Should execute without interruption because context is 'scheduled'
+        const result = await toolNode.invoke(input);
+        expect(result.messages).toHaveLength(1);
+        expect(result.messages[0].content).toContain('Echo: scheduled hello');
+    });
+    test('tool with "never" override skips approval even when default is "always"', async () => {
+        const toolNode = new ToolNode({
+            tools: [searchTool],
+            toolApprovalConfig: {
+                defaultPolicy: 'always',
+                executionContext: 'interactive',
+                overrides: {
+                    search: 'never',
+                },
+            },
+        });
+        const input = {
+            messages: [
+                new AIMessage({
+                    content: '',
+                    tool_calls: [
+                        {
+                            name: 'search',
+                            args: { query: 'test query' },
+                            id: 'call_3',
+                            type: 'tool_call',
+                        },
+                    ],
+                }),
+            ],
+        };
+        // Should execute without interruption because override is 'never'
+        const result = await toolNode.invoke(input);
+        expect(result.messages).toHaveLength(1);
+        expect(result.messages[0].content).toContain('Result for: test query');
+    });
+    test('browser execution context bypasses all approvals', async () => {
+        // Browser extension has no HITL UI — all approvals should be auto-granted
+        const toolNode = new ToolNode({
+            tools: [sendEmailTool],
+            toolApprovalConfig: {
+                defaultPolicy: 'always',
+                executionContext: 'browser',
+            },
+        });
+        const input = {
+            messages: [
+                new AIMessage({
+                    content: '',
+                    tool_calls: [
+                        {
+                            name: 'send_email',
+                            args: { to: 'user@example.com', subject: 'Test' },
+                            id: 'call_browser_1',
+                            type: 'tool_call',
+                        },
+                    ],
+                }),
+            ],
+        };
+        // Should execute without interruption because context is 'browser'
+        const result = await toolNode.invoke(input);
+        expect(result.messages).toHaveLength(1);
+        expect(result.messages[0].content).toContain('sent');
+    });
+    test('browser context bypasses custom function policy', async () => {
+        // Even a custom function that always returns true should be skipped in browser context
+        const alwaysRequire = () => true;
+        const toolNode = new ToolNode({
+            tools: [echoTool],
+            toolApprovalConfig: {
+                defaultPolicy: alwaysRequire,
+                executionContext: 'browser',
+            },
+        });
+        const input = {
+            messages: [
+                new AIMessage({
+                    content: '',
+                    tool_calls: [
+                        {
+                            name: 'echo',
+                            args: { message: 'browser test' },
+                            id: 'call_browser_2',
+                            type: 'tool_call',
+                        },
+                    ],
+                }),
+            ],
+        };
+        const result = await toolNode.invoke(input);
+        expect(result.messages).toHaveLength(1);
+        expect(result.messages[0].content).toContain('Echo: browser test');
+    });
+    test('browser context bypasses per-tool overrides', async () => {
+        // Even with an explicit 'always' override for the tool, browser context should skip it
+        const toolNode = new ToolNode({
+            tools: [searchTool],
+            toolApprovalConfig: {
+                defaultPolicy: 'never',
+                executionContext: 'browser',
+                overrides: {
+                    search: 'always',
+                },
+            },
+        });
+        const input = {
+            messages: [
+                new AIMessage({
+                    content: '',
+                    tool_calls: [
+                        {
+                            name: 'search',
+                            args: { query: 'browser override test' },
+                            id: 'call_browser_3',
+                            type: 'tool_call',
+                        },
+                    ],
+                }),
+            ],
+        };
+        const result = await toolNode.invoke(input);
+        expect(result.messages).toHaveLength(1);
+        expect(result.messages[0].content).toContain('Result for: browser override test');
+    });
+    test('interactive context still requires approval (browser does not)', async () => {
+        // Verify that 'interactive' context DOES require approval (contrasts with browser)
+        const approvalEvents = [];
+        const { compiled, config } = createTestGraph([sendEmailTool], {
+            defaultPolicy: 'always',
+            executionContext: 'interactive',
+        }, [
+            // First model call: request tool call
+            [
+                JSON.stringify({
+                    name: 'send_email',
+                    args: { to: 'user@example.com', subject: 'Interactive' },
+                    id: 'call_int_1',
+                    type: 'tool_call',
+                }),
+            ],
+            // Second model call: finish after tool result
+            ['Done.'],
+        ], (event) => {
+            approvalEvents.push(event);
+            // Auto-approve for test to complete
+            event.resolve({ approved: true });
+        });
+        const result = await compiled.invoke({ messages: [new HumanMessage('send an email')] }, config);
+        // Approval WAS required for interactive context
+        expect(approvalEvents.length).toBe(1);
+        expect(approvalEvents[0].toolName).toBe('send_email');
+    });
+});
+// ============================================================================
+// Integration Tests: Event-driven approval flow
+// ============================================================================
+describe('HITL Tool Approval - Event-Driven Flow', () => {
+    jest.setTimeout(15000);
+    test('interactive mode with "always" policy dispatches approval event and waits', async () => {
+        const approvalEvents = [];
+        const { compiled, config } = createTestGraph([echoTool], {
+            defaultPolicy: 'always',
+            executionContext: 'interactive',
+        }, [
+            // First model response: request tool call
+            [
+                JSON.stringify({
+                    name: 'echo',
+                    args: { message: 'needs approval' },
+                    id: 'call_int_1',
+                    type: 'tool_call',
+                }),
+            ],
+            // Second model response (after tool executes): final text
+            ['The echo returned the result.'],
+        ], (event) => {
+            // Auto-approve after capturing the event
+            approvalEvents.push(event);
+            event.resolve({ approved: true });
+        });
+        const result = await compiled.invoke({ messages: [new HumanMessage('Say hello')] }, config);
+        // Verify the approval event was dispatched
+        expect(approvalEvents).toHaveLength(1);
+        expect(approvalEvents[0].type).toBe('tool_approval_required');
+        expect(approvalEvents[0].toolName).toBe('echo');
+        expect(approvalEvents[0].toolArgs).toEqual({ message: 'needs approval' });
+        // Verify the tool executed after approval
+        const toolMessages = result.messages.filter((m) => m._getType() === 'tool');
+        expect(toolMessages.length).toBeGreaterThan(0);
+        expect(toolMessages[0].content.toString()).toContain('Echo: needs approval');
+        // Verify final model response
+        const lastMessage = result.messages[result.messages.length - 1];
+        expect(lastMessage.content).toContain('The echo returned the result');
+    });
+    test('denial stops tool execution and returns denial message', async () => {
+        const { compiled, config } = createTestGraph([sendEmailTool], {
+            defaultPolicy: 'always',
+            executionContext: 'interactive',
+        }, [
+            // First model response: request send email
+            [
+                JSON.stringify({
+                    name: 'send_email',
+                    args: { to: 'user@example.com', subject: 'Test' },
+                    id: 'call_deny_1',
+                    type: 'tool_call',
+                }),
+            ],
+            // Second model response (after denial): acknowledge
+            ['I understand, the email was not sent.'],
+        ], (event) => {
+            // Deny the tool call
+            event.resolve({ approved: false, feedback: 'Not now' });
+        });
+        const result = await compiled.invoke({ messages: [new HumanMessage('Send an email')] }, config);
+        // Find the tool message - it should be a denial
+        const toolMessages = result.messages.filter((m) => m._getType() === 'tool');
+        expect(toolMessages.length).toBeGreaterThan(0);
+        const denialMsg = toolMessages.find((m) => m.content.toString().includes('denied'));
+        expect(denialMsg).toBeDefined();
+        expect(denialMsg.content.toString()).toContain('Not now');
+        expect(denialMsg.content.toString()).toContain('BLOCKED');
+        expect(denialMsg.content.toString()).toContain('MUST NOT call');
+    });
+    test('custom function override evaluates args to determine approval', async () => {
+        // Only require approval for emails to external domains
+        const approvalEvents = [];
+        const { compiled, config } = createTestGraph([sendEmailTool], {
+            defaultPolicy: 'never',
+            executionContext: 'interactive',
+            overrides: {
+                send_email: (_toolName, args) => {
+                    const to = args.to;
+                    return !to.endsWith('@internal.com');
+                },
+            },
+        }, [
+            // Tool call to internal domain (should NOT require approval)
+            [
+                JSON.stringify({
+                    name: 'send_email',
+                    args: { to: 'colleague@internal.com', subject: 'Internal' },
+                    id: 'call_func_1',
+                    type: 'tool_call',
+                }),
+            ],
+            // Final text
+            ['Email sent to internal address.'],
+        ], (event) => {
+            approvalEvents.push(event);
+            event.resolve({ approved: true });
+        });
+        const result = await compiled.invoke({ messages: [new HumanMessage('Send internal email')] }, config);
+        // No approval events should have been dispatched (internal domain)
+        expect(approvalEvents).toHaveLength(0);
+        // Email should have been sent directly
+        const toolMessages = result.messages.filter((m) => m._getType() === 'tool');
+        expect(toolMessages.length).toBeGreaterThan(0);
+        expect(toolMessages[0].content.toString()).toContain('colleague@internal.com');
+    });
+    test('custom function override triggers approval for external email', async () => {
+        const approvalEvents = [];
+        const { compiled, config } = createTestGraph([sendEmailTool], {
+            defaultPolicy: 'never',
+            executionContext: 'interactive',
+            overrides: {
+                send_email: (_toolName, args) => {
+                    const to = args.to;
+                    return !to.endsWith('@internal.com');
+                },
+            },
+        }, [
+            // Tool call to external domain (SHOULD require approval)
+            [
+                JSON.stringify({
+                    name: 'send_email',
+                    args: { to: 'external@gmail.com', subject: 'External' },
+                    id: 'call_func_2',
+                    type: 'tool_call',
+                }),
+            ],
+            // After approval
+            ['Email sent to external address.'],
+        ], (event) => {
+            approvalEvents.push(event);
+            event.resolve({ approved: true });
+        });
+        const result = await compiled.invoke({ messages: [new HumanMessage('Send external email')] }, config);
+        // Approval event should have been dispatched (external domain)
+        expect(approvalEvents).toHaveLength(1);
+        expect(approvalEvents[0].toolName).toBe('send_email');
+        expect(approvalEvents[0].toolArgs.to).toBe('external@gmail.com');
+        // Email should have been sent after approval
+        const lastMessage = result.messages[result.messages.length - 1];
+        expect(lastMessage.content).toContain('Email sent to external address');
+    });
+    test('modified args are used when human edits the tool call', async () => {
+        const { compiled, config } = createTestGraph([sendEmailTool], {
+            defaultPolicy: 'always',
+            executionContext: 'interactive',
+        }, [
+            // Model requests send email with original args
+            [
+                JSON.stringify({
+                    name: 'send_email',
+                    args: { to: 'wrong@example.com', subject: 'Original Subject' },
+                    id: 'call_edit_1',
+                    type: 'tool_call',
+                }),
+            ],
+            // After tool execution with modified args
+            ['Email sent with corrected details.'],
+        ], (event) => {
+            // Approve with modified args
+            event.resolve({
+                approved: true,
+                modifiedArgs: {
+                    to: 'correct@example.com',
+                    subject: 'Corrected Subject',
+                },
+            });
+        });
+        const result = await compiled.invoke({ messages: [new HumanMessage('Send email')] }, config);
+        // Find the tool message to verify modified args were used
+        const toolMessages = result.messages.filter((m) => m._getType() === 'tool');
+        const sentMsg = toolMessages.find((m) => m.name === 'send_email' &&
+            m.content.toString().includes('correct@example.com'));
+        expect(sentMsg).toBeDefined();
+        expect(sentMsg.content.toString()).toContain('Corrected Subject');
+    });
+    test('scheduled context auto-approves without dispatching events', async () => {
+        const approvalEvents = [];
+        const { compiled, config } = createTestGraph([sendEmailTool], {
+            defaultPolicy: 'always',
+            executionContext: 'scheduled', // Scheduled = auto-approve
+        }, [
+            [
+                JSON.stringify({
+                    name: 'send_email',
+                    args: { to: 'user@example.com', subject: 'Scheduled Email' },
+                    id: 'call_sched_1',
+                    type: 'tool_call',
+                }),
+            ],
+            ['Scheduled email sent.'],
+        ], (event) => {
+            approvalEvents.push(event);
+            event.resolve({ approved: true });
+        });
+        const result = await compiled.invoke({ messages: [new HumanMessage('Send scheduled email')] }, config);
+        // No approval events should be dispatched for scheduled execution
+        expect(approvalEvents).toHaveLength(0);
+        // Email should have been sent directly
+        const toolMessages = result.messages.filter((m) => m._getType() === 'tool');
+        expect(toolMessages.length).toBeGreaterThan(0);
+        expect(toolMessages[0].content.toString()).toContain('user@example.com');
+    });
+    test('multiple tool calls each get individual approval', async () => {
+        const approvalEvents = [];
+        const toolNode = new ToolNode({
+            tools: [echoTool, sendEmailTool],
+            toolApprovalConfig: {
+                defaultPolicy: 'always',
+                executionContext: 'interactive',
+            },
+        });
+        const StateAnnotation = Annotation.Root({
+            messages: Annotation({
+                reducer: messagesStateReducer,
+                default: () => [],
+            }),
+        });
+        let callCount = 0;
+        const callModel = async (_state) => {
+            callCount++;
+            if (callCount === 1) {
+                return {
+                    messages: [
+                        new AIMessage({
+                            content: '',
+                            tool_calls: [
+                                {
+                                    name: 'echo',
+                                    args: { message: 'test' },
+                                    id: 'c1',
+                                    type: 'tool_call',
+                                },
+                                {
+                                    name: 'send_email',
+                                    args: { to: 'a@b.com', subject: 'Hi' },
+                                    id: 'c2',
+                                    type: 'tool_call',
+                                },
+                            ],
+                        }),
+                    ],
+                };
+            }
+            return { messages: [new AIMessage({ content: 'Both done.' })] };
+        };
+        const routeMessage = (state) => toolsCondition(state, 'tools');
+        const workflow = new StateGraph(StateAnnotation)
+            .addNode('agent', callModel)
+            .addNode('tools', toolNode)
+            .addEdge(START, 'agent')
+            .addConditionalEdges('agent', routeMessage)
+            .addEdge('tools', 'agent');
+        const compiled = workflow.compile();
+        const config = {
+            version: 'v2',
+            callbacks: [
+                {
+                    handleCustomEvent: async (eventName, data) => {
+                        if (eventName === GraphEvents.ON_TOOL_APPROVAL_REQUIRED) {
+                            const event = data;
+                            approvalEvents.push(event);
+                            event.resolve({ approved: true });
+                        }
+                    },
+                },
+            ],
+        };
+        const result = await compiled.invoke({ messages: [new HumanMessage('Do both')] }, config);
+        // Both tools should have triggered approval events
+        expect(approvalEvents).toHaveLength(2);
+        expect(approvalEvents.map((e) => e.toolName).sort()).toEqual([
+            'echo',
+            'send_email',
+        ]);
+        // Both tools should have executed
+        const toolMessages = result.messages.filter((m) => m._getType() === 'tool');
+        expect(toolMessages).toHaveLength(2);
+    });
+});
+// ============================================================================
+// Type Tests: ToolApprovalConfig shape validation
+// ============================================================================
+describe('HITL Types', () => {
+    test('ToolApprovalConfig accepts valid configurations', () => {
+        const config1 = {
+            defaultPolicy: 'always',
+            executionContext: 'interactive',
+        };
+        expect(config1.defaultPolicy).toBe('always');
+        const config2 = {
+            defaultPolicy: 'never',
+            executionContext: 'scheduled',
+        };
+        expect(config2.executionContext).toBe('scheduled');
+        const config3 = {
+            defaultPolicy: 'always',
+            executionContext: 'interactive',
+            overrides: {
+                search: 'never',
+                send_email: 'always',
+                custom_tool: (_toolName, args) => args.dangerous === true,
+            },
+        };
+        expect(config3.overrides).toBeDefined();
+        expect(config3.overrides.search).toBe('never');
+    });
+    test('ToolApprovalRequest has correct shape', () => {
+        const request = {
+            type: 'tool_approval_required',
+            toolCallId: 'call_123',
+            toolName: 'send_email',
+            toolArgs: { to: 'user@example.com' },
+            agentId: 'agent-1',
+            description: 'Send email to user',
+        };
+        expect(request.type).toBe('tool_approval_required');
+        expect(request.toolName).toBe('send_email');
+    });
+    test('ToolApprovalResponse covers all response types', () => {
+        const approved = { approved: true };
+        expect(approved.approved).toBe(true);
+        const denied = {
+            approved: false,
+            feedback: 'Too risky',
+        };
+        expect(denied.feedback).toBe('Too risky');
+        const edited = {
+            approved: true,
+            modifiedArgs: { to: 'corrected@example.com' },
+        };
+        expect(edited.modifiedArgs).toBeDefined();
+    });
+    test('ToolApprovalRule function receives toolName as first argument', () => {
+        // Verify the function-based policy can use toolName for classification
+        const actionPrefixes = ['send', 'create', 'delete', 'post'];
+        const policy = (toolName, _args) => {
+            const lower = toolName.toLowerCase();
+            return actionPrefixes.some((prefix) => lower.includes(prefix));
+        };
+        // Action tools should require approval
+        expect(policy('send_email', { to: 'a@b.com' })).toBe(true);
+        expect(policy('create_document', {})).toBe(true);
+        expect(policy('delete_file', {})).toBe(true);
+        expect(policy('post_message', {})).toBe(true);
+        // Read-only tools should not
+        expect(policy('search', { query: 'test' })).toBe(false);
+        expect(policy('list_emails', {})).toBe(false);
+        expect(policy('get_calendar', {})).toBe(false);
+        const config = {
+            defaultPolicy: policy,
+            executionContext: 'interactive',
+        };
+        expect(typeof config.defaultPolicy).toBe('function');
+    });
+});
+//# sourceMappingURL=ToolApproval.test.js.map