illuma-agents 1.0.20 → 1.0.22

package/src/tools/BrowserInterruptTools.ts

@@ -0,0 +1,571 @@
+/**
+ * Browser Automation Tools with LangGraph Interrupt Support
+ * 
+ * These tools use LangGraph's interrupt() mechanism to pause execution
+ * and wait for the browser extension to execute actions and return results.
+ * 
+ * Flow:
+ * 1. Agent calls browser tool (e.g., browser_navigate)
+ * 2. Tool calls interrupt() with action details
+ * 3. Graph pauses and returns interrupt to client
+ * 4. Extension executes action in browser
+ * 5. Extension sends resume Command with actual result
+ * 6. Graph continues with real browser data
+ * 
+ * This enables proper chain-of-thought with browser context because
+ * the agent receives ACTUAL results (page elements, screenshots, etc.)
+ * instead of placeholder acknowledgments.
+ */
+
+import { z } from 'zod';
+import { interrupt } from '@langchain/langgraph';
+import { tool, DynamicStructuredTool } from '@langchain/core/tools';
+
+// ============================================
+// Browser Interrupt Types
+// ============================================
+
+/**
+ * Interrupt payload sent to the extension
+ */
+export interface BrowserInterrupt {
+  /** Type of browser action to execute */
+  type: 'browser_interrupt';
+  /** The specific action to perform */
+  action: BrowserAction;
+  /** Unique ID for this interrupt (for matching resume) */
+  interruptId: string;
+}
+
+/**
+ * Browser action types
+ */
+export type BrowserAction = 
+  | { type: 'navigate'; url: string; reason?: string }
+  | { type: 'click'; index?: number; coordinates?: { x: number; y: number }; reason?: string }
+  | { type: 'type'; index: number; text: string; clear?: boolean; pressEnter?: boolean }
+  | { type: 'scroll'; direction: 'up' | 'down' | 'left' | 'right'; amount?: number }
+  | { type: 'extract'; query?: string; selector?: string }
+  | { type: 'hover'; index: number }
+  | { type: 'wait'; duration?: number; reason?: string }
+  | { type: 'back'; reason?: string }
+  | { type: 'screenshot'; fullPage?: boolean }
+  | { type: 'get_page_state'; reason?: string };
+
+/**
+ * Result returned from extension after executing action
+ */
+export interface BrowserActionResult {
+  success: boolean;
+  error?: string;
+  /** Page state after action (for navigate, click, get_page_state) */
+  pageState?: {
+    url: string;
+    title: string;
+    /** Formatted element list for LLM */
+    elementList: string;
+    elementCount: number;
+    scrollPosition: number;
+    scrollHeight: number;
+    viewportHeight: number;
+  };
+  /** Screenshot data URL */
+  screenshot?: string;
+  /** Extracted content */
+  extractedContent?: string;
+  /** Any additional data */
+  data?: unknown;
+}
+
+// ============================================
+// Tool Schemas (same as BrowserTools.ts)
+// ============================================
+
+const BrowserClickSchema = z.object({
+  index: z.number().optional().describe(
+    'The index of the element to click, as shown in the page context'
+  ),
+  coordinates: z.object({
+    x: z.number().describe('X coordinate in viewport pixels'),
+    y: z.number().describe('Y coordinate in viewport pixels'),
+  }).optional().describe('Coordinates for clicking by position'),
+  reason: z.string().optional().describe('Why you are clicking this element'),
+});
+
+const BrowserTypeSchema = z.object({
+  index: z.number().describe('The index of the input element to type into'),
+  text: z.string().describe('The text to type'),
+  clear: z.boolean().optional().describe('Clear existing content first'),
+  pressEnter: z.boolean().optional().describe('Press Enter after typing'),
+});
+
+const BrowserNavigateSchema = z.object({
+  url: z.string().describe('The URL to navigate to'),
+  reason: z.string().optional().describe('Why navigating to this URL'),
+});
+
+const BrowserScrollSchema = z.object({
+  direction: z.enum(['up', 'down', 'left', 'right']).describe('Scroll direction'),
+  amount: z.number().optional().describe('Pixels to scroll (default: 500)'),
+});
+
+const BrowserExtractSchema = z.object({
+  query: z.string().optional().describe('Query to filter extracted content'),
+  selector: z.string().optional().describe('CSS selector to extract from'),
+});
+
+const BrowserHoverSchema = z.object({
+  index: z.number().describe('Element index to hover over'),
+});
+
+const BrowserWaitSchema = z.object({
+  duration: z.number().optional().describe('Milliseconds to wait (default: 1000)'),
+  reason: z.string().optional().describe('Why waiting'),
+});
+
+const BrowserGoBackSchema = z.object({
+  reason: z.string().optional().describe('Why going back'),
+});
+
+const BrowserScreenshotSchema = z.object({
+  fullPage: z.boolean().optional().describe('Capture full page vs viewport'),
+});
+
+const BrowserGetPageStateSchema = z.object({
+  reason: z.string().optional().describe('Why getting page state'),
+});
+
+// ============================================
+// Helper to generate interrupt IDs
+// ============================================
+
+let interruptCounter = 0;
+function generateInterruptId(): string {
+  return `browser_${Date.now()}_${++interruptCounter}`;
+}
+
+// ============================================
+// Interrupt-based Tool Implementations
+// ============================================
+
+/**
+ * Navigate tool - navigates to URL and returns page state
+ */
+export function createBrowserNavigateInterruptTool(): DynamicStructuredTool<typeof BrowserNavigateSchema> {
+  return tool<typeof BrowserNavigateSchema>(
+    async ({ url, reason }) => {
+      const interruptId = generateInterruptId();
+      
+      // This will pause the graph and wait for extension to provide result
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'navigate', url, reason },
+        interruptId,
+      });
+      
+      // When resumed, result contains actual page state
+      if (!result.success) {
+        return `Navigation failed: ${result.error}`;
+      }
+      
+      if (result.pageState) {
+        return `Successfully navigated to ${result.pageState.url} (${result.pageState.title})
+
+## Interactive Elements (${result.pageState.elementCount} total)
+${result.pageState.elementList}
+
+Viewport: ${result.pageState.viewportHeight}px, Scroll: ${result.pageState.scrollPosition}/${result.pageState.scrollHeight}px`;
+      }
+      
+      return `Successfully navigated to ${url}`;
+    },
+    {
+      name: 'browser_navigate',
+      description: `Navigate to a URL. Returns the page state with interactive elements after navigation completes.
+
+Example: browser_navigate({ url: "https://www.amazon.com" })
+Returns: Page title, URL, and list of interactive elements with their [index] numbers.`,
+      schema: BrowserNavigateSchema,
+    }
+  );
+}
+
+/**
+ * Click tool - clicks element and returns updated state
+ */
+export function createBrowserClickInterruptTool(): DynamicStructuredTool<typeof BrowserClickSchema> {
+  return tool<typeof BrowserClickSchema>(
+    async ({ index, coordinates, reason }) => {
+      const interruptId = generateInterruptId();
+      
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'click', index, coordinates, reason },
+        interruptId,
+      });
+      
+      if (!result.success) {
+        return `Click failed: ${result.error}`;
+      }
+      
+      // If click caused navigation, return new page state
+      if (result.pageState) {
+        return `Clicked element. Page updated:
+
+URL: ${result.pageState.url}
+Title: ${result.pageState.title}
+
+## Interactive Elements (${result.pageState.elementCount} total)
+${result.pageState.elementList}`;
+      }
+      
+      return `Successfully clicked element${index !== undefined ? ` [${index}]` : ''}`;
+    },
+    {
+      name: 'browser_click',
+      description: `Click an element by index or coordinates.
+
+Use the [index] number from the interactive elements list.
+Example: browser_click({ index: 5 }) to click element [5]`,
+      schema: BrowserClickSchema,
+    }
+  );
+}
+
+/**
+ * Type tool - types text into input field
+ */
+export function createBrowserTypeInterruptTool(): DynamicStructuredTool<typeof BrowserTypeSchema> {
+  return tool<typeof BrowserTypeSchema>(
+    async ({ index, text, clear, pressEnter }) => {
+      const interruptId = generateInterruptId();
+      
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'type', index, text, clear, pressEnter },
+        interruptId,
+      });
+      
+      if (!result.success) {
+        return `Type failed: ${result.error}`;
+      }
+      
+      // If typing + enter caused navigation (e.g., search), return new state
+      if (result.pageState) {
+        return `Typed "${text}"${pressEnter ? ' and pressed Enter' : ''}. Page updated:
+
+URL: ${result.pageState.url}
+Title: ${result.pageState.title}
+
+## Interactive Elements (${result.pageState.elementCount} total)
+${result.pageState.elementList}`;
+      }
+      
+      return `Successfully typed "${text}" into element [${index}]${pressEnter ? ' and pressed Enter' : ''}`;
+    },
+    {
+      name: 'browser_type',
+      description: `Type text into an input field.
+
+Use the [index] from interactive elements list. Look for <input> elements.
+Set pressEnter: true to submit after typing (for search fields).
+
+Example: browser_type({ index: 3, text: "laptop 16gb ram", pressEnter: true })`,
+      schema: BrowserTypeSchema,
+    }
+  );
+}
+
+/**
+ * Get page state tool - captures current page elements
+ */
+export function createBrowserGetPageStateInterruptTool(): DynamicStructuredTool<typeof BrowserGetPageStateSchema> {
+  return tool<typeof BrowserGetPageStateSchema>(
+    async ({ reason }) => {
+      const interruptId = generateInterruptId();
+      
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'get_page_state', reason },
+        interruptId,
+      });
+      
+      if (!result.success) {
+        return `Failed to get page state: ${result.error}`;
+      }
+      
+      if (result.pageState) {
+        return `## Current Page
+URL: ${result.pageState.url}
+Title: ${result.pageState.title}
+
+## Interactive Elements (${result.pageState.elementCount} total)
+${result.pageState.elementList}
+
+Viewport: ${result.pageState.viewportHeight}px
+Scroll: ${result.pageState.scrollPosition}/${result.pageState.scrollHeight}px`;
+      }
+      
+      return 'Page state captured but no elements found.';
+    },
+    {
+      name: 'browser_get_page_state',
+      description: `Get the current page state with all interactive elements.
+
+Returns the list of clickable/typeable elements with their [index] numbers.
+Use this to see what elements are available on the current page.`,
+      schema: BrowserGetPageStateSchema,
+    }
+  );
+}
+
+/**
+ * Scroll tool
+ */
+export function createBrowserScrollInterruptTool(): DynamicStructuredTool<typeof BrowserScrollSchema> {
+  return tool<typeof BrowserScrollSchema>(
+    async ({ direction, amount }) => {
+      const interruptId = generateInterruptId();
+      
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'scroll', direction, amount },
+        interruptId,
+      });
+      
+      if (!result.success) {
+        return `Scroll failed: ${result.error}`;
+      }
+      
+      if (result.pageState) {
+        return `Scrolled ${direction}. New elements visible:
+
+## Interactive Elements (${result.pageState.elementCount} total)
+${result.pageState.elementList}
+
+Scroll: ${result.pageState.scrollPosition}/${result.pageState.scrollHeight}px`;
+      }
+      
+      return `Successfully scrolled ${direction}${amount ? ` ${amount}px` : ''}`;
+    },
+    {
+      name: 'browser_scroll',
+      description: `Scroll the page in a direction.
+
+Example: browser_scroll({ direction: "down", amount: 500 })`,
+      schema: BrowserScrollSchema,
+    }
+  );
+}
+
+/**
+ * Extract tool
+ */
+export function createBrowserExtractInterruptTool(): DynamicStructuredTool<typeof BrowserExtractSchema> {
+  return tool<typeof BrowserExtractSchema>(
+    async ({ query, selector }) => {
+      const interruptId = generateInterruptId();
+      
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'extract', query, selector },
+        interruptId,
+      });
+      
+      if (!result.success) {
+        return `Extract failed: ${result.error}`;
+      }
+      
+      return result.extractedContent || 'No content extracted.';
+    },
+    {
+      name: 'browser_extract',
+      description: `Extract text content from the page.
+
+Example: browser_extract({ query: "price" })`,
+      schema: BrowserExtractSchema,
+    }
+  );
+}
+
+/**
+ * Hover tool
+ */
+export function createBrowserHoverInterruptTool(): DynamicStructuredTool<typeof BrowserHoverSchema> {
+  return tool<typeof BrowserHoverSchema>(
+    async ({ index }) => {
+      const interruptId = generateInterruptId();
+      
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'hover', index },
+        interruptId,
+      });
+      
+      if (!result.success) {
+        return `Hover failed: ${result.error}`;
+      }
+      
+      return `Successfully hovered over element [${index}]`;
+    },
+    {
+      name: 'browser_hover',
+      description: `Hover over an element to reveal tooltips or menus.`,
+      schema: BrowserHoverSchema,
+    }
+  );
+}
+
+/**
+ * Wait tool
+ */
+export function createBrowserWaitInterruptTool(): DynamicStructuredTool<typeof BrowserWaitSchema> {
+  return tool<typeof BrowserWaitSchema>(
+    async ({ duration, reason }) => {
+      const interruptId = generateInterruptId();
+      
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'wait', duration, reason },
+        interruptId,
+      });
+      
+      if (!result.success) {
+        return `Wait failed: ${result.error}`;
+      }
+      
+      return `Waited ${duration || 1000}ms`;
+    },
+    {
+      name: 'browser_wait',
+      description: `Wait for a duration before next action.`,
+      schema: BrowserWaitSchema,
+    }
+  );
+}
+
+/**
+ * Go back tool
+ */
+export function createBrowserGoBackInterruptTool(): DynamicStructuredTool<typeof BrowserGoBackSchema> {
+  return tool<typeof BrowserGoBackSchema>(
+    async ({ reason }) => {
+      const interruptId = generateInterruptId();
+      
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'back', reason },
+        interruptId,
+      });
+      
+      if (!result.success) {
+        return `Go back failed: ${result.error}`;
+      }
+      
+      if (result.pageState) {
+        return `Went back to: ${result.pageState.url}
+
+## Interactive Elements (${result.pageState.elementCount} total)
+${result.pageState.elementList}`;
+      }
+      
+      return 'Successfully went back';
+    },
+    {
+      name: 'browser_back',
+      description: `Go back to the previous page in history.`,
+      schema: BrowserGoBackSchema,
+    }
+  );
+}
+
+/**
+ * Screenshot tool
+ */
+export function createBrowserScreenshotInterruptTool(): DynamicStructuredTool<typeof BrowserScreenshotSchema> {
+  return tool<typeof BrowserScreenshotSchema>(
+    async ({ fullPage }) => {
+      const interruptId = generateInterruptId();
+      
+      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
+        type: 'browser_interrupt',
+        action: { type: 'screenshot', fullPage },
+        interruptId,
+      });
+      
+      if (!result.success) {
+        return `Screenshot failed: ${result.error}`;
+      }
+      
+      if (result.screenshot) {
+        return `Screenshot captured. [Image data available]`;
+      }
+      
+      return 'Screenshot captured';
+    },
+    {
+      name: 'browser_screenshot',
+      description: `Capture a screenshot of the current page.`,
+      schema: BrowserScreenshotSchema,
+    }
+  );
+}
+
+// ============================================
+// Tool Collection
+// ============================================
+
+export const EBrowserInterruptTools = {
+  CLICK: 'browser_click',
+  TYPE: 'browser_type',
+  NAVIGATE: 'browser_navigate',
+  SCROLL: 'browser_scroll',
+  EXTRACT: 'browser_extract',
+  HOVER: 'browser_hover',
+  WAIT: 'browser_wait',
+  BACK: 'browser_back',
+  SCREENSHOT: 'browser_screenshot',
+  GET_PAGE_STATE: 'browser_get_page_state',
+} as const;
+
+export const BROWSER_INTERRUPT_TOOL_NAMES = Object.values(EBrowserInterruptTools);
+
+export type BrowserInterruptToolName = typeof BROWSER_INTERRUPT_TOOL_NAMES[number];
+
+export function isBrowserInterruptToolCall(toolName: string): toolName is BrowserInterruptToolName {
+  return BROWSER_INTERRUPT_TOOL_NAMES.includes(toolName as BrowserInterruptToolName);
+}
+
+/**
+ * Create all interrupt-based browser tools
+ * 
+ * Use these when the client is a browser extension that can:
+ * 1. Detect browser_interrupt events in the stream
+ * 2. Execute browser actions locally
+ * 3. Send Command({ resume: result }) to continue the graph
+ */
+export function createBrowserInterruptTools(): DynamicStructuredTool[] {
+  return [
+    createBrowserNavigateInterruptTool(),
+    createBrowserClickInterruptTool(),
+    createBrowserTypeInterruptTool(),
+    createBrowserGetPageStateInterruptTool(),
+    createBrowserScrollInterruptTool(),
+    createBrowserExtractInterruptTool(),
+    createBrowserHoverInterruptTool(),
+    createBrowserWaitInterruptTool(),
+    createBrowserGoBackInterruptTool(),
+    createBrowserScreenshotInterruptTool(),
+  ];
+}
+
+/**
+ * Check if an interrupt is a browser interrupt
+ */
+export function isBrowserInterrupt(value: unknown): value is BrowserInterrupt {
+  return (
+    typeof value === 'object' &&
+    value !== null &&
+    (value as BrowserInterrupt).type === 'browser_interrupt'
+  );
+}

package/src/tools/BrowserTools.test.ts

@@ -8,6 +8,7 @@ import {
   createBrowserWaitTool,
   createBrowserGoBackTool,
   createBrowserScreenshotTool,
+  createBrowserGetPageStateTool,
   createBrowserTools,
   EBrowserTools,
   BROWSER_TOOL_NAMES,
@@ -31,10 +32,11 @@ describe('BrowserTools', () => {
       expect(EBrowserTools.WAIT).toBe('browser_wait');
       expect(EBrowserTools.BACK).toBe('browser_back');
       expect(EBrowserTools.SCREENSHOT).toBe('browser_screenshot');
+      expect(EBrowserTools.GET_PAGE_STATE).toBe('browser_get_page_state');
     });
 
-    it('should have exactly 9 tool names', () => {
-      expect(Object.keys(EBrowserTools)).toHaveLength(9);
+    it('should have exactly 10 tool names', () => {
+      expect(Object.keys(EBrowserTools)).toHaveLength(10);
     });
   });
 
@@ -52,10 +54,11 @@ describe('BrowserTools', () => {
       expect(BROWSER_TOOL_NAMES).toContain(EBrowserTools.WAIT);
       expect(BROWSER_TOOL_NAMES).toContain(EBrowserTools.BACK);
       expect(BROWSER_TOOL_NAMES).toContain(EBrowserTools.SCREENSHOT);
+      expect(BROWSER_TOOL_NAMES).toContain(EBrowserTools.GET_PAGE_STATE);
     });
 
-    it('should have exactly 9 entries', () => {
-      expect(BROWSER_TOOL_NAMES).toHaveLength(9);
+    it('should have exactly 10 entries', () => {
+      expect(BROWSER_TOOL_NAMES).toHaveLength(10);
     });
   });
 
@@ -73,6 +76,7 @@ describe('BrowserTools', () => {
       expect(isBrowserToolCall('browser_wait')).toBe(true);
       expect(isBrowserToolCall('browser_back')).toBe(true);
       expect(isBrowserToolCall('browser_screenshot')).toBe(true);
+      expect(isBrowserToolCall('browser_get_page_state')).toBe(true);
     });
 
     it('should return false for non-browser tool names', () => {
@@ -406,13 +410,40 @@ describe('BrowserTools', () => {
     });
   });
 
+  describe('createBrowserGetPageStateTool', () => {
+    it('should create a tool with correct name', () => {
+      const tool = createBrowserGetPageStateTool();
+      expect(tool.name).toBe(EBrowserTools.GET_PAGE_STATE);
+    });
+
+    it('should return browser_action for get page state', async () => {
+      const tool = createBrowserGetPageStateTool();
+      const result = await tool.invoke({ reason: 'checking elements' });
+      const parsed = JSON.parse(result);
+      
+      expect(parsed.type).toBe('browser_action');
+      expect(parsed.action.type).toBe('get_page_state');
+      expect(parsed.action.reason).toBe('checking elements');
+      expect(parsed.requiresBrowserExecution).toBe(true);
+    });
+
+    it('should work with empty input', async () => {
+      const tool = createBrowserGetPageStateTool();
+      const result = await tool.invoke({});
+      const parsed = JSON.parse(result);
+      
+      expect(parsed.type).toBe('browser_action');
+      expect(parsed.action.type).toBe('get_page_state');
+    });
+  });
+
   // ============================================
   // createBrowserTools Tests
   // ============================================
   describe('createBrowserTools', () => {
-    it('should create all 9 tools by default', () => {
+    it('should create all 10 tools by default', () => {
       const tools = createBrowserTools();
-      expect(tools).toHaveLength(9);
+      expect(tools).toHaveLength(10);
     });
 
     it('should create tools with correct names', () => {
@@ -428,6 +459,7 @@ describe('BrowserTools', () => {
       expect(toolNames).toContain(EBrowserTools.WAIT);
       expect(toolNames).toContain(EBrowserTools.BACK);
       expect(toolNames).toContain(EBrowserTools.SCREENSHOT);
+      expect(toolNames).toContain(EBrowserTools.GET_PAGE_STATE);
     });
 
     it('should allow disabling specific tools', () => {
@@ -455,6 +487,7 @@ describe('BrowserTools', () => {
         enableWait: false,
         enableBack: false,
         enableScreenshot: false,
+        enableGetPageState: false,
       };
       const tools = createBrowserTools(config);
       expect(tools).toHaveLength(0);
@@ -471,6 +504,7 @@ describe('BrowserTools', () => {
         enableWait: false,
         enableBack: false,
         enableScreenshot: false,
+        enableGetPageState: false,
       };
       const tools = createBrowserTools(config);
       expect(tools).toHaveLength(2);
@@ -499,6 +533,7 @@ describe('BrowserTools', () => {
         { tool: EBrowserTools.WAIT, input: {} },
         { tool: EBrowserTools.BACK, input: {} },
         { tool: EBrowserTools.SCREENSHOT, input: {} },
+        { tool: EBrowserTools.GET_PAGE_STATE, input: {} },
       ];
       
       for (const { tool: toolName, input } of testInputs) {