illuma-agents 1.0.22 → 1.0.24

package/src/tools/BrowserInterruptTools.ts

@@ -1,571 +0,0 @@
-/**
- * Browser Automation Tools with LangGraph Interrupt Support
- * 
- * These tools use LangGraph's interrupt() mechanism to pause execution
- * and wait for the browser extension to execute actions and return results.
- * 
- * Flow:
- * 1. Agent calls browser tool (e.g., browser_navigate)
- * 2. Tool calls interrupt() with action details
- * 3. Graph pauses and returns interrupt to client
- * 4. Extension executes action in browser
- * 5. Extension sends resume Command with actual result
- * 6. Graph continues with real browser data
- * 
- * This enables proper chain-of-thought with browser context because
- * the agent receives ACTUAL results (page elements, screenshots, etc.)
- * instead of placeholder acknowledgments.
- */
-
-import { z } from 'zod';
-import { interrupt } from '@langchain/langgraph';
-import { tool, DynamicStructuredTool } from '@langchain/core/tools';
-
-// ============================================
-// Browser Interrupt Types
-// ============================================
-
-/**
- * Interrupt payload sent to the extension
- */
-export interface BrowserInterrupt {
-  /** Type of browser action to execute */
-  type: 'browser_interrupt';
-  /** The specific action to perform */
-  action: BrowserAction;
-  /** Unique ID for this interrupt (for matching resume) */
-  interruptId: string;
-}
-
-/**
- * Browser action types
- */
-export type BrowserAction = 
-  | { type: 'navigate'; url: string; reason?: string }
-  | { type: 'click'; index?: number; coordinates?: { x: number; y: number }; reason?: string }
-  | { type: 'type'; index: number; text: string; clear?: boolean; pressEnter?: boolean }
-  | { type: 'scroll'; direction: 'up' | 'down' | 'left' | 'right'; amount?: number }
-  | { type: 'extract'; query?: string; selector?: string }
-  | { type: 'hover'; index: number }
-  | { type: 'wait'; duration?: number; reason?: string }
-  | { type: 'back'; reason?: string }
-  | { type: 'screenshot'; fullPage?: boolean }
-  | { type: 'get_page_state'; reason?: string };
-
-/**
- * Result returned from extension after executing action
- */
-export interface BrowserActionResult {
-  success: boolean;
-  error?: string;
-  /** Page state after action (for navigate, click, get_page_state) */
-  pageState?: {
-    url: string;
-    title: string;
-    /** Formatted element list for LLM */
-    elementList: string;
-    elementCount: number;
-    scrollPosition: number;
-    scrollHeight: number;
-    viewportHeight: number;
-  };
-  /** Screenshot data URL */
-  screenshot?: string;
-  /** Extracted content */
-  extractedContent?: string;
-  /** Any additional data */
-  data?: unknown;
-}
-
-// ============================================
-// Tool Schemas (same as BrowserTools.ts)
-// ============================================
-
-const BrowserClickSchema = z.object({
-  index: z.number().optional().describe(
-    'The index of the element to click, as shown in the page context'
-  ),
-  coordinates: z.object({
-    x: z.number().describe('X coordinate in viewport pixels'),
-    y: z.number().describe('Y coordinate in viewport pixels'),
-  }).optional().describe('Coordinates for clicking by position'),
-  reason: z.string().optional().describe('Why you are clicking this element'),
-});
-
-const BrowserTypeSchema = z.object({
-  index: z.number().describe('The index of the input element to type into'),
-  text: z.string().describe('The text to type'),
-  clear: z.boolean().optional().describe('Clear existing content first'),
-  pressEnter: z.boolean().optional().describe('Press Enter after typing'),
-});
-
-const BrowserNavigateSchema = z.object({
-  url: z.string().describe('The URL to navigate to'),
-  reason: z.string().optional().describe('Why navigating to this URL'),
-});
-
-const BrowserScrollSchema = z.object({
-  direction: z.enum(['up', 'down', 'left', 'right']).describe('Scroll direction'),
-  amount: z.number().optional().describe('Pixels to scroll (default: 500)'),
-});
-
-const BrowserExtractSchema = z.object({
-  query: z.string().optional().describe('Query to filter extracted content'),
-  selector: z.string().optional().describe('CSS selector to extract from'),
-});
-
-const BrowserHoverSchema = z.object({
-  index: z.number().describe('Element index to hover over'),
-});
-
-const BrowserWaitSchema = z.object({
-  duration: z.number().optional().describe('Milliseconds to wait (default: 1000)'),
-  reason: z.string().optional().describe('Why waiting'),
-});
-
-const BrowserGoBackSchema = z.object({
-  reason: z.string().optional().describe('Why going back'),
-});
-
-const BrowserScreenshotSchema = z.object({
-  fullPage: z.boolean().optional().describe('Capture full page vs viewport'),
-});
-
-const BrowserGetPageStateSchema = z.object({
-  reason: z.string().optional().describe('Why getting page state'),
-});
-
-// ============================================
-// Helper to generate interrupt IDs
-// ============================================
-
-let interruptCounter = 0;
-function generateInterruptId(): string {
-  return `browser_${Date.now()}_${++interruptCounter}`;
-}
-
-// ============================================
-// Interrupt-based Tool Implementations
-// ============================================
-
-/**
- * Navigate tool - navigates to URL and returns page state
- */
-export function createBrowserNavigateInterruptTool(): DynamicStructuredTool<typeof BrowserNavigateSchema> {
-  return tool<typeof BrowserNavigateSchema>(
-    async ({ url, reason }) => {
-      const interruptId = generateInterruptId();
-      
-      // This will pause the graph and wait for extension to provide result
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'navigate', url, reason },
-        interruptId,
-      });
-      
-      // When resumed, result contains actual page state
-      if (!result.success) {
-        return `Navigation failed: ${result.error}`;
-      }
-      
-      if (result.pageState) {
-        return `Successfully navigated to ${result.pageState.url} (${result.pageState.title})
-
-## Interactive Elements (${result.pageState.elementCount} total)
-${result.pageState.elementList}
-
-Viewport: ${result.pageState.viewportHeight}px, Scroll: ${result.pageState.scrollPosition}/${result.pageState.scrollHeight}px`;
-      }
-      
-      return `Successfully navigated to ${url}`;
-    },
-    {
-      name: 'browser_navigate',
-      description: `Navigate to a URL. Returns the page state with interactive elements after navigation completes.
-
-Example: browser_navigate({ url: "https://www.amazon.com" })
-Returns: Page title, URL, and list of interactive elements with their [index] numbers.`,
-      schema: BrowserNavigateSchema,
-    }
-  );
-}
-
-/**
- * Click tool - clicks element and returns updated state
- */
-export function createBrowserClickInterruptTool(): DynamicStructuredTool<typeof BrowserClickSchema> {
-  return tool<typeof BrowserClickSchema>(
-    async ({ index, coordinates, reason }) => {
-      const interruptId = generateInterruptId();
-      
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'click', index, coordinates, reason },
-        interruptId,
-      });
-      
-      if (!result.success) {
-        return `Click failed: ${result.error}`;
-      }
-      
-      // If click caused navigation, return new page state
-      if (result.pageState) {
-        return `Clicked element. Page updated:
-
-URL: ${result.pageState.url}
-Title: ${result.pageState.title}
-
-## Interactive Elements (${result.pageState.elementCount} total)
-${result.pageState.elementList}`;
-      }
-      
-      return `Successfully clicked element${index !== undefined ? ` [${index}]` : ''}`;
-    },
-    {
-      name: 'browser_click',
-      description: `Click an element by index or coordinates.
-
-Use the [index] number from the interactive elements list.
-Example: browser_click({ index: 5 }) to click element [5]`,
-      schema: BrowserClickSchema,
-    }
-  );
-}
-
-/**
- * Type tool - types text into input field
- */
-export function createBrowserTypeInterruptTool(): DynamicStructuredTool<typeof BrowserTypeSchema> {
-  return tool<typeof BrowserTypeSchema>(
-    async ({ index, text, clear, pressEnter }) => {
-      const interruptId = generateInterruptId();
-      
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'type', index, text, clear, pressEnter },
-        interruptId,
-      });
-      
-      if (!result.success) {
-        return `Type failed: ${result.error}`;
-      }
-      
-      // If typing + enter caused navigation (e.g., search), return new state
-      if (result.pageState) {
-        return `Typed "${text}"${pressEnter ? ' and pressed Enter' : ''}. Page updated:
-
-URL: ${result.pageState.url}
-Title: ${result.pageState.title}
-
-## Interactive Elements (${result.pageState.elementCount} total)
-${result.pageState.elementList}`;
-      }
-      
-      return `Successfully typed "${text}" into element [${index}]${pressEnter ? ' and pressed Enter' : ''}`;
-    },
-    {
-      name: 'browser_type',
-      description: `Type text into an input field.
-
-Use the [index] from interactive elements list. Look for <input> elements.
-Set pressEnter: true to submit after typing (for search fields).
-
-Example: browser_type({ index: 3, text: "laptop 16gb ram", pressEnter: true })`,
-      schema: BrowserTypeSchema,
-    }
-  );
-}
-
-/**
- * Get page state tool - captures current page elements
- */
-export function createBrowserGetPageStateInterruptTool(): DynamicStructuredTool<typeof BrowserGetPageStateSchema> {
-  return tool<typeof BrowserGetPageStateSchema>(
-    async ({ reason }) => {
-      const interruptId = generateInterruptId();
-      
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'get_page_state', reason },
-        interruptId,
-      });
-      
-      if (!result.success) {
-        return `Failed to get page state: ${result.error}`;
-      }
-      
-      if (result.pageState) {
-        return `## Current Page
-URL: ${result.pageState.url}
-Title: ${result.pageState.title}
-
-## Interactive Elements (${result.pageState.elementCount} total)
-${result.pageState.elementList}
-
-Viewport: ${result.pageState.viewportHeight}px
-Scroll: ${result.pageState.scrollPosition}/${result.pageState.scrollHeight}px`;
-      }
-      
-      return 'Page state captured but no elements found.';
-    },
-    {
-      name: 'browser_get_page_state',
-      description: `Get the current page state with all interactive elements.
-
-Returns the list of clickable/typeable elements with their [index] numbers.
-Use this to see what elements are available on the current page.`,
-      schema: BrowserGetPageStateSchema,
-    }
-  );
-}
-
-/**
- * Scroll tool
- */
-export function createBrowserScrollInterruptTool(): DynamicStructuredTool<typeof BrowserScrollSchema> {
-  return tool<typeof BrowserScrollSchema>(
-    async ({ direction, amount }) => {
-      const interruptId = generateInterruptId();
-      
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'scroll', direction, amount },
-        interruptId,
-      });
-      
-      if (!result.success) {
-        return `Scroll failed: ${result.error}`;
-      }
-      
-      if (result.pageState) {
-        return `Scrolled ${direction}. New elements visible:
-
-## Interactive Elements (${result.pageState.elementCount} total)
-${result.pageState.elementList}
-
-Scroll: ${result.pageState.scrollPosition}/${result.pageState.scrollHeight}px`;
-      }
-      
-      return `Successfully scrolled ${direction}${amount ? ` ${amount}px` : ''}`;
-    },
-    {
-      name: 'browser_scroll',
-      description: `Scroll the page in a direction.
-
-Example: browser_scroll({ direction: "down", amount: 500 })`,
-      schema: BrowserScrollSchema,
-    }
-  );
-}
-
-/**
- * Extract tool
- */
-export function createBrowserExtractInterruptTool(): DynamicStructuredTool<typeof BrowserExtractSchema> {
-  return tool<typeof BrowserExtractSchema>(
-    async ({ query, selector }) => {
-      const interruptId = generateInterruptId();
-      
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'extract', query, selector },
-        interruptId,
-      });
-      
-      if (!result.success) {
-        return `Extract failed: ${result.error}`;
-      }
-      
-      return result.extractedContent || 'No content extracted.';
-    },
-    {
-      name: 'browser_extract',
-      description: `Extract text content from the page.
-
-Example: browser_extract({ query: "price" })`,
-      schema: BrowserExtractSchema,
-    }
-  );
-}
-
-/**
- * Hover tool
- */
-export function createBrowserHoverInterruptTool(): DynamicStructuredTool<typeof BrowserHoverSchema> {
-  return tool<typeof BrowserHoverSchema>(
-    async ({ index }) => {
-      const interruptId = generateInterruptId();
-      
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'hover', index },
-        interruptId,
-      });
-      
-      if (!result.success) {
-        return `Hover failed: ${result.error}`;
-      }
-      
-      return `Successfully hovered over element [${index}]`;
-    },
-    {
-      name: 'browser_hover',
-      description: `Hover over an element to reveal tooltips or menus.`,
-      schema: BrowserHoverSchema,
-    }
-  );
-}
-
-/**
- * Wait tool
- */
-export function createBrowserWaitInterruptTool(): DynamicStructuredTool<typeof BrowserWaitSchema> {
-  return tool<typeof BrowserWaitSchema>(
-    async ({ duration, reason }) => {
-      const interruptId = generateInterruptId();
-      
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'wait', duration, reason },
-        interruptId,
-      });
-      
-      if (!result.success) {
-        return `Wait failed: ${result.error}`;
-      }
-      
-      return `Waited ${duration || 1000}ms`;
-    },
-    {
-      name: 'browser_wait',
-      description: `Wait for a duration before next action.`,
-      schema: BrowserWaitSchema,
-    }
-  );
-}
-
-/**
- * Go back tool
- */
-export function createBrowserGoBackInterruptTool(): DynamicStructuredTool<typeof BrowserGoBackSchema> {
-  return tool<typeof BrowserGoBackSchema>(
-    async ({ reason }) => {
-      const interruptId = generateInterruptId();
-      
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'back', reason },
-        interruptId,
-      });
-      
-      if (!result.success) {
-        return `Go back failed: ${result.error}`;
-      }
-      
-      if (result.pageState) {
-        return `Went back to: ${result.pageState.url}
-
-## Interactive Elements (${result.pageState.elementCount} total)
-${result.pageState.elementList}`;
-      }
-      
-      return 'Successfully went back';
-    },
-    {
-      name: 'browser_back',
-      description: `Go back to the previous page in history.`,
-      schema: BrowserGoBackSchema,
-    }
-  );
-}
-
-/**
- * Screenshot tool
- */
-export function createBrowserScreenshotInterruptTool(): DynamicStructuredTool<typeof BrowserScreenshotSchema> {
-  return tool<typeof BrowserScreenshotSchema>(
-    async ({ fullPage }) => {
-      const interruptId = generateInterruptId();
-      
-      const result = interrupt<BrowserInterrupt, BrowserActionResult>({
-        type: 'browser_interrupt',
-        action: { type: 'screenshot', fullPage },
-        interruptId,
-      });
-      
-      if (!result.success) {
-        return `Screenshot failed: ${result.error}`;
-      }
-      
-      if (result.screenshot) {
-        return `Screenshot captured. [Image data available]`;
-      }
-      
-      return 'Screenshot captured';
-    },
-    {
-      name: 'browser_screenshot',
-      description: `Capture a screenshot of the current page.`,
-      schema: BrowserScreenshotSchema,
-    }
-  );
-}
-
-// ============================================
-// Tool Collection
-// ============================================
-
-export const EBrowserInterruptTools = {
-  CLICK: 'browser_click',
-  TYPE: 'browser_type',
-  NAVIGATE: 'browser_navigate',
-  SCROLL: 'browser_scroll',
-  EXTRACT: 'browser_extract',
-  HOVER: 'browser_hover',
-  WAIT: 'browser_wait',
-  BACK: 'browser_back',
-  SCREENSHOT: 'browser_screenshot',
-  GET_PAGE_STATE: 'browser_get_page_state',
-} as const;
-
-export const BROWSER_INTERRUPT_TOOL_NAMES = Object.values(EBrowserInterruptTools);
-
-export type BrowserInterruptToolName = typeof BROWSER_INTERRUPT_TOOL_NAMES[number];
-
-export function isBrowserInterruptToolCall(toolName: string): toolName is BrowserInterruptToolName {
-  return BROWSER_INTERRUPT_TOOL_NAMES.includes(toolName as BrowserInterruptToolName);
-}
-
-/**
- * Create all interrupt-based browser tools
- * 
- * Use these when the client is a browser extension that can:
- * 1. Detect browser_interrupt events in the stream
- * 2. Execute browser actions locally
- * 3. Send Command({ resume: result }) to continue the graph
- */
-export function createBrowserInterruptTools(): DynamicStructuredTool[] {
-  return [
-    createBrowserNavigateInterruptTool(),
-    createBrowserClickInterruptTool(),
-    createBrowserTypeInterruptTool(),
-    createBrowserGetPageStateInterruptTool(),
-    createBrowserScrollInterruptTool(),
-    createBrowserExtractInterruptTool(),
-    createBrowserHoverInterruptTool(),
-    createBrowserWaitInterruptTool(),
-    createBrowserGoBackInterruptTool(),
-    createBrowserScreenshotInterruptTool(),
-  ];
-}
-
-/**
- * Check if an interrupt is a browser interrupt
- */
-export function isBrowserInterrupt(value: unknown): value is BrowserInterrupt {
-  return (
-    typeof value === 'object' &&
-    value !== null &&
-    (value as BrowserInterrupt).type === 'browser_interrupt'
-  );
-}