illuma-agents 1.0.43 → 1.0.45

package/src/tools/DesktopTools.ts

@@ -1,11 +1,6 @@
 import { z } from 'zod';
 import { tool, DynamicStructuredTool } from '@langchain/core/tools';
 
-/**
- * Type for tool configuration passed by LangChain runtime
- */
-type ToolCallConfig = { toolCall?: { id?: string } };
-
 /**
  * Desktop tool names - keep in sync with Ranger Desktop Electron app
  * These tools execute locally in the Electron app, NOT on the server
@@ -26,6 +21,12 @@ export const EDesktopTools = {
   CLIPBOARD_WRITE: 'clipboard_write',
   CLIPBOARD_PASTE: 'clipboard_paste',
   WAIT: 'computer_wait',
+  // Native UI Automation tools (Windows) - faster and more reliable than screenshot-based
+  UI_FIND_ELEMENT: 'ui_find_element',
+  UI_CLICK_ELEMENT: 'ui_click_element',
+  UI_GET_WINDOW_TREE: 'ui_get_window_tree',
+  UI_FIND_BUTTONS: 'ui_find_buttons',
+  UI_FIND_INPUTS: 'ui_find_inputs',
 } as const;
 
 export type DesktopToolName =
@@ -65,6 +66,23 @@ export interface DesktopActionResult {
   };
   mousePosition?: { x: number; y: number };
   clipboard?: string;
+  // UI Automation results
+  uiElement?: {
+    name: string;
+    automationId: string;
+    controlType: string;
+    boundingRectangle?: { x: number; y: number; width: number; height: number };
+    isEnabled: boolean;
+  };
+  uiElements?: Array<{
+    name: string;
+    automationId: string;
+    controlType: string;
+    boundingRectangle?: { x: number; y: number; width: number; height: number };
+  }>;
+  uiTree?: unknown;
+  // Generic data for extended results
+  data?: unknown;
 }
 
 /**
@@ -154,6 +172,29 @@ const WaitSchema = z.object({
   ms: z.number().describe('Milliseconds to wait'),
 });
 
+// ============ Native UI Automation Schemas (Windows) ============
+
+const UIFindElementSchema = z.object({
+  name: z.string().optional().describe('Element name/label to find (e.g., "Submit", "OK", "File")'),
+  automationId: z.string().optional().describe('Automation ID of element (unique identifier)'),
+  controlType: z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),
+});
+
+const UIClickElementSchema = z.object({
+  name: z.string().optional().describe('Element name/label to click'),
+  automationId: z.string().optional().describe('Automation ID of element to click'),
+  controlType: z.string().optional().describe('Type of control: Button, Edit, etc.'),
+  clickType: z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),
+});
+
+const UIGetWindowTreeSchema = z.object({
+  maxDepth: z.number().optional().describe('Maximum depth to traverse (default: 3)'),
+});
+
+const UIFindButtonsSchema = z.object({});
+
+const UIFindInputsSchema = z.object({});
+
 /**
  * Desktop tool response interface
  * This is what the Electron app returns after executing the action
@@ -218,44 +259,51 @@ function formatResultForLLM(
     parts.push(`**Clipboard Content:** ${result.clipboard}`);
   }
 
-  if (parts.length === 0) {
-    parts.push(`Desktop action "${action}" completed successfully.`);
+  // UI Automation results
+  if (result.uiElement) {
+    const el = result.uiElement;
+    parts.push(`**UI Element Found:**`);
+    parts.push(`  - Name: "${el.name}"`);
+    parts.push(`  - AutomationId: "${el.automationId}"`);
+    parts.push(`  - Type: ${el.controlType}`);
+    if (el.boundingRectangle) {
+      const b = el.boundingRectangle;
+      parts.push(`  - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);
+      parts.push(`  - Center: (${Math.round(b.x + b.width/2)}, ${Math.round(b.y + b.height/2)})`);
+    }
+    parts.push(`  - Enabled: ${el.isEnabled}`);
   }
 
-  return parts.join('\n');
-}
-
-/**
- * Create a tool result (either wait for callback or return marker)
- */
-async function createToolResult(
-  action: string,
-  args: Record<string, unknown>,
-  config: ToolCallConfig | undefined,
-  waitForResult?: DesktopToolCallback
-): Promise<string> {
-  const toolCallId = config?.toolCall?.id || `desktop-${Date.now()}`;
-
-  if (waitForResult) {
-    // Server context: wait for actual result from Electron app
-    try {
-      const result = await waitForResult(action, args, toolCallId);
-      return formatResultForLLM(result, action);
-    } catch (error) {
-      const errorMessage =
-        error instanceof Error ? error.message : String(error);
-      return `Desktop action "${action}" failed: ${errorMessage}`;
+  if (result.uiElements && result.uiElements.length > 0) {
+    parts.push(`**UI Elements Found (${result.uiElements.length}):**`);
+    for (const el of result.uiElements.slice(0, 20)) { // Limit to 20
+      const bounds = el.boundingRectangle ? 
+        ` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';
+      parts.push(`  - [${el.controlType}] "${el.name}"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);
+    }
+    if (result.uiElements.length > 20) {
+      parts.push(`  ... and ${result.uiElements.length - 20} more`);
     }
   }
 
-  // Non-server context: return marker for later processing
-  const response: DesktopToolResponse = {
-    requiresDesktopExecution: true,
-    action,
-    args,
-    toolCallId,
-  };
-  return JSON.stringify(response);
+  if (result.uiTree) {
+    parts.push(`**UI Tree:**`);
+    parts.push('```json');
+    parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size
+    parts.push('```');
+  }
+
+  if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {
+    // Generic data fallback
+    parts.push(`**Result:**`);
+    parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));
+  }
+
+  if (parts.length === 0) {
+    parts.push(`Desktop action "${action}" completed successfully.`);
+  }
+
+  return parts.join('\\n');
 }
 
 /**
@@ -266,275 +314,249 @@ export function createDesktopTools(
   options: CreateDesktopToolsOptions = {}
 ): DynamicStructuredTool[] {
   const { waitForResult } = options;
+  const tools: DynamicStructuredTool[] = [];
 
-  return [
-    // computer_screenshot
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.SCREENSHOT,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.SCREENSHOT,
-        description:
-          'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',
-        schema: ScreenshotSchema,
-      }
-    ),
-
-    // computer_click
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.CLICK,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.CLICK,
-        description:
-          'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',
-        schema: ClickSchema,
-      }
-    ),
-
-    // computer_double_click
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.DOUBLE_CLICK,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.DOUBLE_CLICK,
-        description:
-          'Double-click the mouse at the specified screen coordinates.',
-        schema: DoubleClickSchema,
-      }
-    ),
-
-    // computer_right_click
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.RIGHT_CLICK,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.RIGHT_CLICK,
-        description:
-          'Right-click the mouse at the specified screen coordinates to open context menus.',
-        schema: RightClickSchema,
-      }
-    ),
-
-    // computer_type
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.TYPE,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.TYPE,
-        description:
-          'Type text using the keyboard. Make sure the target input field is focused first (use click).',
-        schema: TypeSchema,
-      }
-    ),
-
-    // computer_key
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.KEY,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.KEY,
-        description:
-          'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',
-        schema: KeySchema,
-      }
-    ),
-
-    // computer_key_combo
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.KEY_COMBO,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.KEY_COMBO,
-        description:
-          'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',
-        schema: KeyComboSchema,
-      }
-    ),
-
-    // computer_scroll
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.SCROLL,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.SCROLL,
-        description:
-          'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',
-        schema: ScrollSchema,
-      }
-    ),
-
-    // computer_drag
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.DRAG,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.DRAG,
-        description:
-          'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',
-        schema: DragSchema,
-      }
-    ),
-
-    // computer_get_active_window
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.GET_ACTIVE_WINDOW,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.GET_ACTIVE_WINDOW,
-        description:
-          'Get information about the currently active window (title, application name, position, size).',
-        schema: GetActiveWindowSchema,
-      }
-    ),
-
-    // computer_get_mouse_position
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.GET_MOUSE_POSITION,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.GET_MOUSE_POSITION,
-        description: 'Get the current mouse cursor position on screen.',
-        schema: GetMousePositionSchema,
-      }
-    ),
-
-    // clipboard_read
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.CLIPBOARD_READ,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.CLIPBOARD_READ,
-        description: 'Read the current contents of the system clipboard.',
-        schema: ClipboardReadSchema,
-      }
-    ),
-
-    // clipboard_write
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.CLIPBOARD_WRITE,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.CLIPBOARD_WRITE,
-        description: 'Write text to the system clipboard.',
-        schema: ClipboardWriteSchema,
+  /**
+   * Helper to create tool function that optionally waits for results
+   * The toolCallId is extracted from the RunnableConfig passed by LangChain
+   */
+  const createToolFunction = (action: string) => {
+    return async (
+      args: Record<string, unknown>,
+      config?: { toolCall?: { id?: string } }
+    ): Promise<string> => {
+      const toolCallId =
+        config?.toolCall?.id ??
+        `desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+
+      // Create marker for Electron app
+      const marker: DesktopToolResponse = {
+        requiresDesktopExecution: true,
+        action,
+        args,
+        toolCallId,
+      };
+
+      // If no callback, return marker immediately (Electron handles via SSE interception)
+      if (!waitForResult) {
+        return JSON.stringify(marker);
       }
-    ),
 
-    // clipboard_paste
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.CLIPBOARD_PASTE,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.CLIPBOARD_PASTE,
-        description:
-          'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',
-        schema: ClipboardPasteSchema,
+      // With callback: wait for actual results from Electron app
+      try {
+        const result = await waitForResult(action, args, toolCallId);
+        return formatResultForLLM(result, action);
+      } catch (error) {
+        const errorMessage =
+          error instanceof Error ? error.message : String(error);
+        return `Desktop action "${action}" failed: ${errorMessage}`;
       }
-    ),
+    };
+  };
 
-    // computer_wait
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.WAIT,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.WAIT,
-        description:
-          'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',
-        schema: WaitSchema,
-      }
-    ),
-  ];
+  // computer_screenshot
+  tools.push(
+    tool(createToolFunction(EDesktopTools.SCREENSHOT), {
+      name: EDesktopTools.SCREENSHOT,
+      description:
+        'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',
+      schema: ScreenshotSchema,
+    })
+  );
+
+  // computer_click
+  tools.push(
+    tool(createToolFunction(EDesktopTools.CLICK), {
+      name: EDesktopTools.CLICK,
+      description:
+        'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',
+      schema: ClickSchema,
+    })
+  );
+
+  // computer_double_click
+  tools.push(
+    tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {
+      name: EDesktopTools.DOUBLE_CLICK,
+      description:
+        'Double-click the mouse at the specified screen coordinates.',
+      schema: DoubleClickSchema,
+    })
+  );
+
+  // computer_right_click
+  tools.push(
+    tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {
+      name: EDesktopTools.RIGHT_CLICK,
+      description:
+        'Right-click the mouse at the specified screen coordinates to open context menus.',
+      schema: RightClickSchema,
+    })
+  );
+
+  // computer_type
+  tools.push(
+    tool(createToolFunction(EDesktopTools.TYPE), {
+      name: EDesktopTools.TYPE,
+      description:
+        'Type text using the keyboard. Make sure the target input field is focused first (use click).',
+      schema: TypeSchema,
+    })
+  );
+
+  // computer_key
+  tools.push(
+    tool(createToolFunction(EDesktopTools.KEY), {
+      name: EDesktopTools.KEY,
+      description:
+        'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',
+      schema: KeySchema,
+    })
+  );
+
+  // computer_key_combo
+  tools.push(
+    tool(createToolFunction(EDesktopTools.KEY_COMBO), {
+      name: EDesktopTools.KEY_COMBO,
+      description:
+        'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',
+      schema: KeyComboSchema,
+    })
+  );
+
+  // computer_scroll
+  tools.push(
+    tool(createToolFunction(EDesktopTools.SCROLL), {
+      name: EDesktopTools.SCROLL,
+      description:
+        'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',
+      schema: ScrollSchema,
+    })
+  );
+
+  // computer_drag
+  tools.push(
+    tool(createToolFunction(EDesktopTools.DRAG), {
+      name: EDesktopTools.DRAG,
+      description:
+        'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',
+      schema: DragSchema,
+    })
+  );
+
+  // computer_get_active_window
+  tools.push(
+    tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {
+      name: EDesktopTools.GET_ACTIVE_WINDOW,
+      description:
+        'Get information about the currently active window (title, application name, position, size).',
+      schema: GetActiveWindowSchema,
+    })
+  );
+
+  // computer_get_mouse_position
+  tools.push(
+    tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {
+      name: EDesktopTools.GET_MOUSE_POSITION,
+      description: 'Get the current mouse cursor position on screen.',
+      schema: GetMousePositionSchema,
+    })
+  );
+
+  // clipboard_read
+  tools.push(
+    tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {
+      name: EDesktopTools.CLIPBOARD_READ,
+      description: 'Read the current contents of the system clipboard.',
+      schema: ClipboardReadSchema,
+    })
+  );
+
+  // clipboard_write
+  tools.push(
+    tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {
+      name: EDesktopTools.CLIPBOARD_WRITE,
+      description: 'Write text to the system clipboard.',
+      schema: ClipboardWriteSchema,
+    })
+  );
+
+  // clipboard_paste
+  tools.push(
+    tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {
+      name: EDesktopTools.CLIPBOARD_PASTE,
+      description:
+        'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',
+      schema: ClipboardPasteSchema,
+    })
+  );
+
+  // computer_wait
+  tools.push(
+    tool(createToolFunction(EDesktopTools.WAIT), {
+      name: EDesktopTools.WAIT,
+      description:
+        'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',
+      schema: WaitSchema,
+    })
+  );
+
+  // ============ Native UI Automation Tools (Windows) ============
+  // These are FASTER and MORE RELIABLE than screenshot-based automation
+  // They find elements by semantic properties (name, automationId, type)
+  // instead of relying on pixel coordinates from screenshots
+
+  // ui_find_element
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {
+      name: EDesktopTools.UI_FIND_ELEMENT,
+      description:
+        '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',
+      schema: UIFindElementSchema,
+    })
+  );
+
+  // ui_click_element
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {
+      name: EDesktopTools.UI_CLICK_ELEMENT,
+      description:
+        '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: "OK"}) or ui_click_element({controlType: "Button", name: "Submit"}). Windows only.',
+      schema: UIClickElementSchema,
+    })
+  );
+
+  // ui_get_window_tree
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {
+      name: EDesktopTools.UI_GET_WINDOW_TREE,
+      description:
+        'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',
+      schema: UIGetWindowTreeSchema,
+    })
+  );
+
+  // ui_find_buttons
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {
+      name: EDesktopTools.UI_FIND_BUTTONS,
+      description:
+        'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',
+      schema: UIFindButtonsSchema,
+    })
+  );
+
+  // ui_find_inputs
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {
+      name: EDesktopTools.UI_FIND_INPUTS,
+      description:
+        'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',
+      schema: UIFindInputsSchema,
+    })
+  );
+
+  return tools;
 }
 
 /**