illuma-agents 1.0.44 → 1.0.45

package/dist/cjs/tools/DesktopTools.cjs

@@ -23,6 +23,12 @@ const EDesktopTools = {
     CLIPBOARD_WRITE: 'clipboard_write',
     CLIPBOARD_PASTE: 'clipboard_paste',
     WAIT: 'computer_wait',
+    // Native UI Automation tools (Windows) - faster and more reliable than screenshot-based
+    UI_FIND_ELEMENT: 'ui_find_element',
+    UI_CLICK_ELEMENT: 'ui_click_element',
+    UI_GET_WINDOW_TREE: 'ui_get_window_tree',
+    UI_FIND_BUTTONS: 'ui_find_buttons',
+    UI_FIND_INPUTS: 'ui_find_inputs',
 };
 /**
  * Check if desktop capability is available based on request headers or context
@@ -87,6 +93,23 @@ const ClipboardPasteSchema = zod.z.object({});
 const WaitSchema = zod.z.object({
     ms: zod.z.number().describe('Milliseconds to wait'),
 });
+// ============ Native UI Automation Schemas (Windows) ============
+const UIFindElementSchema = zod.z.object({
+    name: zod.z.string().optional().describe('Element name/label to find (e.g., "Submit", "OK", "File")'),
+    automationId: zod.z.string().optional().describe('Automation ID of element (unique identifier)'),
+    controlType: zod.z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),
+});
+const UIClickElementSchema = zod.z.object({
+    name: zod.z.string().optional().describe('Element name/label to click'),
+    automationId: zod.z.string().optional().describe('Automation ID of element to click'),
+    controlType: zod.z.string().optional().describe('Type of control: Button, Edit, etc.'),
+    clickType: zod.z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),
+});
+const UIGetWindowTreeSchema = zod.z.object({
+    maxDepth: zod.z.number().optional().describe('Maximum depth to traverse (default: 3)'),
+});
+const UIFindButtonsSchema = zod.z.object({});
+const UIFindInputsSchema = zod.z.object({});
 /**
  * Format desktop action result for LLM consumption
  */
@@ -115,10 +138,46 @@ function formatResultForLLM(result, action) {
     if (result.clipboard !== undefined) {
         parts.push(`**Clipboard Content:** ${result.clipboard}`);
     }
+    // UI Automation results
+    if (result.uiElement) {
+        const el = result.uiElement;
+        parts.push(`**UI Element Found:**`);
+        parts.push(`  - Name: "${el.name}"`);
+        parts.push(`  - AutomationId: "${el.automationId}"`);
+        parts.push(`  - Type: ${el.controlType}`);
+        if (el.boundingRectangle) {
+            const b = el.boundingRectangle;
+            parts.push(`  - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);
+            parts.push(`  - Center: (${Math.round(b.x + b.width / 2)}, ${Math.round(b.y + b.height / 2)})`);
+        }
+        parts.push(`  - Enabled: ${el.isEnabled}`);
+    }
+    if (result.uiElements && result.uiElements.length > 0) {
+        parts.push(`**UI Elements Found (${result.uiElements.length}):**`);
+        for (const el of result.uiElements.slice(0, 20)) { // Limit to 20
+            const bounds = el.boundingRectangle ?
+                ` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';
+            parts.push(`  - [${el.controlType}] "${el.name}"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);
+        }
+        if (result.uiElements.length > 20) {
+            parts.push(`  ... and ${result.uiElements.length - 20} more`);
+        }
+    }
+    if (result.uiTree) {
+        parts.push(`**UI Tree:**`);
+        parts.push('```json');
+        parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size
+        parts.push('```');
+    }
+    if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {
+        // Generic data fallback
+        parts.push(`**Result:**`);
+        parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));
+    }
     if (parts.length === 0) {
         parts.push(`Desktop action "${action}" completed successfully.`);
     }
-    return parts.join('\n');
+    return parts.join('\\n');
 }
 /**
  * Create desktop automation tools for the agent
@@ -247,6 +306,40 @@ function createDesktopTools(options = {}) {
         description: 'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',
         schema: WaitSchema,
     }));
+    // ============ Native UI Automation Tools (Windows) ============
+    // These are FASTER and MORE RELIABLE than screenshot-based automation
+    // They find elements by semantic properties (name, automationId, type)
+    // instead of relying on pixel coordinates from screenshots
+    // ui_find_element
+    tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {
+        name: EDesktopTools.UI_FIND_ELEMENT,
+        description: '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',
+        schema: UIFindElementSchema,
+    }));
+    // ui_click_element
+    tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {
+        name: EDesktopTools.UI_CLICK_ELEMENT,
+        description: '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: "OK"}) or ui_click_element({controlType: "Button", name: "Submit"}). Windows only.',
+        schema: UIClickElementSchema,
+    }));
+    // ui_get_window_tree
+    tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {
+        name: EDesktopTools.UI_GET_WINDOW_TREE,
+        description: 'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',
+        schema: UIGetWindowTreeSchema,
+    }));
+    // ui_find_buttons
+    tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {
+        name: EDesktopTools.UI_FIND_BUTTONS,
+        description: 'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',
+        schema: UIFindButtonsSchema,
+    }));
+    // ui_find_inputs
+    tools$1.push(tools.tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {
+        name: EDesktopTools.UI_FIND_INPUTS,
+        description: 'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',
+        schema: UIFindInputsSchema,
+    }));
     return tools$1;
 }
 /**

package/dist/cjs/tools/DesktopTools.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"DesktopTools.cjs","sources":["../../../src/tools/DesktopTools.ts"],"sourcesContent":["import { z } from 'zod';\r\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\r\n\r\n/**\r\n * Desktop tool names - keep in sync with Ranger Desktop Electron app\r\n * These tools execute locally in the Electron app, NOT on the server\r\n */\r\nexport const EDesktopTools = {\r\n  SCREENSHOT: 'computer_screenshot',\r\n  CLICK: 'computer_click',\r\n  DOUBLE_CLICK: 'computer_double_click',\r\n  RIGHT_CLICK: 'computer_right_click',\r\n  TYPE: 'computer_type',\r\n  KEY: 'computer_key',\r\n  KEY_COMBO: 'computer_key_combo',\r\n  SCROLL: 'computer_scroll',\r\n  DRAG: 'computer_drag',\r\n  GET_ACTIVE_WINDOW: 'computer_get_active_window',\r\n  GET_MOUSE_POSITION: 'computer_get_mouse_position',\r\n  CLIPBOARD_READ: 'clipboard_read',\r\n  CLIPBOARD_WRITE: 'clipboard_write',\r\n  CLIPBOARD_PASTE: 'clipboard_paste',\r\n  WAIT: 'computer_wait',\r\n} as const;\r\n\r\nexport type DesktopToolName =\r\n  (typeof EDesktopTools)[keyof typeof EDesktopTools];\r\n\r\n/**\r\n * Callback function type for waiting on desktop action results\r\n * This allows the server (Ranger) to provide a callback that waits for the Electron app\r\n * to POST results back to the server before returning to the LLM.\r\n *\r\n * @param action - The desktop action (click, type, screenshot, etc.)\r\n * @param args - Arguments for the action\r\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\r\n * @returns Promise that resolves with the actual desktop result\r\n */\r\nexport type DesktopToolCallback = (\r\n  action: string,\r\n  args: Record<string, unknown>,\r\n  toolCallId: string\r\n) => Promise<DesktopActionResult>;\r\n\r\n/**\r\n * Result returned from desktop action execution\r\n */\r\nexport interface DesktopActionResult {\r\n  success: boolean;\r\n  error?: string;\r\n  screenshot?: {\r\n    base64: string;\r\n    width: number;\r\n    height: number;\r\n  };\r\n  activeWindow?: {\r\n    title: string;\r\n    app: string;\r\n    bounds?: { x: number; y: number; width: number; height: number };\r\n  };\r\n  mousePosition?: { x: number; y: number };\r\n  clipboard?: string;\r\n}\r\n\r\n/**\r\n * Check if desktop capability is available based on request headers or context\r\n * The Ranger Desktop Electron app sets these headers when connected:\r\n * - X-Ranger-Desktop: true\r\n * - X-Ranger-Desktop-Capable: true\r\n */\r\nexport function hasDesktopCapability(req?: {\r\n  headers?: Record<string, string | string[] | undefined>;\r\n}): boolean {\r\n  if (!req?.headers) {\r\n    return false;\r\n  }\r\n\r\n  const desktopApp = req.headers['x-ranger-desktop'];\r\n  const desktopCapable = req.headers['x-ranger-desktop-capable'];\r\n\r\n  return desktopApp === 'true' || desktopCapable === 'true';\r\n}\r\n\r\n// Tool schemas\r\nconst ScreenshotSchema = z.object({});\r\n\r\nconst ClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to click'),\r\n  y: z.number().describe('Y coordinate to click'),\r\n});\r\n\r\nconst DoubleClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to double-click'),\r\n  y: z.number().describe('Y coordinate to double-click'),\r\n});\r\n\r\nconst RightClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to right-click'),\r\n  y: z.number().describe('Y coordinate to right-click'),\r\n});\r\n\r\nconst TypeSchema = z.object({\r\n  text: z.string().describe('Text to type'),\r\n});\r\n\r\nconst KeySchema = z.object({\r\n  key: z\r\n    .string()\r\n    .describe(\r\n      'Key to press (e.g., \"Enter\", \"Tab\", \"Escape\", \"Backspace\", \"Delete\", \"ArrowUp\", \"ArrowDown\", \"ArrowLeft\", \"ArrowRight\", \"Home\", \"End\", \"PageUp\", \"PageDown\", \"F1\"-\"F12\")'\r\n    ),\r\n});\r\n\r\nconst KeyComboSchema = z.object({\r\n  keys: z\r\n    .array(z.string())\r\n    .describe(\r\n      'Array of keys to press together (e.g., [\"Control\", \"c\"] for copy, [\"Alt\", \"Tab\"] for window switch)'\r\n    ),\r\n});\r\n\r\nconst ScrollSchema = z.object({\r\n  x: z.number().describe('X coordinate to scroll at'),\r\n  y: z.number().describe('Y coordinate to scroll at'),\r\n  deltaX: z.number().optional().describe('Horizontal scroll amount (pixels)'),\r\n  deltaY: z.number().describe('Vertical scroll amount (pixels, negative = up, positive = down)'),\r\n});\r\n\r\nconst DragSchema = z.object({\r\n  startX: z.number().describe('Starting X coordinate'),\r\n  startY: z.number().describe('Starting Y coordinate'),\r\n  endX: z.number().describe('Ending X coordinate'),\r\n  endY: z.number().describe('Ending Y coordinate'),\r\n});\r\n\r\nconst GetActiveWindowSchema = z.object({});\r\n\r\nconst GetMousePositionSchema = z.object({});\r\n\r\nconst ClipboardReadSchema = z.object({});\r\n\r\nconst ClipboardWriteSchema = z.object({\r\n  text: z.string().describe('Text to write to clipboard'),\r\n});\r\n\r\nconst ClipboardPasteSchema = z.object({});\r\n\r\nconst WaitSchema = z.object({\r\n  ms: z.number().describe('Milliseconds to wait'),\r\n});\r\n\r\n/**\r\n * Desktop tool response interface\r\n * This is what the Electron app returns after executing the action\r\n */\r\nexport interface DesktopToolResponse {\r\n  requiresDesktopExecution: true;\r\n  action: string;\r\n  args: Record<string, unknown>;\r\n  toolCallId?: string;\r\n}\r\n\r\n/**\r\n * Options for creating desktop tools\r\n */\r\nexport interface CreateDesktopToolsOptions {\r\n  /**\r\n   * Optional callback that waits for desktop action results.\r\n   * When provided, tools will await this callback to get actual results from the Electron app.\r\n   * When not provided, tools return markers immediately (for non-server contexts).\r\n   */\r\n  waitForResult?: DesktopToolCallback;\r\n}\r\n\r\n/**\r\n * Format desktop action result for LLM consumption\r\n */\r\nfunction formatResultForLLM(\r\n  result: DesktopActionResult,\r\n  action: string\r\n): string {\r\n  if (!result.success && result.error) {\r\n    return `Desktop action \"${action}\" failed: ${result.error}`;\r\n  }\r\n\r\n  const parts: string[] = [];\r\n\r\n  if (result.screenshot) {\r\n    parts.push(\r\n      `Screenshot captured (${result.screenshot.width}x${result.screenshot.height})`\r\n    );\r\n    // The base64 image will be handled separately by the message formatter\r\n  }\r\n\r\n  if (result.activeWindow) {\r\n    parts.push(`**Active Window:**`);\r\n    parts.push(`  - Title: ${result.activeWindow.title}`);\r\n    parts.push(`  - App: ${result.activeWindow.app}`);\r\n    if (result.activeWindow.bounds) {\r\n      const b = result.activeWindow.bounds;\r\n      parts.push(`  - Position: (${b.x}, ${b.y})`);\r\n      parts.push(`  - Size: ${b.width}x${b.height}`);\r\n    }\r\n  }\r\n\r\n  if (result.mousePosition) {\r\n    parts.push(\r\n      `**Mouse Position:** (${result.mousePosition.x}, ${result.mousePosition.y})`\r\n    );\r\n  }\r\n\r\n  if (result.clipboard !== undefined) {\r\n    parts.push(`**Clipboard Content:** ${result.clipboard}`);\r\n  }\r\n\r\n  if (parts.length === 0) {\r\n    parts.push(`Desktop action \"${action}\" completed successfully.`);\r\n  }\r\n\r\n  return parts.join('\\n');\r\n}\r\n\r\n/**\r\n * Create desktop automation tools for the agent\r\n * These tools allow AI to control the user's desktop when Ranger Desktop is running\r\n */\r\nexport function createDesktopTools(\r\n  options: CreateDesktopToolsOptions = {}\r\n): DynamicStructuredTool[] {\r\n  const { waitForResult } = options;\r\n  const tools: DynamicStructuredTool[] = [];\r\n\r\n  /**\r\n   * Helper to create tool function that optionally waits for results\r\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\r\n   */\r\n  const createToolFunction = (action: string) => {\r\n    return async (\r\n      args: Record<string, unknown>,\r\n      config?: { toolCall?: { id?: string } }\r\n    ): Promise<string> => {\r\n      const toolCallId =\r\n        config?.toolCall?.id ??\r\n        `desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;\r\n\r\n      // Create marker for Electron app\r\n      const marker: DesktopToolResponse = {\r\n        requiresDesktopExecution: true,\r\n        action,\r\n        args,\r\n        toolCallId,\r\n      };\r\n\r\n      // If no callback, return marker immediately (Electron handles via SSE interception)\r\n      if (!waitForResult) {\r\n        return JSON.stringify(marker);\r\n      }\r\n\r\n      // With callback: wait for actual results from Electron app\r\n      try {\r\n        const result = await waitForResult(action, args, toolCallId);\r\n        return formatResultForLLM(result, action);\r\n      } catch (error) {\r\n        const errorMessage =\r\n          error instanceof Error ? error.message : String(error);\r\n        return `Desktop action \"${action}\" failed: ${errorMessage}`;\r\n      }\r\n    };\r\n  };\r\n\r\n  // computer_screenshot\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCREENSHOT), {\r\n      name: EDesktopTools.SCREENSHOT,\r\n      description:\r\n        'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',\r\n      schema: ScreenshotSchema,\r\n    })\r\n  );\r\n\r\n  // computer_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLICK), {\r\n      name: EDesktopTools.CLICK,\r\n      description:\r\n        'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',\r\n      schema: ClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_double_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {\r\n      name: EDesktopTools.DOUBLE_CLICK,\r\n      description:\r\n        'Double-click the mouse at the specified screen coordinates.',\r\n      schema: DoubleClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_right_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {\r\n      name: EDesktopTools.RIGHT_CLICK,\r\n      description:\r\n        'Right-click the mouse at the specified screen coordinates to open context menus.',\r\n      schema: RightClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_type\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.TYPE), {\r\n      name: EDesktopTools.TYPE,\r\n      description:\r\n        'Type text using the keyboard. Make sure the target input field is focused first (use click).',\r\n      schema: TypeSchema,\r\n    })\r\n  );\r\n\r\n  // computer_key\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY), {\r\n      name: EDesktopTools.KEY,\r\n      description:\r\n        'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',\r\n      schema: KeySchema,\r\n    })\r\n  );\r\n\r\n  // computer_key_combo\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY_COMBO), {\r\n      name: EDesktopTools.KEY_COMBO,\r\n      description:\r\n        'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',\r\n      schema: KeyComboSchema,\r\n    })\r\n  );\r\n\r\n  // computer_scroll\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCROLL), {\r\n      name: EDesktopTools.SCROLL,\r\n      description:\r\n        'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',\r\n      schema: ScrollSchema,\r\n    })\r\n  );\r\n\r\n  // computer_drag\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DRAG), {\r\n      name: EDesktopTools.DRAG,\r\n      description:\r\n        'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',\r\n      schema: DragSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_active_window\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {\r\n      name: EDesktopTools.GET_ACTIVE_WINDOW,\r\n      description:\r\n        'Get information about the currently active window (title, application name, position, size).',\r\n      schema: GetActiveWindowSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_mouse_position\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {\r\n      name: EDesktopTools.GET_MOUSE_POSITION,\r\n      description: 'Get the current mouse cursor position on screen.',\r\n      schema: GetMousePositionSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_read\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {\r\n      name: EDesktopTools.CLIPBOARD_READ,\r\n      description: 'Read the current contents of the system clipboard.',\r\n      schema: ClipboardReadSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_write\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {\r\n      name: EDesktopTools.CLIPBOARD_WRITE,\r\n      description: 'Write text to the system clipboard.',\r\n      schema: ClipboardWriteSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_paste\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {\r\n      name: EDesktopTools.CLIPBOARD_PASTE,\r\n      description:\r\n        'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',\r\n      schema: ClipboardPasteSchema,\r\n    })\r\n  );\r\n\r\n  // computer_wait\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.WAIT), {\r\n      name: EDesktopTools.WAIT,\r\n      description:\r\n        'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',\r\n      schema: WaitSchema,\r\n    })\r\n  );\r\n\r\n  return tools;\r\n}\r\n\r\n/**\r\n * Get all desktop tool names\r\n */\r\nexport function getDesktopToolNames(): DesktopToolName[] {\r\n  return Object.values(EDesktopTools);\r\n}\r\n\r\n/**\r\n * Check if a tool name is a desktop tool\r\n */\r\nexport function isDesktopTool(name: string): name is DesktopToolName {\r\n  return Object.values(EDesktopTools).includes(name as DesktopToolName);\r\n}\r\n"],"names":["z","tools","tool"],"mappings":";;;;;AAGA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,UAAU,EAAE,qBAAqB;AACjC,IAAA,KAAK,EAAE,gBAAgB;AACvB,IAAA,YAAY,EAAE,uBAAuB;AACrC,IAAA,WAAW,EAAE,sBAAsB;AACnC,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,GAAG,EAAE,cAAc;AACnB,IAAA,SAAS,EAAE,oBAAoB;AAC/B,IAAA,MAAM,EAAE,iBAAiB;AACzB,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,iBAAiB,EAAE,4BAA4B;AAC/C,IAAA,kBAAkB,EAAE,6BAA6B;AACjD,IAAA,cAAc,EAAE,gBAAgB;AAChC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,IAAI,EAAE,eAAe;;AA0CvB;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAClD,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,UAAU,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AAC3D;AAEA;AACA,MAAM,gBAAgB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAErC,MAAM,WAAW,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC3B,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IAC/C,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;AAChD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;IACjC,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IACtD,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;AACvD,CAAA,CAAC;AAEF,MAAM,gBAAgB,GAAGA,KAAC,CAAC,MAAM,CAAC;IAChC,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACrD,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;AACtD,CAAA,CAAC;AAEF,MAAM,UAAU,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC1B,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;AAC1C,CAAA,CAAC;AAEF,MAAM,SAAS,GAAGA,KAAC,CAAC,MAAM,CAAC;AACzB,IAAA,GAAG,EAAEA;AACF,SAAA,MAAM;SACN,QAAQ,CACP,0KAA0K,CAC3K;AACJ,CAAA,CAAC;AAEF,MAAM,cAAc,GAAGA,KAAC,CAAC,MAAM,CAAC;AAC9B,IAAA,IAAI,EAAEA;AACH,SAAA,KAAK,CAACA,KAAC,CAAC,MAAM,EAAE;SAChB,QAAQ,CACP,qGAAqG,CACtG;AACJ,CAAA,CAAC;AAEF,MAAM,YAAY,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC5B,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;IACnD,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;AACnD,IAAA,MAAM,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAC3E,MAAM,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iEAAiE,CAAC;AAC/F,CAAA,CAAC;AAEF,MAAM,UAAU,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC1B,MAAM,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,MAAM,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IAChD,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;AACjD,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE1C,MAAM,sBAAsB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE3C,MAAM,mBAAmB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAExC,MAAM,oBAAoB,GAAGA,KAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;AACxD,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEzC,MAAM,UAAU,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC1B,EAAE,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;AAChD,CAAA,CAAC;AAyBF;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,UAAU,EAAE;AACrB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,UAAU,CAAC,KAAK,CAAA,CAAA,EAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAA,CAAA,CAAG,CAC/E;;;AAIH,IAAA,IAAI,MAAM,CAAC,YAAY,EAAE;AACvB,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,kBAAA,CAAoB,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,CAAc,WAAA,EAAA,MAAM,CAAC,YAAY,CAAC,KAAK,CAAE,CAAA,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,CAAY,SAAA,EAAA,MAAM,CAAC,YAAY,CAAC,GAAG,CAAE,CAAA,CAAC;AACjD,QAAA,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;AAC9B,YAAA,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM;AACpC,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,CAAC,CAAC,CAAC,CAAA,EAAA,EAAK,CAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CAAC;AAC5C,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,CAAC,CAAC,KAAK,CAAA,CAAA,EAAI,CAAC,CAAC,MAAM,CAAA,CAAE,CAAC;;;AAIlD,IAAA,IAAI,MAAM,CAAC,aAAa,EAAE;AACxB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,EAAA,EAAK,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,CAAA,CAAG,CAC7E;;AAGH,IAAA,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE;QAClC,KAAK,CAAC,IAAI,CAAC,CAAA,uBAAA,EAA0B,MAAM,CAAC,SAAS,CAAE,CAAA,CAAC;;AAG1D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;AACtB,QAAA,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAA,yBAAA,CAA2B,CAAC;;AAGlE,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;AAGG;AACa,SAAA,kBAAkB,CAChC,OAAA,GAAqC,EAAE,EAAA;AAEvC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IACjC,MAAMC,OAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAW,QAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAGhE,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;AAI/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGDA,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE;QACjD,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EACT,qGAAqG;AACvG,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;QAC5C,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EACT,4GAA4G;AAC9G,QAAA,MAAM,EAAE,WAAW;AACpB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE;QACnD,IAAI,EAAE,aAAa,CAAC,YAAY;AAChC,QAAA,WAAW,EACT,6DAA6D;AAC/D,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE;QAClD,IAAI,EAAE,aAAa,CAAC,WAAW;AAC/B,QAAA,WAAW,EACT,kFAAkF;AACpF,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE;QAC1C,IAAI,EAAE,aAAa,CAAC,GAAG;AACvB,QAAA,WAAW,EACT,2FAA2F;AAC7F,QAAA,MAAM,EAAE,SAAS;AAClB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE;QAChD,IAAI,EAAE,aAAa,CAAC,SAAS;AAC7B,QAAA,WAAW,EACT,6FAA6F;AAC/F,QAAA,MAAM,EAAE,cAAc;AACvB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;QAC7C,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EACT,wGAAwG;AAC1G,QAAA,MAAM,EAAE,YAAY;AACrB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,yFAAyF;AAC3F,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,iBAAiB,CAAC,EAAE;QACxD,IAAI,EAAE,aAAa,CAAC,iBAAiB;AACrC,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;QACzD,IAAI,EAAE,aAAa,CAAC,kBAAkB;AACtC,QAAA,WAAW,EAAE,kDAAkD;AAC/D,QAAA,MAAM,EAAE,sBAAsB;AAC/B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE;QACrD,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,oDAAoD;AACjE,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EAAE,qCAAqC;AAClD,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,oGAAoG;AACtG,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,+FAA+F;AACjG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;AAED,IAAA,OAAOD,OAAK;AACd;AAEA;;AAEG;SACa,mBAAmB,GAAA;AACjC,IAAA,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;AACrC;AAEA;;AAEG;AACG,SAAU,aAAa,CAAC,IAAY,EAAA;IACxC,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAuB,CAAC;AACvE;;;;;;;;"}
+{"version":3,"file":"DesktopTools.cjs","sources":["../../../src/tools/DesktopTools.ts"],"sourcesContent":["import { z } from 'zod';\r\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\r\n\r\n/**\r\n * Desktop tool names - keep in sync with Ranger Desktop Electron app\r\n * These tools execute locally in the Electron app, NOT on the server\r\n */\r\nexport const EDesktopTools = {\r\n  SCREENSHOT: 'computer_screenshot',\r\n  CLICK: 'computer_click',\r\n  DOUBLE_CLICK: 'computer_double_click',\r\n  RIGHT_CLICK: 'computer_right_click',\r\n  TYPE: 'computer_type',\r\n  KEY: 'computer_key',\r\n  KEY_COMBO: 'computer_key_combo',\r\n  SCROLL: 'computer_scroll',\r\n  DRAG: 'computer_drag',\r\n  GET_ACTIVE_WINDOW: 'computer_get_active_window',\r\n  GET_MOUSE_POSITION: 'computer_get_mouse_position',\r\n  CLIPBOARD_READ: 'clipboard_read',\r\n  CLIPBOARD_WRITE: 'clipboard_write',\r\n  CLIPBOARD_PASTE: 'clipboard_paste',\r\n  WAIT: 'computer_wait',\r\n  // Native UI Automation tools (Windows) - faster and more reliable than screenshot-based\r\n  UI_FIND_ELEMENT: 'ui_find_element',\r\n  UI_CLICK_ELEMENT: 'ui_click_element',\r\n  UI_GET_WINDOW_TREE: 'ui_get_window_tree',\r\n  UI_FIND_BUTTONS: 'ui_find_buttons',\r\n  UI_FIND_INPUTS: 'ui_find_inputs',\r\n} as const;\r\n\r\nexport type DesktopToolName =\r\n  (typeof EDesktopTools)[keyof typeof EDesktopTools];\r\n\r\n/**\r\n * Callback function type for waiting on desktop action results\r\n * This allows the server (Ranger) to provide a callback that waits for the Electron app\r\n * to POST results back to the server before returning to the LLM.\r\n *\r\n * @param action - The desktop action (click, type, screenshot, etc.)\r\n * @param args - Arguments for the action\r\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\r\n * @returns Promise that resolves with the actual desktop result\r\n */\r\nexport type DesktopToolCallback = (\r\n  action: string,\r\n  args: Record<string, unknown>,\r\n  toolCallId: string\r\n) => Promise<DesktopActionResult>;\r\n\r\n/**\r\n * Result returned from desktop action execution\r\n */\r\nexport interface DesktopActionResult {\r\n  success: boolean;\r\n  error?: string;\r\n  screenshot?: {\r\n    base64: string;\r\n    width: number;\r\n    height: number;\r\n  };\r\n  activeWindow?: {\r\n    title: string;\r\n    app: string;\r\n    bounds?: { x: number; y: number; width: number; height: number };\r\n  };\r\n  mousePosition?: { x: number; y: number };\r\n  clipboard?: string;\r\n  // UI Automation results\r\n  uiElement?: {\r\n    name: string;\r\n    automationId: string;\r\n    controlType: string;\r\n    boundingRectangle?: { x: number; y: number; width: number; height: number };\r\n    isEnabled: boolean;\r\n  };\r\n  uiElements?: Array<{\r\n    name: string;\r\n    automationId: string;\r\n    controlType: string;\r\n    boundingRectangle?: { x: number; y: number; width: number; height: number };\r\n  }>;\r\n  uiTree?: unknown;\r\n  // Generic data for extended results\r\n  data?: unknown;\r\n}\r\n\r\n/**\r\n * Check if desktop capability is available based on request headers or context\r\n * The Ranger Desktop Electron app sets these headers when connected:\r\n * - X-Ranger-Desktop: true\r\n * - X-Ranger-Desktop-Capable: true\r\n */\r\nexport function hasDesktopCapability(req?: {\r\n  headers?: Record<string, string | string[] | undefined>;\r\n}): boolean {\r\n  if (!req?.headers) {\r\n    return false;\r\n  }\r\n\r\n  const desktopApp = req.headers['x-ranger-desktop'];\r\n  const desktopCapable = req.headers['x-ranger-desktop-capable'];\r\n\r\n  return desktopApp === 'true' || desktopCapable === 'true';\r\n}\r\n\r\n// Tool schemas\r\nconst ScreenshotSchema = z.object({});\r\n\r\nconst ClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to click'),\r\n  y: z.number().describe('Y coordinate to click'),\r\n});\r\n\r\nconst DoubleClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to double-click'),\r\n  y: z.number().describe('Y coordinate to double-click'),\r\n});\r\n\r\nconst RightClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to right-click'),\r\n  y: z.number().describe('Y coordinate to right-click'),\r\n});\r\n\r\nconst TypeSchema = z.object({\r\n  text: z.string().describe('Text to type'),\r\n});\r\n\r\nconst KeySchema = z.object({\r\n  key: z\r\n    .string()\r\n    .describe(\r\n      'Key to press (e.g., \"Enter\", \"Tab\", \"Escape\", \"Backspace\", \"Delete\", \"ArrowUp\", \"ArrowDown\", \"ArrowLeft\", \"ArrowRight\", \"Home\", \"End\", \"PageUp\", \"PageDown\", \"F1\"-\"F12\")'\r\n    ),\r\n});\r\n\r\nconst KeyComboSchema = z.object({\r\n  keys: z\r\n    .array(z.string())\r\n    .describe(\r\n      'Array of keys to press together (e.g., [\"Control\", \"c\"] for copy, [\"Alt\", \"Tab\"] for window switch)'\r\n    ),\r\n});\r\n\r\nconst ScrollSchema = z.object({\r\n  x: z.number().describe('X coordinate to scroll at'),\r\n  y: z.number().describe('Y coordinate to scroll at'),\r\n  deltaX: z.number().optional().describe('Horizontal scroll amount (pixels)'),\r\n  deltaY: z.number().describe('Vertical scroll amount (pixels, negative = up, positive = down)'),\r\n});\r\n\r\nconst DragSchema = z.object({\r\n  startX: z.number().describe('Starting X coordinate'),\r\n  startY: z.number().describe('Starting Y coordinate'),\r\n  endX: z.number().describe('Ending X coordinate'),\r\n  endY: z.number().describe('Ending Y coordinate'),\r\n});\r\n\r\nconst GetActiveWindowSchema = z.object({});\r\n\r\nconst GetMousePositionSchema = z.object({});\r\n\r\nconst ClipboardReadSchema = z.object({});\r\n\r\nconst ClipboardWriteSchema = z.object({\r\n  text: z.string().describe('Text to write to clipboard'),\r\n});\r\n\r\nconst ClipboardPasteSchema = z.object({});\r\n\r\nconst WaitSchema = z.object({\r\n  ms: z.number().describe('Milliseconds to wait'),\r\n});\r\n\r\n// ============ Native UI Automation Schemas (Windows) ============\r\n\r\nconst UIFindElementSchema = z.object({\r\n  name: z.string().optional().describe('Element name/label to find (e.g., \"Submit\", \"OK\", \"File\")'),\r\n  automationId: z.string().optional().describe('Automation ID of element (unique identifier)'),\r\n  controlType: z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),\r\n});\r\n\r\nconst UIClickElementSchema = z.object({\r\n  name: z.string().optional().describe('Element name/label to click'),\r\n  automationId: z.string().optional().describe('Automation ID of element to click'),\r\n  controlType: z.string().optional().describe('Type of control: Button, Edit, etc.'),\r\n  clickType: z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),\r\n});\r\n\r\nconst UIGetWindowTreeSchema = z.object({\r\n  maxDepth: z.number().optional().describe('Maximum depth to traverse (default: 3)'),\r\n});\r\n\r\nconst UIFindButtonsSchema = z.object({});\r\n\r\nconst UIFindInputsSchema = z.object({});\r\n\r\n/**\r\n * Desktop tool response interface\r\n * This is what the Electron app returns after executing the action\r\n */\r\nexport interface DesktopToolResponse {\r\n  requiresDesktopExecution: true;\r\n  action: string;\r\n  args: Record<string, unknown>;\r\n  toolCallId?: string;\r\n}\r\n\r\n/**\r\n * Options for creating desktop tools\r\n */\r\nexport interface CreateDesktopToolsOptions {\r\n  /**\r\n   * Optional callback that waits for desktop action results.\r\n   * When provided, tools will await this callback to get actual results from the Electron app.\r\n   * When not provided, tools return markers immediately (for non-server contexts).\r\n   */\r\n  waitForResult?: DesktopToolCallback;\r\n}\r\n\r\n/**\r\n * Format desktop action result for LLM consumption\r\n */\r\nfunction formatResultForLLM(\r\n  result: DesktopActionResult,\r\n  action: string\r\n): string {\r\n  if (!result.success && result.error) {\r\n    return `Desktop action \"${action}\" failed: ${result.error}`;\r\n  }\r\n\r\n  const parts: string[] = [];\r\n\r\n  if (result.screenshot) {\r\n    parts.push(\r\n      `Screenshot captured (${result.screenshot.width}x${result.screenshot.height})`\r\n    );\r\n    // The base64 image will be handled separately by the message formatter\r\n  }\r\n\r\n  if (result.activeWindow) {\r\n    parts.push(`**Active Window:**`);\r\n    parts.push(`  - Title: ${result.activeWindow.title}`);\r\n    parts.push(`  - App: ${result.activeWindow.app}`);\r\n    if (result.activeWindow.bounds) {\r\n      const b = result.activeWindow.bounds;\r\n      parts.push(`  - Position: (${b.x}, ${b.y})`);\r\n      parts.push(`  - Size: ${b.width}x${b.height}`);\r\n    }\r\n  }\r\n\r\n  if (result.mousePosition) {\r\n    parts.push(\r\n      `**Mouse Position:** (${result.mousePosition.x}, ${result.mousePosition.y})`\r\n    );\r\n  }\r\n\r\n  if (result.clipboard !== undefined) {\r\n    parts.push(`**Clipboard Content:** ${result.clipboard}`);\r\n  }\r\n\r\n  // UI Automation results\r\n  if (result.uiElement) {\r\n    const el = result.uiElement;\r\n    parts.push(`**UI Element Found:**`);\r\n    parts.push(`  - Name: \"${el.name}\"`);\r\n    parts.push(`  - AutomationId: \"${el.automationId}\"`);\r\n    parts.push(`  - Type: ${el.controlType}`);\r\n    if (el.boundingRectangle) {\r\n      const b = el.boundingRectangle;\r\n      parts.push(`  - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);\r\n      parts.push(`  - Center: (${Math.round(b.x + b.width/2)}, ${Math.round(b.y + b.height/2)})`);\r\n    }\r\n    parts.push(`  - Enabled: ${el.isEnabled}`);\r\n  }\r\n\r\n  if (result.uiElements && result.uiElements.length > 0) {\r\n    parts.push(`**UI Elements Found (${result.uiElements.length}):**`);\r\n    for (const el of result.uiElements.slice(0, 20)) { // Limit to 20\r\n      const bounds = el.boundingRectangle ? \r\n        ` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';\r\n      parts.push(`  - [${el.controlType}] \"${el.name}\"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);\r\n    }\r\n    if (result.uiElements.length > 20) {\r\n      parts.push(`  ... and ${result.uiElements.length - 20} more`);\r\n    }\r\n  }\r\n\r\n  if (result.uiTree) {\r\n    parts.push(`**UI Tree:**`);\r\n    parts.push('```json');\r\n    parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size\r\n    parts.push('```');\r\n  }\r\n\r\n  if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {\r\n    // Generic data fallback\r\n    parts.push(`**Result:**`);\r\n    parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));\r\n  }\r\n\r\n  if (parts.length === 0) {\r\n    parts.push(`Desktop action \"${action}\" completed successfully.`);\r\n  }\r\n\r\n  return parts.join('\\\\n');\r\n}\r\n\r\n/**\r\n * Create desktop automation tools for the agent\r\n * These tools allow AI to control the user's desktop when Ranger Desktop is running\r\n */\r\nexport function createDesktopTools(\r\n  options: CreateDesktopToolsOptions = {}\r\n): DynamicStructuredTool[] {\r\n  const { waitForResult } = options;\r\n  const tools: DynamicStructuredTool[] = [];\r\n\r\n  /**\r\n   * Helper to create tool function that optionally waits for results\r\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\r\n   */\r\n  const createToolFunction = (action: string) => {\r\n    return async (\r\n      args: Record<string, unknown>,\r\n      config?: { toolCall?: { id?: string } }\r\n    ): Promise<string> => {\r\n      const toolCallId =\r\n        config?.toolCall?.id ??\r\n        `desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;\r\n\r\n      // Create marker for Electron app\r\n      const marker: DesktopToolResponse = {\r\n        requiresDesktopExecution: true,\r\n        action,\r\n        args,\r\n        toolCallId,\r\n      };\r\n\r\n      // If no callback, return marker immediately (Electron handles via SSE interception)\r\n      if (!waitForResult) {\r\n        return JSON.stringify(marker);\r\n      }\r\n\r\n      // With callback: wait for actual results from Electron app\r\n      try {\r\n        const result = await waitForResult(action, args, toolCallId);\r\n        return formatResultForLLM(result, action);\r\n      } catch (error) {\r\n        const errorMessage =\r\n          error instanceof Error ? error.message : String(error);\r\n        return `Desktop action \"${action}\" failed: ${errorMessage}`;\r\n      }\r\n    };\r\n  };\r\n\r\n  // computer_screenshot\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCREENSHOT), {\r\n      name: EDesktopTools.SCREENSHOT,\r\n      description:\r\n        'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',\r\n      schema: ScreenshotSchema,\r\n    })\r\n  );\r\n\r\n  // computer_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLICK), {\r\n      name: EDesktopTools.CLICK,\r\n      description:\r\n        'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',\r\n      schema: ClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_double_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {\r\n      name: EDesktopTools.DOUBLE_CLICK,\r\n      description:\r\n        'Double-click the mouse at the specified screen coordinates.',\r\n      schema: DoubleClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_right_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {\r\n      name: EDesktopTools.RIGHT_CLICK,\r\n      description:\r\n        'Right-click the mouse at the specified screen coordinates to open context menus.',\r\n      schema: RightClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_type\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.TYPE), {\r\n      name: EDesktopTools.TYPE,\r\n      description:\r\n        'Type text using the keyboard. Make sure the target input field is focused first (use click).',\r\n      schema: TypeSchema,\r\n    })\r\n  );\r\n\r\n  // computer_key\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY), {\r\n      name: EDesktopTools.KEY,\r\n      description:\r\n        'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',\r\n      schema: KeySchema,\r\n    })\r\n  );\r\n\r\n  // computer_key_combo\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY_COMBO), {\r\n      name: EDesktopTools.KEY_COMBO,\r\n      description:\r\n        'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',\r\n      schema: KeyComboSchema,\r\n    })\r\n  );\r\n\r\n  // computer_scroll\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCROLL), {\r\n      name: EDesktopTools.SCROLL,\r\n      description:\r\n        'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',\r\n      schema: ScrollSchema,\r\n    })\r\n  );\r\n\r\n  // computer_drag\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DRAG), {\r\n      name: EDesktopTools.DRAG,\r\n      description:\r\n        'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',\r\n      schema: DragSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_active_window\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {\r\n      name: EDesktopTools.GET_ACTIVE_WINDOW,\r\n      description:\r\n        'Get information about the currently active window (title, application name, position, size).',\r\n      schema: GetActiveWindowSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_mouse_position\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {\r\n      name: EDesktopTools.GET_MOUSE_POSITION,\r\n      description: 'Get the current mouse cursor position on screen.',\r\n      schema: GetMousePositionSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_read\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {\r\n      name: EDesktopTools.CLIPBOARD_READ,\r\n      description: 'Read the current contents of the system clipboard.',\r\n      schema: ClipboardReadSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_write\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {\r\n      name: EDesktopTools.CLIPBOARD_WRITE,\r\n      description: 'Write text to the system clipboard.',\r\n      schema: ClipboardWriteSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_paste\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {\r\n      name: EDesktopTools.CLIPBOARD_PASTE,\r\n      description:\r\n        'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',\r\n      schema: ClipboardPasteSchema,\r\n    })\r\n  );\r\n\r\n  // computer_wait\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.WAIT), {\r\n      name: EDesktopTools.WAIT,\r\n      description:\r\n        'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',\r\n      schema: WaitSchema,\r\n    })\r\n  );\r\n\r\n  // ============ Native UI Automation Tools (Windows) ============\r\n  // These are FASTER and MORE RELIABLE than screenshot-based automation\r\n  // They find elements by semantic properties (name, automationId, type)\r\n  // instead of relying on pixel coordinates from screenshots\r\n\r\n  // ui_find_element\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {\r\n      name: EDesktopTools.UI_FIND_ELEMENT,\r\n      description:\r\n        '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',\r\n      schema: UIFindElementSchema,\r\n    })\r\n  );\r\n\r\n  // ui_click_element\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {\r\n      name: EDesktopTools.UI_CLICK_ELEMENT,\r\n      description:\r\n        '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: \"OK\"}) or ui_click_element({controlType: \"Button\", name: \"Submit\"}). Windows only.',\r\n      schema: UIClickElementSchema,\r\n    })\r\n  );\r\n\r\n  // ui_get_window_tree\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {\r\n      name: EDesktopTools.UI_GET_WINDOW_TREE,\r\n      description:\r\n        'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',\r\n      schema: UIGetWindowTreeSchema,\r\n    })\r\n  );\r\n\r\n  // ui_find_buttons\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {\r\n      name: EDesktopTools.UI_FIND_BUTTONS,\r\n      description:\r\n        'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',\r\n      schema: UIFindButtonsSchema,\r\n    })\r\n  );\r\n\r\n  // ui_find_inputs\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {\r\n      name: EDesktopTools.UI_FIND_INPUTS,\r\n      description:\r\n        'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',\r\n      schema: UIFindInputsSchema,\r\n    })\r\n  );\r\n\r\n  return tools;\r\n}\r\n\r\n/**\r\n * Get all desktop tool names\r\n */\r\nexport function getDesktopToolNames(): DesktopToolName[] {\r\n  return Object.values(EDesktopTools);\r\n}\r\n\r\n/**\r\n * Check if a tool name is a desktop tool\r\n */\r\nexport function isDesktopTool(name: string): name is DesktopToolName {\r\n  return Object.values(EDesktopTools).includes(name as DesktopToolName);\r\n}\r\n"],"names":["z","tools","tool"],"mappings":";;;;;AAGA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,UAAU,EAAE,qBAAqB;AACjC,IAAA,KAAK,EAAE,gBAAgB;AACvB,IAAA,YAAY,EAAE,uBAAuB;AACrC,IAAA,WAAW,EAAE,sBAAsB;AACnC,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,GAAG,EAAE,cAAc;AACnB,IAAA,SAAS,EAAE,oBAAoB;AAC/B,IAAA,MAAM,EAAE,iBAAiB;AACzB,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,iBAAiB,EAAE,4BAA4B;AAC/C,IAAA,kBAAkB,EAAE,6BAA6B;AACjD,IAAA,cAAc,EAAE,gBAAgB;AAChC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,IAAI,EAAE,eAAe;;AAErB,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,gBAAgB,EAAE,kBAAkB;AACpC,IAAA,kBAAkB,EAAE,oBAAoB;AACxC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,cAAc,EAAE,gBAAgB;;AA2DlC;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAClD,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,UAAU,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AAC3D;AAEA;AACA,MAAM,gBAAgB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAErC,MAAM,WAAW,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC3B,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IAC/C,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;AAChD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAGA,KAAC,CAAC,MAAM,CAAC;IACjC,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IACtD,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;AACvD,CAAA,CAAC;AAEF,MAAM,gBAAgB,GAAGA,KAAC,CAAC,MAAM,CAAC;IAChC,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACrD,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;AACtD,CAAA,CAAC;AAEF,MAAM,UAAU,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC1B,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;AAC1C,CAAA,CAAC;AAEF,MAAM,SAAS,GAAGA,KAAC,CAAC,MAAM,CAAC;AACzB,IAAA,GAAG,EAAEA;AACF,SAAA,MAAM;SACN,QAAQ,CACP,0KAA0K,CAC3K;AACJ,CAAA,CAAC;AAEF,MAAM,cAAc,GAAGA,KAAC,CAAC,MAAM,CAAC;AAC9B,IAAA,IAAI,EAAEA;AACH,SAAA,KAAK,CAACA,KAAC,CAAC,MAAM,EAAE;SAChB,QAAQ,CACP,qGAAqG,CACtG;AACJ,CAAA,CAAC;AAEF,MAAM,YAAY,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC5B,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;IACnD,CAAC,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;AACnD,IAAA,MAAM,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAC3E,MAAM,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iEAAiE,CAAC;AAC/F,CAAA,CAAC;AAEF,MAAM,UAAU,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC1B,MAAM,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,MAAM,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IAChD,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;AACjD,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE1C,MAAM,sBAAsB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE3C,MAAM,mBAAmB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAExC,MAAM,oBAAoB,GAAGA,KAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;AACxD,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEzC,MAAM,UAAU,GAAGA,KAAC,CAAC,MAAM,CAAC;IAC1B,EAAE,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;AAChD,CAAA,CAAC;AAEF;AAEA,MAAM,mBAAmB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACnC,IAAA,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2DAA2D,CAAC;AACjG,IAAA,YAAY,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;AAC5F,IAAA,WAAW,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mFAAmF,CAAC;AACjI,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACpC,IAAA,IAAI,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;AACnE,IAAA,YAAY,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AACjF,IAAA,WAAW,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAClF,SAAS,EAAEA,KAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;AACjG,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAGA,KAAC,CAAC,MAAM,CAAC;AACrC,IAAA,QAAQ,EAAEA,KAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;AACnF,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAExC,MAAM,kBAAkB,GAAGA,KAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAyBvC;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,UAAU,EAAE;AACrB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,UAAU,CAAC,KAAK,CAAA,CAAA,EAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAA,CAAA,CAAG,CAC/E;;;AAIH,IAAA,IAAI,MAAM,CAAC,YAAY,EAAE;AACvB,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,kBAAA,CAAoB,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,CAAc,WAAA,EAAA,MAAM,CAAC,YAAY,CAAC,KAAK,CAAE,CAAA,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,CAAY,SAAA,EAAA,MAAM,CAAC,YAAY,CAAC,GAAG,CAAE,CAAA,CAAC;AACjD,QAAA,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;AAC9B,YAAA,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM;AACpC,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,CAAC,CAAC,CAAC,CAAA,EAAA,EAAK,CAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CAAC;AAC5C,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,CAAC,CAAC,KAAK,CAAA,CAAA,EAAI,CAAC,CAAC,MAAM,CAAA,CAAE,CAAC;;;AAIlD,IAAA,IAAI,MAAM,CAAC,aAAa,EAAE;AACxB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,EAAA,EAAK,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,CAAA,CAAG,CAC7E;;AAGH,IAAA,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE;QAClC,KAAK,CAAC,IAAI,CAAC,CAAA,uBAAA,EAA0B,MAAM,CAAC,SAAS,CAAE,CAAA,CAAC;;;AAI1D,IAAA,IAAI,MAAM,CAAC,SAAS,EAAE;AACpB,QAAA,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS;AAC3B,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,qBAAA,CAAuB,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,CAAA,WAAA,EAAc,EAAE,CAAC,IAAI,CAAG,CAAA,CAAA,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,CAAA,mBAAA,EAAsB,EAAE,CAAC,YAAY,CAAG,CAAA,CAAA,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,EAAE,CAAC,WAAW,CAAE,CAAA,CAAC;AACzC,QAAA,IAAI,EAAE,CAAC,iBAAiB,EAAE;AACxB,YAAA,MAAM,CAAC,GAAG,EAAE,CAAC,iBAAiB;YAC9B,KAAK,CAAC,IAAI,CAAC,CAAA,aAAA,EAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAK,EAAA,EAAA,CAAC,CAAC,KAAK,CAAA,CAAA,EAAI,CAAC,CAAC,MAAM,CAAE,CAAA,CAAC;AACjE,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAC,CAAC,CAAC,CAAA,EAAA,EAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,GAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CAAC;;QAE7F,KAAK,CAAC,IAAI,CAAC,CAAA,aAAA,EAAgB,EAAE,CAAC,SAAS,CAAE,CAAA,CAAC;;AAG5C,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;QACrD,KAAK,CAAC,IAAI,CAAC,CAAwB,qBAAA,EAAA,MAAM,CAAC,UAAU,CAAC,MAAM,CAAM,IAAA,CAAA,CAAC;AAClE,QAAA,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;AAC/C,YAAA,MAAM,MAAM,GAAG,EAAE,CAAC,iBAAiB;AACjC,gBAAA,CAAA,KAAA,EAAQ,EAAE,CAAC,iBAAiB,CAAC,CAAC,KAAK,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAG,CAAA,CAAA,GAAG,EAAE;AACnE,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,KAAA,EAAQ,EAAE,CAAC,WAAW,CAAA,GAAA,EAAM,EAAE,CAAC,IAAI,CAAA,CAAA,EAAI,EAAE,CAAC,YAAY,GAAG,CAAA,MAAA,EAAS,EAAE,CAAC,YAAY,CAAG,CAAA,CAAA,GAAG,EAAE,CAAA,EAAG,MAAM,CAAA,CAAE,CAAC;;QAElH,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,EAAE,EAAE;AACjC,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAA,KAAA,CAAO,CAAC;;;AAIjE,IAAA,IAAI,MAAM,CAAC,MAAM,EAAE;AACjB,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,YAAA,CAAc,CAAC;AAC1B,QAAA,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AAClE,QAAA,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;;AAGnB,IAAA,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;;AAE5E,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,WAAA,CAAa,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;;AAGjE,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;AACtB,QAAA,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAA,yBAAA,CAA2B,CAAC;;AAGlE,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;AAC1B;AAEA;;;AAGG;AACa,SAAA,kBAAkB,CAChC,OAAA,GAAqC,EAAE,EAAA;AAEvC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IACjC,MAAMC,OAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAW,QAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAGhE,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;AAI/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGDA,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE;QACjD,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EACT,qGAAqG;AACvG,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;QAC5C,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EACT,4GAA4G;AAC9G,QAAA,MAAM,EAAE,WAAW;AACpB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE;QACnD,IAAI,EAAE,aAAa,CAAC,YAAY;AAChC,QAAA,WAAW,EACT,6DAA6D;AAC/D,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE;QAClD,IAAI,EAAE,aAAa,CAAC,WAAW;AAC/B,QAAA,WAAW,EACT,kFAAkF;AACpF,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE;QAC1C,IAAI,EAAE,aAAa,CAAC,GAAG;AACvB,QAAA,WAAW,EACT,2FAA2F;AAC7F,QAAA,MAAM,EAAE,SAAS;AAClB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE;QAChD,IAAI,EAAE,aAAa,CAAC,SAAS;AAC7B,QAAA,WAAW,EACT,6FAA6F;AAC/F,QAAA,MAAM,EAAE,cAAc;AACvB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;QAC7C,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EACT,wGAAwG;AAC1G,QAAA,MAAM,EAAE,YAAY;AACrB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,yFAAyF;AAC3F,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,iBAAiB,CAAC,EAAE;QACxD,IAAI,EAAE,aAAa,CAAC,iBAAiB;AACrC,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;QACzD,IAAI,EAAE,aAAa,CAAC,kBAAkB;AACtC,QAAA,WAAW,EAAE,kDAAkD;AAC/D,QAAA,MAAM,EAAE,sBAAsB;AAC/B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE;QACrD,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,oDAAoD;AACjE,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EAAE,qCAAqC;AAClD,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,oGAAoG;AACtG,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,+FAA+F;AACjG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;;;;;IAQDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,oLAAoL;AACtL,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE;QACvD,IAAI,EAAE,aAAa,CAAC,gBAAgB;AACpC,QAAA,WAAW,EACT,mOAAmO;AACrO,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;QACzD,IAAI,EAAE,aAAa,CAAC,kBAAkB;AACtC,QAAA,WAAW,EACT,uLAAuL;AACzL,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,iJAAiJ;AACnJ,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGDD,OAAK,CAAC,IAAI,CACRC,UAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE;QACrD,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EACT,2IAA2I;AAC7I,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;AAED,IAAA,OAAOD,OAAK;AACd;AAEA;;AAEG;SACa,mBAAmB,GAAA;AACjC,IAAA,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;AACrC;AAEA;;AAEG;AACG,SAAU,aAAa,CAAC,IAAY,EAAA;IACxC,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAuB,CAAC;AACvE;;;;;;;;"}

package/dist/esm/tools/DesktopTools.mjs

@@ -21,6 +21,12 @@ const EDesktopTools = {
     CLIPBOARD_WRITE: 'clipboard_write',
     CLIPBOARD_PASTE: 'clipboard_paste',
     WAIT: 'computer_wait',
+    // Native UI Automation tools (Windows) - faster and more reliable than screenshot-based
+    UI_FIND_ELEMENT: 'ui_find_element',
+    UI_CLICK_ELEMENT: 'ui_click_element',
+    UI_GET_WINDOW_TREE: 'ui_get_window_tree',
+    UI_FIND_BUTTONS: 'ui_find_buttons',
+    UI_FIND_INPUTS: 'ui_find_inputs',
 };
 /**
  * Check if desktop capability is available based on request headers or context
@@ -85,6 +91,23 @@ const ClipboardPasteSchema = z.object({});
 const WaitSchema = z.object({
     ms: z.number().describe('Milliseconds to wait'),
 });
+// ============ Native UI Automation Schemas (Windows) ============
+const UIFindElementSchema = z.object({
+    name: z.string().optional().describe('Element name/label to find (e.g., "Submit", "OK", "File")'),
+    automationId: z.string().optional().describe('Automation ID of element (unique identifier)'),
+    controlType: z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),
+});
+const UIClickElementSchema = z.object({
+    name: z.string().optional().describe('Element name/label to click'),
+    automationId: z.string().optional().describe('Automation ID of element to click'),
+    controlType: z.string().optional().describe('Type of control: Button, Edit, etc.'),
+    clickType: z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),
+});
+const UIGetWindowTreeSchema = z.object({
+    maxDepth: z.number().optional().describe('Maximum depth to traverse (default: 3)'),
+});
+const UIFindButtonsSchema = z.object({});
+const UIFindInputsSchema = z.object({});
 /**
  * Format desktop action result for LLM consumption
  */
@@ -113,10 +136,46 @@ function formatResultForLLM(result, action) {
     if (result.clipboard !== undefined) {
         parts.push(`**Clipboard Content:** ${result.clipboard}`);
     }
+    // UI Automation results
+    if (result.uiElement) {
+        const el = result.uiElement;
+        parts.push(`**UI Element Found:**`);
+        parts.push(`  - Name: "${el.name}"`);
+        parts.push(`  - AutomationId: "${el.automationId}"`);
+        parts.push(`  - Type: ${el.controlType}`);
+        if (el.boundingRectangle) {
+            const b = el.boundingRectangle;
+            parts.push(`  - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);
+            parts.push(`  - Center: (${Math.round(b.x + b.width / 2)}, ${Math.round(b.y + b.height / 2)})`);
+        }
+        parts.push(`  - Enabled: ${el.isEnabled}`);
+    }
+    if (result.uiElements && result.uiElements.length > 0) {
+        parts.push(`**UI Elements Found (${result.uiElements.length}):**`);
+        for (const el of result.uiElements.slice(0, 20)) { // Limit to 20
+            const bounds = el.boundingRectangle ?
+                ` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';
+            parts.push(`  - [${el.controlType}] "${el.name}"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);
+        }
+        if (result.uiElements.length > 20) {
+            parts.push(`  ... and ${result.uiElements.length - 20} more`);
+        }
+    }
+    if (result.uiTree) {
+        parts.push(`**UI Tree:**`);
+        parts.push('```json');
+        parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size
+        parts.push('```');
+    }
+    if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {
+        // Generic data fallback
+        parts.push(`**Result:**`);
+        parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));
+    }
     if (parts.length === 0) {
         parts.push(`Desktop action "${action}" completed successfully.`);
     }
-    return parts.join('\n');
+    return parts.join('\\n');
 }
 /**
  * Create desktop automation tools for the agent
@@ -245,6 +304,40 @@ function createDesktopTools(options = {}) {
         description: 'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',
         schema: WaitSchema,
     }));
+    // ============ Native UI Automation Tools (Windows) ============
+    // These are FASTER and MORE RELIABLE than screenshot-based automation
+    // They find elements by semantic properties (name, automationId, type)
+    // instead of relying on pixel coordinates from screenshots
+    // ui_find_element
+    tools.push(tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {
+        name: EDesktopTools.UI_FIND_ELEMENT,
+        description: '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',
+        schema: UIFindElementSchema,
+    }));
+    // ui_click_element
+    tools.push(tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {
+        name: EDesktopTools.UI_CLICK_ELEMENT,
+        description: '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: "OK"}) or ui_click_element({controlType: "Button", name: "Submit"}). Windows only.',
+        schema: UIClickElementSchema,
+    }));
+    // ui_get_window_tree
+    tools.push(tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {
+        name: EDesktopTools.UI_GET_WINDOW_TREE,
+        description: 'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',
+        schema: UIGetWindowTreeSchema,
+    }));
+    // ui_find_buttons
+    tools.push(tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {
+        name: EDesktopTools.UI_FIND_BUTTONS,
+        description: 'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',
+        schema: UIFindButtonsSchema,
+    }));
+    // ui_find_inputs
+    tools.push(tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {
+        name: EDesktopTools.UI_FIND_INPUTS,
+        description: 'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',
+        schema: UIFindInputsSchema,
+    }));
     return tools;
 }
 /**

package/dist/esm/tools/DesktopTools.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"DesktopTools.mjs","sources":["../../../src/tools/DesktopTools.ts"],"sourcesContent":["import { z } from 'zod';\r\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\r\n\r\n/**\r\n * Desktop tool names - keep in sync with Ranger Desktop Electron app\r\n * These tools execute locally in the Electron app, NOT on the server\r\n */\r\nexport const EDesktopTools = {\r\n  SCREENSHOT: 'computer_screenshot',\r\n  CLICK: 'computer_click',\r\n  DOUBLE_CLICK: 'computer_double_click',\r\n  RIGHT_CLICK: 'computer_right_click',\r\n  TYPE: 'computer_type',\r\n  KEY: 'computer_key',\r\n  KEY_COMBO: 'computer_key_combo',\r\n  SCROLL: 'computer_scroll',\r\n  DRAG: 'computer_drag',\r\n  GET_ACTIVE_WINDOW: 'computer_get_active_window',\r\n  GET_MOUSE_POSITION: 'computer_get_mouse_position',\r\n  CLIPBOARD_READ: 'clipboard_read',\r\n  CLIPBOARD_WRITE: 'clipboard_write',\r\n  CLIPBOARD_PASTE: 'clipboard_paste',\r\n  WAIT: 'computer_wait',\r\n} as const;\r\n\r\nexport type DesktopToolName =\r\n  (typeof EDesktopTools)[keyof typeof EDesktopTools];\r\n\r\n/**\r\n * Callback function type for waiting on desktop action results\r\n * This allows the server (Ranger) to provide a callback that waits for the Electron app\r\n * to POST results back to the server before returning to the LLM.\r\n *\r\n * @param action - The desktop action (click, type, screenshot, etc.)\r\n * @param args - Arguments for the action\r\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\r\n * @returns Promise that resolves with the actual desktop result\r\n */\r\nexport type DesktopToolCallback = (\r\n  action: string,\r\n  args: Record<string, unknown>,\r\n  toolCallId: string\r\n) => Promise<DesktopActionResult>;\r\n\r\n/**\r\n * Result returned from desktop action execution\r\n */\r\nexport interface DesktopActionResult {\r\n  success: boolean;\r\n  error?: string;\r\n  screenshot?: {\r\n    base64: string;\r\n    width: number;\r\n    height: number;\r\n  };\r\n  activeWindow?: {\r\n    title: string;\r\n    app: string;\r\n    bounds?: { x: number; y: number; width: number; height: number };\r\n  };\r\n  mousePosition?: { x: number; y: number };\r\n  clipboard?: string;\r\n}\r\n\r\n/**\r\n * Check if desktop capability is available based on request headers or context\r\n * The Ranger Desktop Electron app sets these headers when connected:\r\n * - X-Ranger-Desktop: true\r\n * - X-Ranger-Desktop-Capable: true\r\n */\r\nexport function hasDesktopCapability(req?: {\r\n  headers?: Record<string, string | string[] | undefined>;\r\n}): boolean {\r\n  if (!req?.headers) {\r\n    return false;\r\n  }\r\n\r\n  const desktopApp = req.headers['x-ranger-desktop'];\r\n  const desktopCapable = req.headers['x-ranger-desktop-capable'];\r\n\r\n  return desktopApp === 'true' || desktopCapable === 'true';\r\n}\r\n\r\n// Tool schemas\r\nconst ScreenshotSchema = z.object({});\r\n\r\nconst ClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to click'),\r\n  y: z.number().describe('Y coordinate to click'),\r\n});\r\n\r\nconst DoubleClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to double-click'),\r\n  y: z.number().describe('Y coordinate to double-click'),\r\n});\r\n\r\nconst RightClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to right-click'),\r\n  y: z.number().describe('Y coordinate to right-click'),\r\n});\r\n\r\nconst TypeSchema = z.object({\r\n  text: z.string().describe('Text to type'),\r\n});\r\n\r\nconst KeySchema = z.object({\r\n  key: z\r\n    .string()\r\n    .describe(\r\n      'Key to press (e.g., \"Enter\", \"Tab\", \"Escape\", \"Backspace\", \"Delete\", \"ArrowUp\", \"ArrowDown\", \"ArrowLeft\", \"ArrowRight\", \"Home\", \"End\", \"PageUp\", \"PageDown\", \"F1\"-\"F12\")'\r\n    ),\r\n});\r\n\r\nconst KeyComboSchema = z.object({\r\n  keys: z\r\n    .array(z.string())\r\n    .describe(\r\n      'Array of keys to press together (e.g., [\"Control\", \"c\"] for copy, [\"Alt\", \"Tab\"] for window switch)'\r\n    ),\r\n});\r\n\r\nconst ScrollSchema = z.object({\r\n  x: z.number().describe('X coordinate to scroll at'),\r\n  y: z.number().describe('Y coordinate to scroll at'),\r\n  deltaX: z.number().optional().describe('Horizontal scroll amount (pixels)'),\r\n  deltaY: z.number().describe('Vertical scroll amount (pixels, negative = up, positive = down)'),\r\n});\r\n\r\nconst DragSchema = z.object({\r\n  startX: z.number().describe('Starting X coordinate'),\r\n  startY: z.number().describe('Starting Y coordinate'),\r\n  endX: z.number().describe('Ending X coordinate'),\r\n  endY: z.number().describe('Ending Y coordinate'),\r\n});\r\n\r\nconst GetActiveWindowSchema = z.object({});\r\n\r\nconst GetMousePositionSchema = z.object({});\r\n\r\nconst ClipboardReadSchema = z.object({});\r\n\r\nconst ClipboardWriteSchema = z.object({\r\n  text: z.string().describe('Text to write to clipboard'),\r\n});\r\n\r\nconst ClipboardPasteSchema = z.object({});\r\n\r\nconst WaitSchema = z.object({\r\n  ms: z.number().describe('Milliseconds to wait'),\r\n});\r\n\r\n/**\r\n * Desktop tool response interface\r\n * This is what the Electron app returns after executing the action\r\n */\r\nexport interface DesktopToolResponse {\r\n  requiresDesktopExecution: true;\r\n  action: string;\r\n  args: Record<string, unknown>;\r\n  toolCallId?: string;\r\n}\r\n\r\n/**\r\n * Options for creating desktop tools\r\n */\r\nexport interface CreateDesktopToolsOptions {\r\n  /**\r\n   * Optional callback that waits for desktop action results.\r\n   * When provided, tools will await this callback to get actual results from the Electron app.\r\n   * When not provided, tools return markers immediately (for non-server contexts).\r\n   */\r\n  waitForResult?: DesktopToolCallback;\r\n}\r\n\r\n/**\r\n * Format desktop action result for LLM consumption\r\n */\r\nfunction formatResultForLLM(\r\n  result: DesktopActionResult,\r\n  action: string\r\n): string {\r\n  if (!result.success && result.error) {\r\n    return `Desktop action \"${action}\" failed: ${result.error}`;\r\n  }\r\n\r\n  const parts: string[] = [];\r\n\r\n  if (result.screenshot) {\r\n    parts.push(\r\n      `Screenshot captured (${result.screenshot.width}x${result.screenshot.height})`\r\n    );\r\n    // The base64 image will be handled separately by the message formatter\r\n  }\r\n\r\n  if (result.activeWindow) {\r\n    parts.push(`**Active Window:**`);\r\n    parts.push(`  - Title: ${result.activeWindow.title}`);\r\n    parts.push(`  - App: ${result.activeWindow.app}`);\r\n    if (result.activeWindow.bounds) {\r\n      const b = result.activeWindow.bounds;\r\n      parts.push(`  - Position: (${b.x}, ${b.y})`);\r\n      parts.push(`  - Size: ${b.width}x${b.height}`);\r\n    }\r\n  }\r\n\r\n  if (result.mousePosition) {\r\n    parts.push(\r\n      `**Mouse Position:** (${result.mousePosition.x}, ${result.mousePosition.y})`\r\n    );\r\n  }\r\n\r\n  if (result.clipboard !== undefined) {\r\n    parts.push(`**Clipboard Content:** ${result.clipboard}`);\r\n  }\r\n\r\n  if (parts.length === 0) {\r\n    parts.push(`Desktop action \"${action}\" completed successfully.`);\r\n  }\r\n\r\n  return parts.join('\\n');\r\n}\r\n\r\n/**\r\n * Create desktop automation tools for the agent\r\n * These tools allow AI to control the user's desktop when Ranger Desktop is running\r\n */\r\nexport function createDesktopTools(\r\n  options: CreateDesktopToolsOptions = {}\r\n): DynamicStructuredTool[] {\r\n  const { waitForResult } = options;\r\n  const tools: DynamicStructuredTool[] = [];\r\n\r\n  /**\r\n   * Helper to create tool function that optionally waits for results\r\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\r\n   */\r\n  const createToolFunction = (action: string) => {\r\n    return async (\r\n      args: Record<string, unknown>,\r\n      config?: { toolCall?: { id?: string } }\r\n    ): Promise<string> => {\r\n      const toolCallId =\r\n        config?.toolCall?.id ??\r\n        `desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;\r\n\r\n      // Create marker for Electron app\r\n      const marker: DesktopToolResponse = {\r\n        requiresDesktopExecution: true,\r\n        action,\r\n        args,\r\n        toolCallId,\r\n      };\r\n\r\n      // If no callback, return marker immediately (Electron handles via SSE interception)\r\n      if (!waitForResult) {\r\n        return JSON.stringify(marker);\r\n      }\r\n\r\n      // With callback: wait for actual results from Electron app\r\n      try {\r\n        const result = await waitForResult(action, args, toolCallId);\r\n        return formatResultForLLM(result, action);\r\n      } catch (error) {\r\n        const errorMessage =\r\n          error instanceof Error ? error.message : String(error);\r\n        return `Desktop action \"${action}\" failed: ${errorMessage}`;\r\n      }\r\n    };\r\n  };\r\n\r\n  // computer_screenshot\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCREENSHOT), {\r\n      name: EDesktopTools.SCREENSHOT,\r\n      description:\r\n        'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',\r\n      schema: ScreenshotSchema,\r\n    })\r\n  );\r\n\r\n  // computer_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLICK), {\r\n      name: EDesktopTools.CLICK,\r\n      description:\r\n        'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',\r\n      schema: ClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_double_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {\r\n      name: EDesktopTools.DOUBLE_CLICK,\r\n      description:\r\n        'Double-click the mouse at the specified screen coordinates.',\r\n      schema: DoubleClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_right_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {\r\n      name: EDesktopTools.RIGHT_CLICK,\r\n      description:\r\n        'Right-click the mouse at the specified screen coordinates to open context menus.',\r\n      schema: RightClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_type\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.TYPE), {\r\n      name: EDesktopTools.TYPE,\r\n      description:\r\n        'Type text using the keyboard. Make sure the target input field is focused first (use click).',\r\n      schema: TypeSchema,\r\n    })\r\n  );\r\n\r\n  // computer_key\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY), {\r\n      name: EDesktopTools.KEY,\r\n      description:\r\n        'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',\r\n      schema: KeySchema,\r\n    })\r\n  );\r\n\r\n  // computer_key_combo\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY_COMBO), {\r\n      name: EDesktopTools.KEY_COMBO,\r\n      description:\r\n        'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',\r\n      schema: KeyComboSchema,\r\n    })\r\n  );\r\n\r\n  // computer_scroll\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCROLL), {\r\n      name: EDesktopTools.SCROLL,\r\n      description:\r\n        'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',\r\n      schema: ScrollSchema,\r\n    })\r\n  );\r\n\r\n  // computer_drag\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DRAG), {\r\n      name: EDesktopTools.DRAG,\r\n      description:\r\n        'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',\r\n      schema: DragSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_active_window\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {\r\n      name: EDesktopTools.GET_ACTIVE_WINDOW,\r\n      description:\r\n        'Get information about the currently active window (title, application name, position, size).',\r\n      schema: GetActiveWindowSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_mouse_position\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {\r\n      name: EDesktopTools.GET_MOUSE_POSITION,\r\n      description: 'Get the current mouse cursor position on screen.',\r\n      schema: GetMousePositionSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_read\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {\r\n      name: EDesktopTools.CLIPBOARD_READ,\r\n      description: 'Read the current contents of the system clipboard.',\r\n      schema: ClipboardReadSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_write\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {\r\n      name: EDesktopTools.CLIPBOARD_WRITE,\r\n      description: 'Write text to the system clipboard.',\r\n      schema: ClipboardWriteSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_paste\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {\r\n      name: EDesktopTools.CLIPBOARD_PASTE,\r\n      description:\r\n        'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',\r\n      schema: ClipboardPasteSchema,\r\n    })\r\n  );\r\n\r\n  // computer_wait\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.WAIT), {\r\n      name: EDesktopTools.WAIT,\r\n      description:\r\n        'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',\r\n      schema: WaitSchema,\r\n    })\r\n  );\r\n\r\n  return tools;\r\n}\r\n\r\n/**\r\n * Get all desktop tool names\r\n */\r\nexport function getDesktopToolNames(): DesktopToolName[] {\r\n  return Object.values(EDesktopTools);\r\n}\r\n\r\n/**\r\n * Check if a tool name is a desktop tool\r\n */\r\nexport function isDesktopTool(name: string): name is DesktopToolName {\r\n  return Object.values(EDesktopTools).includes(name as DesktopToolName);\r\n}\r\n"],"names":[],"mappings":";;;AAGA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,UAAU,EAAE,qBAAqB;AACjC,IAAA,KAAK,EAAE,gBAAgB;AACvB,IAAA,YAAY,EAAE,uBAAuB;AACrC,IAAA,WAAW,EAAE,sBAAsB;AACnC,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,GAAG,EAAE,cAAc;AACnB,IAAA,SAAS,EAAE,oBAAoB;AAC/B,IAAA,MAAM,EAAE,iBAAiB;AACzB,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,iBAAiB,EAAE,4BAA4B;AAC/C,IAAA,kBAAkB,EAAE,6BAA6B;AACjD,IAAA,cAAc,EAAE,gBAAgB;AAChC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,IAAI,EAAE,eAAe;;AA0CvB;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAClD,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,UAAU,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AAC3D;AAEA;AACA,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAErC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IAC/C,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;AAChD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IACtD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;AACvD,CAAA,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACrD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;AACtD,CAAA,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;AAC1C,CAAA,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;AACzB,IAAA,GAAG,EAAE;AACF,SAAA,MAAM;SACN,QAAQ,CACP,0KAA0K,CAC3K;AACJ,CAAA,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;AAC9B,IAAA,IAAI,EAAE;AACH,SAAA,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE;SAChB,QAAQ,CACP,qGAAqG,CACtG;AACJ,CAAA,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;IACnD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;AACnD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAC3E,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iEAAiE,CAAC;AAC/F,CAAA,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IAChD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;AACjD,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE1C,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE3C,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAExC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;AACxD,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEzC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;AAChD,CAAA,CAAC;AAyBF;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,UAAU,EAAE;AACrB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,UAAU,CAAC,KAAK,CAAA,CAAA,EAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAA,CAAA,CAAG,CAC/E;;;AAIH,IAAA,IAAI,MAAM,CAAC,YAAY,EAAE;AACvB,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,kBAAA,CAAoB,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,CAAc,WAAA,EAAA,MAAM,CAAC,YAAY,CAAC,KAAK,CAAE,CAAA,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,CAAY,SAAA,EAAA,MAAM,CAAC,YAAY,CAAC,GAAG,CAAE,CAAA,CAAC;AACjD,QAAA,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;AAC9B,YAAA,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM;AACpC,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,CAAC,CAAC,CAAC,CAAA,EAAA,EAAK,CAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CAAC;AAC5C,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,CAAC,CAAC,KAAK,CAAA,CAAA,EAAI,CAAC,CAAC,MAAM,CAAA,CAAE,CAAC;;;AAIlD,IAAA,IAAI,MAAM,CAAC,aAAa,EAAE;AACxB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,EAAA,EAAK,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,CAAA,CAAG,CAC7E;;AAGH,IAAA,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE;QAClC,KAAK,CAAC,IAAI,CAAC,CAAA,uBAAA,EAA0B,MAAM,CAAC,SAAS,CAAE,CAAA,CAAC;;AAG1D,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;AACtB,QAAA,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAA,yBAAA,CAA2B,CAAC;;AAGlE,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;AACzB;AAEA;;;AAGG;AACa,SAAA,kBAAkB,CAChC,OAAA,GAAqC,EAAE,EAAA;AAEvC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IACjC,MAAM,KAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAW,QAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAGhE,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;AAI/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE;QACjD,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EACT,qGAAqG;AACvG,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;QAC5C,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EACT,4GAA4G;AAC9G,QAAA,MAAM,EAAE,WAAW;AACpB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE;QACnD,IAAI,EAAE,aAAa,CAAC,YAAY;AAChC,QAAA,WAAW,EACT,6DAA6D;AAC/D,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE;QAClD,IAAI,EAAE,aAAa,CAAC,WAAW;AAC/B,QAAA,WAAW,EACT,kFAAkF;AACpF,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE;QAC1C,IAAI,EAAE,aAAa,CAAC,GAAG;AACvB,QAAA,WAAW,EACT,2FAA2F;AAC7F,QAAA,MAAM,EAAE,SAAS;AAClB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE;QAChD,IAAI,EAAE,aAAa,CAAC,SAAS;AAC7B,QAAA,WAAW,EACT,6FAA6F;AAC/F,QAAA,MAAM,EAAE,cAAc;AACvB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;QAC7C,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EACT,wGAAwG;AAC1G,QAAA,MAAM,EAAE,YAAY;AACrB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,yFAAyF;AAC3F,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,iBAAiB,CAAC,EAAE;QACxD,IAAI,EAAE,aAAa,CAAC,iBAAiB;AACrC,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;QACzD,IAAI,EAAE,aAAa,CAAC,kBAAkB;AACtC,QAAA,WAAW,EAAE,kDAAkD;AAC/D,QAAA,MAAM,EAAE,sBAAsB;AAC/B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE;QACrD,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,oDAAoD;AACjE,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EAAE,qCAAqC;AAClD,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,oGAAoG;AACtG,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,+FAA+F;AACjG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;AAED,IAAA,OAAO,KAAK;AACd;AAEA;;AAEG;SACa,mBAAmB,GAAA;AACjC,IAAA,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;AACrC;AAEA;;AAEG;AACG,SAAU,aAAa,CAAC,IAAY,EAAA;IACxC,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAuB,CAAC;AACvE;;;;"}
+{"version":3,"file":"DesktopTools.mjs","sources":["../../../src/tools/DesktopTools.ts"],"sourcesContent":["import { z } from 'zod';\r\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\r\n\r\n/**\r\n * Desktop tool names - keep in sync with Ranger Desktop Electron app\r\n * These tools execute locally in the Electron app, NOT on the server\r\n */\r\nexport const EDesktopTools = {\r\n  SCREENSHOT: 'computer_screenshot',\r\n  CLICK: 'computer_click',\r\n  DOUBLE_CLICK: 'computer_double_click',\r\n  RIGHT_CLICK: 'computer_right_click',\r\n  TYPE: 'computer_type',\r\n  KEY: 'computer_key',\r\n  KEY_COMBO: 'computer_key_combo',\r\n  SCROLL: 'computer_scroll',\r\n  DRAG: 'computer_drag',\r\n  GET_ACTIVE_WINDOW: 'computer_get_active_window',\r\n  GET_MOUSE_POSITION: 'computer_get_mouse_position',\r\n  CLIPBOARD_READ: 'clipboard_read',\r\n  CLIPBOARD_WRITE: 'clipboard_write',\r\n  CLIPBOARD_PASTE: 'clipboard_paste',\r\n  WAIT: 'computer_wait',\r\n  // Native UI Automation tools (Windows) - faster and more reliable than screenshot-based\r\n  UI_FIND_ELEMENT: 'ui_find_element',\r\n  UI_CLICK_ELEMENT: 'ui_click_element',\r\n  UI_GET_WINDOW_TREE: 'ui_get_window_tree',\r\n  UI_FIND_BUTTONS: 'ui_find_buttons',\r\n  UI_FIND_INPUTS: 'ui_find_inputs',\r\n} as const;\r\n\r\nexport type DesktopToolName =\r\n  (typeof EDesktopTools)[keyof typeof EDesktopTools];\r\n\r\n/**\r\n * Callback function type for waiting on desktop action results\r\n * This allows the server (Ranger) to provide a callback that waits for the Electron app\r\n * to POST results back to the server before returning to the LLM.\r\n *\r\n * @param action - The desktop action (click, type, screenshot, etc.)\r\n * @param args - Arguments for the action\r\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\r\n * @returns Promise that resolves with the actual desktop result\r\n */\r\nexport type DesktopToolCallback = (\r\n  action: string,\r\n  args: Record<string, unknown>,\r\n  toolCallId: string\r\n) => Promise<DesktopActionResult>;\r\n\r\n/**\r\n * Result returned from desktop action execution\r\n */\r\nexport interface DesktopActionResult {\r\n  success: boolean;\r\n  error?: string;\r\n  screenshot?: {\r\n    base64: string;\r\n    width: number;\r\n    height: number;\r\n  };\r\n  activeWindow?: {\r\n    title: string;\r\n    app: string;\r\n    bounds?: { x: number; y: number; width: number; height: number };\r\n  };\r\n  mousePosition?: { x: number; y: number };\r\n  clipboard?: string;\r\n  // UI Automation results\r\n  uiElement?: {\r\n    name: string;\r\n    automationId: string;\r\n    controlType: string;\r\n    boundingRectangle?: { x: number; y: number; width: number; height: number };\r\n    isEnabled: boolean;\r\n  };\r\n  uiElements?: Array<{\r\n    name: string;\r\n    automationId: string;\r\n    controlType: string;\r\n    boundingRectangle?: { x: number; y: number; width: number; height: number };\r\n  }>;\r\n  uiTree?: unknown;\r\n  // Generic data for extended results\r\n  data?: unknown;\r\n}\r\n\r\n/**\r\n * Check if desktop capability is available based on request headers or context\r\n * The Ranger Desktop Electron app sets these headers when connected:\r\n * - X-Ranger-Desktop: true\r\n * - X-Ranger-Desktop-Capable: true\r\n */\r\nexport function hasDesktopCapability(req?: {\r\n  headers?: Record<string, string | string[] | undefined>;\r\n}): boolean {\r\n  if (!req?.headers) {\r\n    return false;\r\n  }\r\n\r\n  const desktopApp = req.headers['x-ranger-desktop'];\r\n  const desktopCapable = req.headers['x-ranger-desktop-capable'];\r\n\r\n  return desktopApp === 'true' || desktopCapable === 'true';\r\n}\r\n\r\n// Tool schemas\r\nconst ScreenshotSchema = z.object({});\r\n\r\nconst ClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to click'),\r\n  y: z.number().describe('Y coordinate to click'),\r\n});\r\n\r\nconst DoubleClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to double-click'),\r\n  y: z.number().describe('Y coordinate to double-click'),\r\n});\r\n\r\nconst RightClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to right-click'),\r\n  y: z.number().describe('Y coordinate to right-click'),\r\n});\r\n\r\nconst TypeSchema = z.object({\r\n  text: z.string().describe('Text to type'),\r\n});\r\n\r\nconst KeySchema = z.object({\r\n  key: z\r\n    .string()\r\n    .describe(\r\n      'Key to press (e.g., \"Enter\", \"Tab\", \"Escape\", \"Backspace\", \"Delete\", \"ArrowUp\", \"ArrowDown\", \"ArrowLeft\", \"ArrowRight\", \"Home\", \"End\", \"PageUp\", \"PageDown\", \"F1\"-\"F12\")'\r\n    ),\r\n});\r\n\r\nconst KeyComboSchema = z.object({\r\n  keys: z\r\n    .array(z.string())\r\n    .describe(\r\n      'Array of keys to press together (e.g., [\"Control\", \"c\"] for copy, [\"Alt\", \"Tab\"] for window switch)'\r\n    ),\r\n});\r\n\r\nconst ScrollSchema = z.object({\r\n  x: z.number().describe('X coordinate to scroll at'),\r\n  y: z.number().describe('Y coordinate to scroll at'),\r\n  deltaX: z.number().optional().describe('Horizontal scroll amount (pixels)'),\r\n  deltaY: z.number().describe('Vertical scroll amount (pixels, negative = up, positive = down)'),\r\n});\r\n\r\nconst DragSchema = z.object({\r\n  startX: z.number().describe('Starting X coordinate'),\r\n  startY: z.number().describe('Starting Y coordinate'),\r\n  endX: z.number().describe('Ending X coordinate'),\r\n  endY: z.number().describe('Ending Y coordinate'),\r\n});\r\n\r\nconst GetActiveWindowSchema = z.object({});\r\n\r\nconst GetMousePositionSchema = z.object({});\r\n\r\nconst ClipboardReadSchema = z.object({});\r\n\r\nconst ClipboardWriteSchema = z.object({\r\n  text: z.string().describe('Text to write to clipboard'),\r\n});\r\n\r\nconst ClipboardPasteSchema = z.object({});\r\n\r\nconst WaitSchema = z.object({\r\n  ms: z.number().describe('Milliseconds to wait'),\r\n});\r\n\r\n// ============ Native UI Automation Schemas (Windows) ============\r\n\r\nconst UIFindElementSchema = z.object({\r\n  name: z.string().optional().describe('Element name/label to find (e.g., \"Submit\", \"OK\", \"File\")'),\r\n  automationId: z.string().optional().describe('Automation ID of element (unique identifier)'),\r\n  controlType: z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),\r\n});\r\n\r\nconst UIClickElementSchema = z.object({\r\n  name: z.string().optional().describe('Element name/label to click'),\r\n  automationId: z.string().optional().describe('Automation ID of element to click'),\r\n  controlType: z.string().optional().describe('Type of control: Button, Edit, etc.'),\r\n  clickType: z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),\r\n});\r\n\r\nconst UIGetWindowTreeSchema = z.object({\r\n  maxDepth: z.number().optional().describe('Maximum depth to traverse (default: 3)'),\r\n});\r\n\r\nconst UIFindButtonsSchema = z.object({});\r\n\r\nconst UIFindInputsSchema = z.object({});\r\n\r\n/**\r\n * Desktop tool response interface\r\n * This is what the Electron app returns after executing the action\r\n */\r\nexport interface DesktopToolResponse {\r\n  requiresDesktopExecution: true;\r\n  action: string;\r\n  args: Record<string, unknown>;\r\n  toolCallId?: string;\r\n}\r\n\r\n/**\r\n * Options for creating desktop tools\r\n */\r\nexport interface CreateDesktopToolsOptions {\r\n  /**\r\n   * Optional callback that waits for desktop action results.\r\n   * When provided, tools will await this callback to get actual results from the Electron app.\r\n   * When not provided, tools return markers immediately (for non-server contexts).\r\n   */\r\n  waitForResult?: DesktopToolCallback;\r\n}\r\n\r\n/**\r\n * Format desktop action result for LLM consumption\r\n */\r\nfunction formatResultForLLM(\r\n  result: DesktopActionResult,\r\n  action: string\r\n): string {\r\n  if (!result.success && result.error) {\r\n    return `Desktop action \"${action}\" failed: ${result.error}`;\r\n  }\r\n\r\n  const parts: string[] = [];\r\n\r\n  if (result.screenshot) {\r\n    parts.push(\r\n      `Screenshot captured (${result.screenshot.width}x${result.screenshot.height})`\r\n    );\r\n    // The base64 image will be handled separately by the message formatter\r\n  }\r\n\r\n  if (result.activeWindow) {\r\n    parts.push(`**Active Window:**`);\r\n    parts.push(`  - Title: ${result.activeWindow.title}`);\r\n    parts.push(`  - App: ${result.activeWindow.app}`);\r\n    if (result.activeWindow.bounds) {\r\n      const b = result.activeWindow.bounds;\r\n      parts.push(`  - Position: (${b.x}, ${b.y})`);\r\n      parts.push(`  - Size: ${b.width}x${b.height}`);\r\n    }\r\n  }\r\n\r\n  if (result.mousePosition) {\r\n    parts.push(\r\n      `**Mouse Position:** (${result.mousePosition.x}, ${result.mousePosition.y})`\r\n    );\r\n  }\r\n\r\n  if (result.clipboard !== undefined) {\r\n    parts.push(`**Clipboard Content:** ${result.clipboard}`);\r\n  }\r\n\r\n  // UI Automation results\r\n  if (result.uiElement) {\r\n    const el = result.uiElement;\r\n    parts.push(`**UI Element Found:**`);\r\n    parts.push(`  - Name: \"${el.name}\"`);\r\n    parts.push(`  - AutomationId: \"${el.automationId}\"`);\r\n    parts.push(`  - Type: ${el.controlType}`);\r\n    if (el.boundingRectangle) {\r\n      const b = el.boundingRectangle;\r\n      parts.push(`  - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);\r\n      parts.push(`  - Center: (${Math.round(b.x + b.width/2)}, ${Math.round(b.y + b.height/2)})`);\r\n    }\r\n    parts.push(`  - Enabled: ${el.isEnabled}`);\r\n  }\r\n\r\n  if (result.uiElements && result.uiElements.length > 0) {\r\n    parts.push(`**UI Elements Found (${result.uiElements.length}):**`);\r\n    for (const el of result.uiElements.slice(0, 20)) { // Limit to 20\r\n      const bounds = el.boundingRectangle ? \r\n        ` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';\r\n      parts.push(`  - [${el.controlType}] \"${el.name}\"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);\r\n    }\r\n    if (result.uiElements.length > 20) {\r\n      parts.push(`  ... and ${result.uiElements.length - 20} more`);\r\n    }\r\n  }\r\n\r\n  if (result.uiTree) {\r\n    parts.push(`**UI Tree:**`);\r\n    parts.push('```json');\r\n    parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size\r\n    parts.push('```');\r\n  }\r\n\r\n  if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {\r\n    // Generic data fallback\r\n    parts.push(`**Result:**`);\r\n    parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));\r\n  }\r\n\r\n  if (parts.length === 0) {\r\n    parts.push(`Desktop action \"${action}\" completed successfully.`);\r\n  }\r\n\r\n  return parts.join('\\\\n');\r\n}\r\n\r\n/**\r\n * Create desktop automation tools for the agent\r\n * These tools allow AI to control the user's desktop when Ranger Desktop is running\r\n */\r\nexport function createDesktopTools(\r\n  options: CreateDesktopToolsOptions = {}\r\n): DynamicStructuredTool[] {\r\n  const { waitForResult } = options;\r\n  const tools: DynamicStructuredTool[] = [];\r\n\r\n  /**\r\n   * Helper to create tool function that optionally waits for results\r\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\r\n   */\r\n  const createToolFunction = (action: string) => {\r\n    return async (\r\n      args: Record<string, unknown>,\r\n      config?: { toolCall?: { id?: string } }\r\n    ): Promise<string> => {\r\n      const toolCallId =\r\n        config?.toolCall?.id ??\r\n        `desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;\r\n\r\n      // Create marker for Electron app\r\n      const marker: DesktopToolResponse = {\r\n        requiresDesktopExecution: true,\r\n        action,\r\n        args,\r\n        toolCallId,\r\n      };\r\n\r\n      // If no callback, return marker immediately (Electron handles via SSE interception)\r\n      if (!waitForResult) {\r\n        return JSON.stringify(marker);\r\n      }\r\n\r\n      // With callback: wait for actual results from Electron app\r\n      try {\r\n        const result = await waitForResult(action, args, toolCallId);\r\n        return formatResultForLLM(result, action);\r\n      } catch (error) {\r\n        const errorMessage =\r\n          error instanceof Error ? error.message : String(error);\r\n        return `Desktop action \"${action}\" failed: ${errorMessage}`;\r\n      }\r\n    };\r\n  };\r\n\r\n  // computer_screenshot\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCREENSHOT), {\r\n      name: EDesktopTools.SCREENSHOT,\r\n      description:\r\n        'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',\r\n      schema: ScreenshotSchema,\r\n    })\r\n  );\r\n\r\n  // computer_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLICK), {\r\n      name: EDesktopTools.CLICK,\r\n      description:\r\n        'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',\r\n      schema: ClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_double_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {\r\n      name: EDesktopTools.DOUBLE_CLICK,\r\n      description:\r\n        'Double-click the mouse at the specified screen coordinates.',\r\n      schema: DoubleClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_right_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {\r\n      name: EDesktopTools.RIGHT_CLICK,\r\n      description:\r\n        'Right-click the mouse at the specified screen coordinates to open context menus.',\r\n      schema: RightClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_type\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.TYPE), {\r\n      name: EDesktopTools.TYPE,\r\n      description:\r\n        'Type text using the keyboard. Make sure the target input field is focused first (use click).',\r\n      schema: TypeSchema,\r\n    })\r\n  );\r\n\r\n  // computer_key\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY), {\r\n      name: EDesktopTools.KEY,\r\n      description:\r\n        'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',\r\n      schema: KeySchema,\r\n    })\r\n  );\r\n\r\n  // computer_key_combo\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY_COMBO), {\r\n      name: EDesktopTools.KEY_COMBO,\r\n      description:\r\n        'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',\r\n      schema: KeyComboSchema,\r\n    })\r\n  );\r\n\r\n  // computer_scroll\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCROLL), {\r\n      name: EDesktopTools.SCROLL,\r\n      description:\r\n        'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',\r\n      schema: ScrollSchema,\r\n    })\r\n  );\r\n\r\n  // computer_drag\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DRAG), {\r\n      name: EDesktopTools.DRAG,\r\n      description:\r\n        'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',\r\n      schema: DragSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_active_window\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {\r\n      name: EDesktopTools.GET_ACTIVE_WINDOW,\r\n      description:\r\n        'Get information about the currently active window (title, application name, position, size).',\r\n      schema: GetActiveWindowSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_mouse_position\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {\r\n      name: EDesktopTools.GET_MOUSE_POSITION,\r\n      description: 'Get the current mouse cursor position on screen.',\r\n      schema: GetMousePositionSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_read\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {\r\n      name: EDesktopTools.CLIPBOARD_READ,\r\n      description: 'Read the current contents of the system clipboard.',\r\n      schema: ClipboardReadSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_write\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {\r\n      name: EDesktopTools.CLIPBOARD_WRITE,\r\n      description: 'Write text to the system clipboard.',\r\n      schema: ClipboardWriteSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_paste\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {\r\n      name: EDesktopTools.CLIPBOARD_PASTE,\r\n      description:\r\n        'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',\r\n      schema: ClipboardPasteSchema,\r\n    })\r\n  );\r\n\r\n  // computer_wait\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.WAIT), {\r\n      name: EDesktopTools.WAIT,\r\n      description:\r\n        'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',\r\n      schema: WaitSchema,\r\n    })\r\n  );\r\n\r\n  // ============ Native UI Automation Tools (Windows) ============\r\n  // These are FASTER and MORE RELIABLE than screenshot-based automation\r\n  // They find elements by semantic properties (name, automationId, type)\r\n  // instead of relying on pixel coordinates from screenshots\r\n\r\n  // ui_find_element\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {\r\n      name: EDesktopTools.UI_FIND_ELEMENT,\r\n      description:\r\n        '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',\r\n      schema: UIFindElementSchema,\r\n    })\r\n  );\r\n\r\n  // ui_click_element\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {\r\n      name: EDesktopTools.UI_CLICK_ELEMENT,\r\n      description:\r\n        '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: \"OK\"}) or ui_click_element({controlType: \"Button\", name: \"Submit\"}). Windows only.',\r\n      schema: UIClickElementSchema,\r\n    })\r\n  );\r\n\r\n  // ui_get_window_tree\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {\r\n      name: EDesktopTools.UI_GET_WINDOW_TREE,\r\n      description:\r\n        'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',\r\n      schema: UIGetWindowTreeSchema,\r\n    })\r\n  );\r\n\r\n  // ui_find_buttons\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {\r\n      name: EDesktopTools.UI_FIND_BUTTONS,\r\n      description:\r\n        'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',\r\n      schema: UIFindButtonsSchema,\r\n    })\r\n  );\r\n\r\n  // ui_find_inputs\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {\r\n      name: EDesktopTools.UI_FIND_INPUTS,\r\n      description:\r\n        'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',\r\n      schema: UIFindInputsSchema,\r\n    })\r\n  );\r\n\r\n  return tools;\r\n}\r\n\r\n/**\r\n * Get all desktop tool names\r\n */\r\nexport function getDesktopToolNames(): DesktopToolName[] {\r\n  return Object.values(EDesktopTools);\r\n}\r\n\r\n/**\r\n * Check if a tool name is a desktop tool\r\n */\r\nexport function isDesktopTool(name: string): name is DesktopToolName {\r\n  return Object.values(EDesktopTools).includes(name as DesktopToolName);\r\n}\r\n"],"names":[],"mappings":";;;AAGA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,UAAU,EAAE,qBAAqB;AACjC,IAAA,KAAK,EAAE,gBAAgB;AACvB,IAAA,YAAY,EAAE,uBAAuB;AACrC,IAAA,WAAW,EAAE,sBAAsB;AACnC,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,GAAG,EAAE,cAAc;AACnB,IAAA,SAAS,EAAE,oBAAoB;AAC/B,IAAA,MAAM,EAAE,iBAAiB;AACzB,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,iBAAiB,EAAE,4BAA4B;AAC/C,IAAA,kBAAkB,EAAE,6BAA6B;AACjD,IAAA,cAAc,EAAE,gBAAgB;AAChC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,IAAI,EAAE,eAAe;;AAErB,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,gBAAgB,EAAE,kBAAkB;AACpC,IAAA,kBAAkB,EAAE,oBAAoB;AACxC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,cAAc,EAAE,gBAAgB;;AA2DlC;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAClD,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,UAAU,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AAC3D;AAEA;AACA,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAErC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IAC/C,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;AAChD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IACtD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;AACvD,CAAA,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACrD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;AACtD,CAAA,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;AAC1C,CAAA,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;AACzB,IAAA,GAAG,EAAE;AACF,SAAA,MAAM;SACN,QAAQ,CACP,0KAA0K,CAC3K;AACJ,CAAA,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;AAC9B,IAAA,IAAI,EAAE;AACH,SAAA,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE;SAChB,QAAQ,CACP,qGAAqG,CACtG;AACJ,CAAA,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;IACnD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;AACnD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAC3E,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iEAAiE,CAAC;AAC/F,CAAA,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IAChD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;AACjD,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE1C,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE3C,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAExC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;AACxD,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEzC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;AAChD,CAAA,CAAC;AAEF;AAEA,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2DAA2D,CAAC;AACjG,IAAA,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;AAC5F,IAAA,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mFAAmF,CAAC;AACjI,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;AACnE,IAAA,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AACjF,IAAA,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAClF,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;AACjG,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;AACnF,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAExC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAyBvC;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,UAAU,EAAE;AACrB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,UAAU,CAAC,KAAK,CAAA,CAAA,EAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAA,CAAA,CAAG,CAC/E;;;AAIH,IAAA,IAAI,MAAM,CAAC,YAAY,EAAE;AACvB,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,kBAAA,CAAoB,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,CAAc,WAAA,EAAA,MAAM,CAAC,YAAY,CAAC,KAAK,CAAE,CAAA,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,CAAY,SAAA,EAAA,MAAM,CAAC,YAAY,CAAC,GAAG,CAAE,CAAA,CAAC;AACjD,QAAA,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;AAC9B,YAAA,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM;AACpC,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,CAAC,CAAC,CAAC,CAAA,EAAA,EAAK,CAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CAAC;AAC5C,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,CAAC,CAAC,KAAK,CAAA,CAAA,EAAI,CAAC,CAAC,MAAM,CAAA,CAAE,CAAC;;;AAIlD,IAAA,IAAI,MAAM,CAAC,aAAa,EAAE;AACxB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,EAAA,EAAK,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,CAAA,CAAG,CAC7E;;AAGH,IAAA,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE;QAClC,KAAK,CAAC,IAAI,CAAC,CAAA,uBAAA,EAA0B,MAAM,CAAC,SAAS,CAAE,CAAA,CAAC;;;AAI1D,IAAA,IAAI,MAAM,CAAC,SAAS,EAAE;AACpB,QAAA,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS;AAC3B,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,qBAAA,CAAuB,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,CAAA,WAAA,EAAc,EAAE,CAAC,IAAI,CAAG,CAAA,CAAA,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,CAAA,mBAAA,EAAsB,EAAE,CAAC,YAAY,CAAG,CAAA,CAAA,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,EAAE,CAAC,WAAW,CAAE,CAAA,CAAC;AACzC,QAAA,IAAI,EAAE,CAAC,iBAAiB,EAAE;AACxB,YAAA,MAAM,CAAC,GAAG,EAAE,CAAC,iBAAiB;YAC9B,KAAK,CAAC,IAAI,CAAC,CAAA,aAAA,EAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAK,EAAA,EAAA,CAAC,CAAC,KAAK,CAAA,CAAA,EAAI,CAAC,CAAC,MAAM,CAAE,CAAA,CAAC;AACjE,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAC,CAAC,CAAC,CAAA,EAAA,EAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,GAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CAAC;;QAE7F,KAAK,CAAC,IAAI,CAAC,CAAA,aAAA,EAAgB,EAAE,CAAC,SAAS,CAAE,CAAA,CAAC;;AAG5C,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;QACrD,KAAK,CAAC,IAAI,CAAC,CAAwB,qBAAA,EAAA,MAAM,CAAC,UAAU,CAAC,MAAM,CAAM,IAAA,CAAA,CAAC;AAClE,QAAA,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;AAC/C,YAAA,MAAM,MAAM,GAAG,EAAE,CAAC,iBAAiB;AACjC,gBAAA,CAAA,KAAA,EAAQ,EAAE,CAAC,iBAAiB,CAAC,CAAC,KAAK,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAG,CAAA,CAAA,GAAG,EAAE;AACnE,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,KAAA,EAAQ,EAAE,CAAC,WAAW,CAAA,GAAA,EAAM,EAAE,CAAC,IAAI,CAAA,CAAA,EAAI,EAAE,CAAC,YAAY,GAAG,CAAA,MAAA,EAAS,EAAE,CAAC,YAAY,CAAG,CAAA,CAAA,GAAG,EAAE,CAAA,EAAG,MAAM,CAAA,CAAE,CAAC;;QAElH,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,EAAE,EAAE;AACjC,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAA,KAAA,CAAO,CAAC;;;AAIjE,IAAA,IAAI,MAAM,CAAC,MAAM,EAAE;AACjB,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,YAAA,CAAc,CAAC;AAC1B,QAAA,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AAClE,QAAA,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;;AAGnB,IAAA,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;;AAE5E,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,WAAA,CAAa,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;;AAGjE,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;AACtB,QAAA,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAA,yBAAA,CAA2B,CAAC;;AAGlE,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;AAC1B;AAEA;;;AAGG;AACa,SAAA,kBAAkB,CAChC,OAAA,GAAqC,EAAE,EAAA;AAEvC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IACjC,MAAM,KAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAW,QAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAGhE,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;AAI/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE;QACjD,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EACT,qGAAqG;AACvG,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;QAC5C,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EACT,4GAA4G;AAC9G,QAAA,MAAM,EAAE,WAAW;AACpB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE;QACnD,IAAI,EAAE,aAAa,CAAC,YAAY;AAChC,QAAA,WAAW,EACT,6DAA6D;AAC/D,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE;QAClD,IAAI,EAAE,aAAa,CAAC,WAAW;AAC/B,QAAA,WAAW,EACT,kFAAkF;AACpF,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE;QAC1C,IAAI,EAAE,aAAa,CAAC,GAAG;AACvB,QAAA,WAAW,EACT,2FAA2F;AAC7F,QAAA,MAAM,EAAE,SAAS;AAClB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE;QAChD,IAAI,EAAE,aAAa,CAAC,SAAS;AAC7B,QAAA,WAAW,EACT,6FAA6F;AAC/F,QAAA,MAAM,EAAE,cAAc;AACvB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;QAC7C,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EACT,wGAAwG;AAC1G,QAAA,MAAM,EAAE,YAAY;AACrB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,yFAAyF;AAC3F,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,iBAAiB,CAAC,EAAE;QACxD,IAAI,EAAE,aAAa,CAAC,iBAAiB;AACrC,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;QACzD,IAAI,EAAE,aAAa,CAAC,kBAAkB;AACtC,QAAA,WAAW,EAAE,kDAAkD;AAC/D,QAAA,MAAM,EAAE,sBAAsB;AAC/B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE;QACrD,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,oDAAoD;AACjE,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EAAE,qCAAqC;AAClD,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,oGAAoG;AACtG,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,+FAA+F;AACjG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;;;;;IAQD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,oLAAoL;AACtL,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE;QACvD,IAAI,EAAE,aAAa,CAAC,gBAAgB;AACpC,QAAA,WAAW,EACT,mOAAmO;AACrO,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;QACzD,IAAI,EAAE,aAAa,CAAC,kBAAkB;AACtC,QAAA,WAAW,EACT,uLAAuL;AACzL,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,iJAAiJ;AACnJ,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE;QACrD,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EACT,2IAA2I;AAC7I,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;AAED,IAAA,OAAO,KAAK;AACd;AAEA;;AAEG;SACa,mBAAmB,GAAA;AACjC,IAAA,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;AACrC;AAEA;;AAEG;AACG,SAAU,aAAa,CAAC,IAAY,EAAA;IACxC,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAuB,CAAC;AACvE;;;;"}

package/dist/types/tools/DesktopTools.d.ts

@@ -19,6 +19,11 @@ export declare const EDesktopTools: {
     readonly CLIPBOARD_WRITE: "clipboard_write";
     readonly CLIPBOARD_PASTE: "clipboard_paste";
     readonly WAIT: "computer_wait";
+    readonly UI_FIND_ELEMENT: "ui_find_element";
+    readonly UI_CLICK_ELEMENT: "ui_click_element";
+    readonly UI_GET_WINDOW_TREE: "ui_get_window_tree";
+    readonly UI_FIND_BUTTONS: "ui_find_buttons";
+    readonly UI_FIND_INPUTS: "ui_find_inputs";
 };
 export type DesktopToolName = (typeof EDesktopTools)[keyof typeof EDesktopTools];
 /**
@@ -58,6 +63,31 @@ export interface DesktopActionResult {
         y: number;
     };
     clipboard?: string;
+    uiElement?: {
+        name: string;
+        automationId: string;
+        controlType: string;
+        boundingRectangle?: {
+            x: number;
+            y: number;
+            width: number;
+            height: number;
+        };
+        isEnabled: boolean;
+    };
+    uiElements?: Array<{
+        name: string;
+        automationId: string;
+        controlType: string;
+        boundingRectangle?: {
+            x: number;
+            y: number;
+            width: number;
+            height: number;
+        };
+    }>;
+    uiTree?: unknown;
+    data?: unknown;
 }
 /**
  * Check if desktop capability is available based on request headers or context

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "illuma-agents",
-  "version": "1.0.44",
+  "version": "1.0.45",
   "main": "./dist/cjs/main.cjs",
   "module": "./dist/esm/main.mjs",
   "types": "./dist/types/index.d.ts",

package/src/tools/DesktopTools.ts

@@ -21,6 +21,12 @@ export const EDesktopTools = {
   CLIPBOARD_WRITE: 'clipboard_write',
   CLIPBOARD_PASTE: 'clipboard_paste',
   WAIT: 'computer_wait',
+  // Native UI Automation tools (Windows) - faster and more reliable than screenshot-based
+  UI_FIND_ELEMENT: 'ui_find_element',
+  UI_CLICK_ELEMENT: 'ui_click_element',
+  UI_GET_WINDOW_TREE: 'ui_get_window_tree',
+  UI_FIND_BUTTONS: 'ui_find_buttons',
+  UI_FIND_INPUTS: 'ui_find_inputs',
 } as const;
 
 export type DesktopToolName =
@@ -60,6 +66,23 @@ export interface DesktopActionResult {
   };
   mousePosition?: { x: number; y: number };
   clipboard?: string;
+  // UI Automation results
+  uiElement?: {
+    name: string;
+    automationId: string;
+    controlType: string;
+    boundingRectangle?: { x: number; y: number; width: number; height: number };
+    isEnabled: boolean;
+  };
+  uiElements?: Array<{
+    name: string;
+    automationId: string;
+    controlType: string;
+    boundingRectangle?: { x: number; y: number; width: number; height: number };
+  }>;
+  uiTree?: unknown;
+  // Generic data for extended results
+  data?: unknown;
 }
 
 /**
@@ -149,6 +172,29 @@ const WaitSchema = z.object({
   ms: z.number().describe('Milliseconds to wait'),
 });
 
+// ============ Native UI Automation Schemas (Windows) ============
+
+const UIFindElementSchema = z.object({
+  name: z.string().optional().describe('Element name/label to find (e.g., "Submit", "OK", "File")'),
+  automationId: z.string().optional().describe('Automation ID of element (unique identifier)'),
+  controlType: z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),
+});
+
+const UIClickElementSchema = z.object({
+  name: z.string().optional().describe('Element name/label to click'),
+  automationId: z.string().optional().describe('Automation ID of element to click'),
+  controlType: z.string().optional().describe('Type of control: Button, Edit, etc.'),
+  clickType: z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),
+});
+
+const UIGetWindowTreeSchema = z.object({
+  maxDepth: z.number().optional().describe('Maximum depth to traverse (default: 3)'),
+});
+
+const UIFindButtonsSchema = z.object({});
+
+const UIFindInputsSchema = z.object({});
+
 /**
  * Desktop tool response interface
  * This is what the Electron app returns after executing the action
@@ -213,11 +259,51 @@ function formatResultForLLM(
     parts.push(`**Clipboard Content:** ${result.clipboard}`);
   }
 
+  // UI Automation results
+  if (result.uiElement) {
+    const el = result.uiElement;
+    parts.push(`**UI Element Found:**`);
+    parts.push(`  - Name: "${el.name}"`);
+    parts.push(`  - AutomationId: "${el.automationId}"`);
+    parts.push(`  - Type: ${el.controlType}`);
+    if (el.boundingRectangle) {
+      const b = el.boundingRectangle;
+      parts.push(`  - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);
+      parts.push(`  - Center: (${Math.round(b.x + b.width/2)}, ${Math.round(b.y + b.height/2)})`);
+    }
+    parts.push(`  - Enabled: ${el.isEnabled}`);
+  }
+
+  if (result.uiElements && result.uiElements.length > 0) {
+    parts.push(`**UI Elements Found (${result.uiElements.length}):**`);
+    for (const el of result.uiElements.slice(0, 20)) { // Limit to 20
+      const bounds = el.boundingRectangle ? 
+        ` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';
+      parts.push(`  - [${el.controlType}] "${el.name}"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);
+    }
+    if (result.uiElements.length > 20) {
+      parts.push(`  ... and ${result.uiElements.length - 20} more`);
+    }
+  }
+
+  if (result.uiTree) {
+    parts.push(`**UI Tree:**`);
+    parts.push('```json');
+    parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size
+    parts.push('```');
+  }
+
+  if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {
+    // Generic data fallback
+    parts.push(`**Result:**`);
+    parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));
+  }
+
   if (parts.length === 0) {
     parts.push(`Desktop action "${action}" completed successfully.`);
   }
 
-  return parts.join('\n');
+  return parts.join('\\n');
 }
 
 /**
@@ -415,6 +501,61 @@ export function createDesktopTools(
     })
   );
 
+  // ============ Native UI Automation Tools (Windows) ============
+  // These are FASTER and MORE RELIABLE than screenshot-based automation
+  // They find elements by semantic properties (name, automationId, type)
+  // instead of relying on pixel coordinates from screenshots
+
+  // ui_find_element
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {
+      name: EDesktopTools.UI_FIND_ELEMENT,
+      description:
+        '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',
+      schema: UIFindElementSchema,
+    })
+  );
+
+  // ui_click_element
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {
+      name: EDesktopTools.UI_CLICK_ELEMENT,
+      description:
+        '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: "OK"}) or ui_click_element({controlType: "Button", name: "Submit"}). Windows only.',
+      schema: UIClickElementSchema,
+    })
+  );
+
+  // ui_get_window_tree
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {
+      name: EDesktopTools.UI_GET_WINDOW_TREE,
+      description:
+        'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',
+      schema: UIGetWindowTreeSchema,
+    })
+  );
+
+  // ui_find_buttons
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {
+      name: EDesktopTools.UI_FIND_BUTTONS,
+      description:
+        'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',
+      schema: UIFindButtonsSchema,
+    })
+  );
+
+  // ui_find_inputs
+  tools.push(
+    tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {
+      name: EDesktopTools.UI_FIND_INPUTS,
+      description:
+        'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',
+      schema: UIFindInputsSchema,
+    })
+  );
+
   return tools;
 }