illuma-agents 1.0.45 → 1.0.47

package/dist/esm/tools/DesktopTools.mjs

@@ -1,357 +0,0 @@
-import { z } from 'zod';
-import { tool } from '@langchain/core/tools';
-
-/**
- * Desktop tool names - keep in sync with Ranger Desktop Electron app
- * These tools execute locally in the Electron app, NOT on the server
- */
-const EDesktopTools = {
-    SCREENSHOT: 'computer_screenshot',
-    CLICK: 'computer_click',
-    DOUBLE_CLICK: 'computer_double_click',
-    RIGHT_CLICK: 'computer_right_click',
-    TYPE: 'computer_type',
-    KEY: 'computer_key',
-    KEY_COMBO: 'computer_key_combo',
-    SCROLL: 'computer_scroll',
-    DRAG: 'computer_drag',
-    GET_ACTIVE_WINDOW: 'computer_get_active_window',
-    GET_MOUSE_POSITION: 'computer_get_mouse_position',
-    CLIPBOARD_READ: 'clipboard_read',
-    CLIPBOARD_WRITE: 'clipboard_write',
-    CLIPBOARD_PASTE: 'clipboard_paste',
-    WAIT: 'computer_wait',
-    // Native UI Automation tools (Windows) - faster and more reliable than screenshot-based
-    UI_FIND_ELEMENT: 'ui_find_element',
-    UI_CLICK_ELEMENT: 'ui_click_element',
-    UI_GET_WINDOW_TREE: 'ui_get_window_tree',
-    UI_FIND_BUTTONS: 'ui_find_buttons',
-    UI_FIND_INPUTS: 'ui_find_inputs',
-};
-/**
- * Check if desktop capability is available based on request headers or context
- * The Ranger Desktop Electron app sets these headers when connected:
- * - X-Ranger-Desktop: true
- * - X-Ranger-Desktop-Capable: true
- */
-function hasDesktopCapability(req) {
-    if (!req?.headers) {
-        return false;
-    }
-    const desktopApp = req.headers['x-ranger-desktop'];
-    const desktopCapable = req.headers['x-ranger-desktop-capable'];
-    return desktopApp === 'true' || desktopCapable === 'true';
-}
-// Tool schemas
-const ScreenshotSchema = z.object({});
-const ClickSchema = z.object({
-    x: z.number().describe('X coordinate to click'),
-    y: z.number().describe('Y coordinate to click'),
-});
-const DoubleClickSchema = z.object({
-    x: z.number().describe('X coordinate to double-click'),
-    y: z.number().describe('Y coordinate to double-click'),
-});
-const RightClickSchema = z.object({
-    x: z.number().describe('X coordinate to right-click'),
-    y: z.number().describe('Y coordinate to right-click'),
-});
-const TypeSchema = z.object({
-    text: z.string().describe('Text to type'),
-});
-const KeySchema = z.object({
-    key: z
-        .string()
-        .describe('Key to press (e.g., "Enter", "Tab", "Escape", "Backspace", "Delete", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Home", "End", "PageUp", "PageDown", "F1"-"F12")'),
-});
-const KeyComboSchema = z.object({
-    keys: z
-        .array(z.string())
-        .describe('Array of keys to press together (e.g., ["Control", "c"] for copy, ["Alt", "Tab"] for window switch)'),
-});
-const ScrollSchema = z.object({
-    x: z.number().describe('X coordinate to scroll at'),
-    y: z.number().describe('Y coordinate to scroll at'),
-    deltaX: z.number().optional().describe('Horizontal scroll amount (pixels)'),
-    deltaY: z.number().describe('Vertical scroll amount (pixels, negative = up, positive = down)'),
-});
-const DragSchema = z.object({
-    startX: z.number().describe('Starting X coordinate'),
-    startY: z.number().describe('Starting Y coordinate'),
-    endX: z.number().describe('Ending X coordinate'),
-    endY: z.number().describe('Ending Y coordinate'),
-});
-const GetActiveWindowSchema = z.object({});
-const GetMousePositionSchema = z.object({});
-const ClipboardReadSchema = z.object({});
-const ClipboardWriteSchema = z.object({
-    text: z.string().describe('Text to write to clipboard'),
-});
-const ClipboardPasteSchema = z.object({});
-const WaitSchema = z.object({
-    ms: z.number().describe('Milliseconds to wait'),
-});
-// ============ Native UI Automation Schemas (Windows) ============
-const UIFindElementSchema = z.object({
-    name: z.string().optional().describe('Element name/label to find (e.g., "Submit", "OK", "File")'),
-    automationId: z.string().optional().describe('Automation ID of element (unique identifier)'),
-    controlType: z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),
-});
-const UIClickElementSchema = z.object({
-    name: z.string().optional().describe('Element name/label to click'),
-    automationId: z.string().optional().describe('Automation ID of element to click'),
-    controlType: z.string().optional().describe('Type of control: Button, Edit, etc.'),
-    clickType: z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),
-});
-const UIGetWindowTreeSchema = z.object({
-    maxDepth: z.number().optional().describe('Maximum depth to traverse (default: 3)'),
-});
-const UIFindButtonsSchema = z.object({});
-const UIFindInputsSchema = z.object({});
-/**
- * Format desktop action result for LLM consumption
- */
-function formatResultForLLM(result, action) {
-    if (!result.success && result.error) {
-        return `Desktop action "${action}" failed: ${result.error}`;
-    }
-    const parts = [];
-    if (result.screenshot) {
-        parts.push(`Screenshot captured (${result.screenshot.width}x${result.screenshot.height})`);
-        // The base64 image will be handled separately by the message formatter
-    }
-    if (result.activeWindow) {
-        parts.push(`**Active Window:**`);
-        parts.push(`  - Title: ${result.activeWindow.title}`);
-        parts.push(`  - App: ${result.activeWindow.app}`);
-        if (result.activeWindow.bounds) {
-            const b = result.activeWindow.bounds;
-            parts.push(`  - Position: (${b.x}, ${b.y})`);
-            parts.push(`  - Size: ${b.width}x${b.height}`);
-        }
-    }
-    if (result.mousePosition) {
-        parts.push(`**Mouse Position:** (${result.mousePosition.x}, ${result.mousePosition.y})`);
-    }
-    if (result.clipboard !== undefined) {
-        parts.push(`**Clipboard Content:** ${result.clipboard}`);
-    }
-    // UI Automation results
-    if (result.uiElement) {
-        const el = result.uiElement;
-        parts.push(`**UI Element Found:**`);
-        parts.push(`  - Name: "${el.name}"`);
-        parts.push(`  - AutomationId: "${el.automationId}"`);
-        parts.push(`  - Type: ${el.controlType}`);
-        if (el.boundingRectangle) {
-            const b = el.boundingRectangle;
-            parts.push(`  - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);
-            parts.push(`  - Center: (${Math.round(b.x + b.width / 2)}, ${Math.round(b.y + b.height / 2)})`);
-        }
-        parts.push(`  - Enabled: ${el.isEnabled}`);
-    }
-    if (result.uiElements && result.uiElements.length > 0) {
-        parts.push(`**UI Elements Found (${result.uiElements.length}):**`);
-        for (const el of result.uiElements.slice(0, 20)) { // Limit to 20
-            const bounds = el.boundingRectangle ?
-                ` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';
-            parts.push(`  - [${el.controlType}] "${el.name}"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);
-        }
-        if (result.uiElements.length > 20) {
-            parts.push(`  ... and ${result.uiElements.length - 20} more`);
-        }
-    }
-    if (result.uiTree) {
-        parts.push(`**UI Tree:**`);
-        parts.push('```json');
-        parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size
-        parts.push('```');
-    }
-    if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {
-        // Generic data fallback
-        parts.push(`**Result:**`);
-        parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));
-    }
-    if (parts.length === 0) {
-        parts.push(`Desktop action "${action}" completed successfully.`);
-    }
-    return parts.join('\\n');
-}
-/**
- * Create desktop automation tools for the agent
- * These tools allow AI to control the user's desktop when Ranger Desktop is running
- */
-function createDesktopTools(options = {}) {
-    const { waitForResult } = options;
-    const tools = [];
-    /**
-     * Helper to create tool function that optionally waits for results
-     * The toolCallId is extracted from the RunnableConfig passed by LangChain
-     */
-    const createToolFunction = (action) => {
-        return async (args, config) => {
-            const toolCallId = config?.toolCall?.id ??
-                `desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;
-            // Create marker for Electron app
-            const marker = {
-                requiresDesktopExecution: true,
-                action,
-                args,
-                toolCallId,
-            };
-            // If no callback, return marker immediately (Electron handles via SSE interception)
-            if (!waitForResult) {
-                return JSON.stringify(marker);
-            }
-            // With callback: wait for actual results from Electron app
-            try {
-                const result = await waitForResult(action, args, toolCallId);
-                return formatResultForLLM(result, action);
-            }
-            catch (error) {
-                const errorMessage = error instanceof Error ? error.message : String(error);
-                return `Desktop action "${action}" failed: ${errorMessage}`;
-            }
-        };
-    };
-    // computer_screenshot
-    tools.push(tool(createToolFunction(EDesktopTools.SCREENSHOT), {
-        name: EDesktopTools.SCREENSHOT,
-        description: 'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',
-        schema: ScreenshotSchema,
-    }));
-    // computer_click
-    tools.push(tool(createToolFunction(EDesktopTools.CLICK), {
-        name: EDesktopTools.CLICK,
-        description: 'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',
-        schema: ClickSchema,
-    }));
-    // computer_double_click
-    tools.push(tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {
-        name: EDesktopTools.DOUBLE_CLICK,
-        description: 'Double-click the mouse at the specified screen coordinates.',
-        schema: DoubleClickSchema,
-    }));
-    // computer_right_click
-    tools.push(tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {
-        name: EDesktopTools.RIGHT_CLICK,
-        description: 'Right-click the mouse at the specified screen coordinates to open context menus.',
-        schema: RightClickSchema,
-    }));
-    // computer_type
-    tools.push(tool(createToolFunction(EDesktopTools.TYPE), {
-        name: EDesktopTools.TYPE,
-        description: 'Type text using the keyboard. Make sure the target input field is focused first (use click).',
-        schema: TypeSchema,
-    }));
-    // computer_key
-    tools.push(tool(createToolFunction(EDesktopTools.KEY), {
-        name: EDesktopTools.KEY,
-        description: 'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',
-        schema: KeySchema,
-    }));
-    // computer_key_combo
-    tools.push(tool(createToolFunction(EDesktopTools.KEY_COMBO), {
-        name: EDesktopTools.KEY_COMBO,
-        description: 'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',
-        schema: KeyComboSchema,
-    }));
-    // computer_scroll
-    tools.push(tool(createToolFunction(EDesktopTools.SCROLL), {
-        name: EDesktopTools.SCROLL,
-        description: 'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',
-        schema: ScrollSchema,
-    }));
-    // computer_drag
-    tools.push(tool(createToolFunction(EDesktopTools.DRAG), {
-        name: EDesktopTools.DRAG,
-        description: 'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',
-        schema: DragSchema,
-    }));
-    // computer_get_active_window
-    tools.push(tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {
-        name: EDesktopTools.GET_ACTIVE_WINDOW,
-        description: 'Get information about the currently active window (title, application name, position, size).',
-        schema: GetActiveWindowSchema,
-    }));
-    // computer_get_mouse_position
-    tools.push(tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {
-        name: EDesktopTools.GET_MOUSE_POSITION,
-        description: 'Get the current mouse cursor position on screen.',
-        schema: GetMousePositionSchema,
-    }));
-    // clipboard_read
-    tools.push(tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {
-        name: EDesktopTools.CLIPBOARD_READ,
-        description: 'Read the current contents of the system clipboard.',
-        schema: ClipboardReadSchema,
-    }));
-    // clipboard_write
-    tools.push(tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {
-        name: EDesktopTools.CLIPBOARD_WRITE,
-        description: 'Write text to the system clipboard.',
-        schema: ClipboardWriteSchema,
-    }));
-    // clipboard_paste
-    tools.push(tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {
-        name: EDesktopTools.CLIPBOARD_PASTE,
-        description: 'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',
-        schema: ClipboardPasteSchema,
-    }));
-    // computer_wait
-    tools.push(tool(createToolFunction(EDesktopTools.WAIT), {
-        name: EDesktopTools.WAIT,
-        description: 'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',
-        schema: WaitSchema,
-    }));
-    // ============ Native UI Automation Tools (Windows) ============
-    // These are FASTER and MORE RELIABLE than screenshot-based automation
-    // They find elements by semantic properties (name, automationId, type)
-    // instead of relying on pixel coordinates from screenshots
-    // ui_find_element
-    tools.push(tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {
-        name: EDesktopTools.UI_FIND_ELEMENT,
-        description: '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',
-        schema: UIFindElementSchema,
-    }));
-    // ui_click_element
-    tools.push(tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {
-        name: EDesktopTools.UI_CLICK_ELEMENT,
-        description: '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: "OK"}) or ui_click_element({controlType: "Button", name: "Submit"}). Windows only.',
-        schema: UIClickElementSchema,
-    }));
-    // ui_get_window_tree
-    tools.push(tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {
-        name: EDesktopTools.UI_GET_WINDOW_TREE,
-        description: 'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',
-        schema: UIGetWindowTreeSchema,
-    }));
-    // ui_find_buttons
-    tools.push(tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {
-        name: EDesktopTools.UI_FIND_BUTTONS,
-        description: 'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',
-        schema: UIFindButtonsSchema,
-    }));
-    // ui_find_inputs
-    tools.push(tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {
-        name: EDesktopTools.UI_FIND_INPUTS,
-        description: 'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',
-        schema: UIFindInputsSchema,
-    }));
-    return tools;
-}
-/**
- * Get all desktop tool names
- */
-function getDesktopToolNames() {
-    return Object.values(EDesktopTools);
-}
-/**
- * Check if a tool name is a desktop tool
- */
-function isDesktopTool(name) {
-    return Object.values(EDesktopTools).includes(name);
-}
-
-export { EDesktopTools, createDesktopTools, getDesktopToolNames, hasDesktopCapability, isDesktopTool };
-//# sourceMappingURL=DesktopTools.mjs.map

package/dist/esm/tools/DesktopTools.mjs.map

@@ -1 +0,0 @@
-{"version":3,"file":"DesktopTools.mjs","sources":["../../../src/tools/DesktopTools.ts"],"sourcesContent":["import { z } from 'zod';\r\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\r\n\r\n/**\r\n * Desktop tool names - keep in sync with Ranger Desktop Electron app\r\n * These tools execute locally in the Electron app, NOT on the server\r\n */\r\nexport const EDesktopTools = {\r\n  SCREENSHOT: 'computer_screenshot',\r\n  CLICK: 'computer_click',\r\n  DOUBLE_CLICK: 'computer_double_click',\r\n  RIGHT_CLICK: 'computer_right_click',\r\n  TYPE: 'computer_type',\r\n  KEY: 'computer_key',\r\n  KEY_COMBO: 'computer_key_combo',\r\n  SCROLL: 'computer_scroll',\r\n  DRAG: 'computer_drag',\r\n  GET_ACTIVE_WINDOW: 'computer_get_active_window',\r\n  GET_MOUSE_POSITION: 'computer_get_mouse_position',\r\n  CLIPBOARD_READ: 'clipboard_read',\r\n  CLIPBOARD_WRITE: 'clipboard_write',\r\n  CLIPBOARD_PASTE: 'clipboard_paste',\r\n  WAIT: 'computer_wait',\r\n  // Native UI Automation tools (Windows) - faster and more reliable than screenshot-based\r\n  UI_FIND_ELEMENT: 'ui_find_element',\r\n  UI_CLICK_ELEMENT: 'ui_click_element',\r\n  UI_GET_WINDOW_TREE: 'ui_get_window_tree',\r\n  UI_FIND_BUTTONS: 'ui_find_buttons',\r\n  UI_FIND_INPUTS: 'ui_find_inputs',\r\n} as const;\r\n\r\nexport type DesktopToolName =\r\n  (typeof EDesktopTools)[keyof typeof EDesktopTools];\r\n\r\n/**\r\n * Callback function type for waiting on desktop action results\r\n * This allows the server (Ranger) to provide a callback that waits for the Electron app\r\n * to POST results back to the server before returning to the LLM.\r\n *\r\n * @param action - The desktop action (click, type, screenshot, etc.)\r\n * @param args - Arguments for the action\r\n * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)\r\n * @returns Promise that resolves with the actual desktop result\r\n */\r\nexport type DesktopToolCallback = (\r\n  action: string,\r\n  args: Record<string, unknown>,\r\n  toolCallId: string\r\n) => Promise<DesktopActionResult>;\r\n\r\n/**\r\n * Result returned from desktop action execution\r\n */\r\nexport interface DesktopActionResult {\r\n  success: boolean;\r\n  error?: string;\r\n  screenshot?: {\r\n    base64: string;\r\n    width: number;\r\n    height: number;\r\n  };\r\n  activeWindow?: {\r\n    title: string;\r\n    app: string;\r\n    bounds?: { x: number; y: number; width: number; height: number };\r\n  };\r\n  mousePosition?: { x: number; y: number };\r\n  clipboard?: string;\r\n  // UI Automation results\r\n  uiElement?: {\r\n    name: string;\r\n    automationId: string;\r\n    controlType: string;\r\n    boundingRectangle?: { x: number; y: number; width: number; height: number };\r\n    isEnabled: boolean;\r\n  };\r\n  uiElements?: Array<{\r\n    name: string;\r\n    automationId: string;\r\n    controlType: string;\r\n    boundingRectangle?: { x: number; y: number; width: number; height: number };\r\n  }>;\r\n  uiTree?: unknown;\r\n  // Generic data for extended results\r\n  data?: unknown;\r\n}\r\n\r\n/**\r\n * Check if desktop capability is available based on request headers or context\r\n * The Ranger Desktop Electron app sets these headers when connected:\r\n * - X-Ranger-Desktop: true\r\n * - X-Ranger-Desktop-Capable: true\r\n */\r\nexport function hasDesktopCapability(req?: {\r\n  headers?: Record<string, string | string[] | undefined>;\r\n}): boolean {\r\n  if (!req?.headers) {\r\n    return false;\r\n  }\r\n\r\n  const desktopApp = req.headers['x-ranger-desktop'];\r\n  const desktopCapable = req.headers['x-ranger-desktop-capable'];\r\n\r\n  return desktopApp === 'true' || desktopCapable === 'true';\r\n}\r\n\r\n// Tool schemas\r\nconst ScreenshotSchema = z.object({});\r\n\r\nconst ClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to click'),\r\n  y: z.number().describe('Y coordinate to click'),\r\n});\r\n\r\nconst DoubleClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to double-click'),\r\n  y: z.number().describe('Y coordinate to double-click'),\r\n});\r\n\r\nconst RightClickSchema = z.object({\r\n  x: z.number().describe('X coordinate to right-click'),\r\n  y: z.number().describe('Y coordinate to right-click'),\r\n});\r\n\r\nconst TypeSchema = z.object({\r\n  text: z.string().describe('Text to type'),\r\n});\r\n\r\nconst KeySchema = z.object({\r\n  key: z\r\n    .string()\r\n    .describe(\r\n      'Key to press (e.g., \"Enter\", \"Tab\", \"Escape\", \"Backspace\", \"Delete\", \"ArrowUp\", \"ArrowDown\", \"ArrowLeft\", \"ArrowRight\", \"Home\", \"End\", \"PageUp\", \"PageDown\", \"F1\"-\"F12\")'\r\n    ),\r\n});\r\n\r\nconst KeyComboSchema = z.object({\r\n  keys: z\r\n    .array(z.string())\r\n    .describe(\r\n      'Array of keys to press together (e.g., [\"Control\", \"c\"] for copy, [\"Alt\", \"Tab\"] for window switch)'\r\n    ),\r\n});\r\n\r\nconst ScrollSchema = z.object({\r\n  x: z.number().describe('X coordinate to scroll at'),\r\n  y: z.number().describe('Y coordinate to scroll at'),\r\n  deltaX: z.number().optional().describe('Horizontal scroll amount (pixels)'),\r\n  deltaY: z.number().describe('Vertical scroll amount (pixels, negative = up, positive = down)'),\r\n});\r\n\r\nconst DragSchema = z.object({\r\n  startX: z.number().describe('Starting X coordinate'),\r\n  startY: z.number().describe('Starting Y coordinate'),\r\n  endX: z.number().describe('Ending X coordinate'),\r\n  endY: z.number().describe('Ending Y coordinate'),\r\n});\r\n\r\nconst GetActiveWindowSchema = z.object({});\r\n\r\nconst GetMousePositionSchema = z.object({});\r\n\r\nconst ClipboardReadSchema = z.object({});\r\n\r\nconst ClipboardWriteSchema = z.object({\r\n  text: z.string().describe('Text to write to clipboard'),\r\n});\r\n\r\nconst ClipboardPasteSchema = z.object({});\r\n\r\nconst WaitSchema = z.object({\r\n  ms: z.number().describe('Milliseconds to wait'),\r\n});\r\n\r\n// ============ Native UI Automation Schemas (Windows) ============\r\n\r\nconst UIFindElementSchema = z.object({\r\n  name: z.string().optional().describe('Element name/label to find (e.g., \"Submit\", \"OK\", \"File\")'),\r\n  automationId: z.string().optional().describe('Automation ID of element (unique identifier)'),\r\n  controlType: z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),\r\n});\r\n\r\nconst UIClickElementSchema = z.object({\r\n  name: z.string().optional().describe('Element name/label to click'),\r\n  automationId: z.string().optional().describe('Automation ID of element to click'),\r\n  controlType: z.string().optional().describe('Type of control: Button, Edit, etc.'),\r\n  clickType: z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),\r\n});\r\n\r\nconst UIGetWindowTreeSchema = z.object({\r\n  maxDepth: z.number().optional().describe('Maximum depth to traverse (default: 3)'),\r\n});\r\n\r\nconst UIFindButtonsSchema = z.object({});\r\n\r\nconst UIFindInputsSchema = z.object({});\r\n\r\n/**\r\n * Desktop tool response interface\r\n * This is what the Electron app returns after executing the action\r\n */\r\nexport interface DesktopToolResponse {\r\n  requiresDesktopExecution: true;\r\n  action: string;\r\n  args: Record<string, unknown>;\r\n  toolCallId?: string;\r\n}\r\n\r\n/**\r\n * Options for creating desktop tools\r\n */\r\nexport interface CreateDesktopToolsOptions {\r\n  /**\r\n   * Optional callback that waits for desktop action results.\r\n   * When provided, tools will await this callback to get actual results from the Electron app.\r\n   * When not provided, tools return markers immediately (for non-server contexts).\r\n   */\r\n  waitForResult?: DesktopToolCallback;\r\n}\r\n\r\n/**\r\n * Format desktop action result for LLM consumption\r\n */\r\nfunction formatResultForLLM(\r\n  result: DesktopActionResult,\r\n  action: string\r\n): string {\r\n  if (!result.success && result.error) {\r\n    return `Desktop action \"${action}\" failed: ${result.error}`;\r\n  }\r\n\r\n  const parts: string[] = [];\r\n\r\n  if (result.screenshot) {\r\n    parts.push(\r\n      `Screenshot captured (${result.screenshot.width}x${result.screenshot.height})`\r\n    );\r\n    // The base64 image will be handled separately by the message formatter\r\n  }\r\n\r\n  if (result.activeWindow) {\r\n    parts.push(`**Active Window:**`);\r\n    parts.push(`  - Title: ${result.activeWindow.title}`);\r\n    parts.push(`  - App: ${result.activeWindow.app}`);\r\n    if (result.activeWindow.bounds) {\r\n      const b = result.activeWindow.bounds;\r\n      parts.push(`  - Position: (${b.x}, ${b.y})`);\r\n      parts.push(`  - Size: ${b.width}x${b.height}`);\r\n    }\r\n  }\r\n\r\n  if (result.mousePosition) {\r\n    parts.push(\r\n      `**Mouse Position:** (${result.mousePosition.x}, ${result.mousePosition.y})`\r\n    );\r\n  }\r\n\r\n  if (result.clipboard !== undefined) {\r\n    parts.push(`**Clipboard Content:** ${result.clipboard}`);\r\n  }\r\n\r\n  // UI Automation results\r\n  if (result.uiElement) {\r\n    const el = result.uiElement;\r\n    parts.push(`**UI Element Found:**`);\r\n    parts.push(`  - Name: \"${el.name}\"`);\r\n    parts.push(`  - AutomationId: \"${el.automationId}\"`);\r\n    parts.push(`  - Type: ${el.controlType}`);\r\n    if (el.boundingRectangle) {\r\n      const b = el.boundingRectangle;\r\n      parts.push(`  - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);\r\n      parts.push(`  - Center: (${Math.round(b.x + b.width/2)}, ${Math.round(b.y + b.height/2)})`);\r\n    }\r\n    parts.push(`  - Enabled: ${el.isEnabled}`);\r\n  }\r\n\r\n  if (result.uiElements && result.uiElements.length > 0) {\r\n    parts.push(`**UI Elements Found (${result.uiElements.length}):**`);\r\n    for (const el of result.uiElements.slice(0, 20)) { // Limit to 20\r\n      const bounds = el.boundingRectangle ? \r\n        ` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';\r\n      parts.push(`  - [${el.controlType}] \"${el.name}\"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);\r\n    }\r\n    if (result.uiElements.length > 20) {\r\n      parts.push(`  ... and ${result.uiElements.length - 20} more`);\r\n    }\r\n  }\r\n\r\n  if (result.uiTree) {\r\n    parts.push(`**UI Tree:**`);\r\n    parts.push('```json');\r\n    parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size\r\n    parts.push('```');\r\n  }\r\n\r\n  if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {\r\n    // Generic data fallback\r\n    parts.push(`**Result:**`);\r\n    parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));\r\n  }\r\n\r\n  if (parts.length === 0) {\r\n    parts.push(`Desktop action \"${action}\" completed successfully.`);\r\n  }\r\n\r\n  return parts.join('\\\\n');\r\n}\r\n\r\n/**\r\n * Create desktop automation tools for the agent\r\n * These tools allow AI to control the user's desktop when Ranger Desktop is running\r\n */\r\nexport function createDesktopTools(\r\n  options: CreateDesktopToolsOptions = {}\r\n): DynamicStructuredTool[] {\r\n  const { waitForResult } = options;\r\n  const tools: DynamicStructuredTool[] = [];\r\n\r\n  /**\r\n   * Helper to create tool function that optionally waits for results\r\n   * The toolCallId is extracted from the RunnableConfig passed by LangChain\r\n   */\r\n  const createToolFunction = (action: string) => {\r\n    return async (\r\n      args: Record<string, unknown>,\r\n      config?: { toolCall?: { id?: string } }\r\n    ): Promise<string> => {\r\n      const toolCallId =\r\n        config?.toolCall?.id ??\r\n        `desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;\r\n\r\n      // Create marker for Electron app\r\n      const marker: DesktopToolResponse = {\r\n        requiresDesktopExecution: true,\r\n        action,\r\n        args,\r\n        toolCallId,\r\n      };\r\n\r\n      // If no callback, return marker immediately (Electron handles via SSE interception)\r\n      if (!waitForResult) {\r\n        return JSON.stringify(marker);\r\n      }\r\n\r\n      // With callback: wait for actual results from Electron app\r\n      try {\r\n        const result = await waitForResult(action, args, toolCallId);\r\n        return formatResultForLLM(result, action);\r\n      } catch (error) {\r\n        const errorMessage =\r\n          error instanceof Error ? error.message : String(error);\r\n        return `Desktop action \"${action}\" failed: ${errorMessage}`;\r\n      }\r\n    };\r\n  };\r\n\r\n  // computer_screenshot\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCREENSHOT), {\r\n      name: EDesktopTools.SCREENSHOT,\r\n      description:\r\n        'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',\r\n      schema: ScreenshotSchema,\r\n    })\r\n  );\r\n\r\n  // computer_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLICK), {\r\n      name: EDesktopTools.CLICK,\r\n      description:\r\n        'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',\r\n      schema: ClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_double_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {\r\n      name: EDesktopTools.DOUBLE_CLICK,\r\n      description:\r\n        'Double-click the mouse at the specified screen coordinates.',\r\n      schema: DoubleClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_right_click\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {\r\n      name: EDesktopTools.RIGHT_CLICK,\r\n      description:\r\n        'Right-click the mouse at the specified screen coordinates to open context menus.',\r\n      schema: RightClickSchema,\r\n    })\r\n  );\r\n\r\n  // computer_type\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.TYPE), {\r\n      name: EDesktopTools.TYPE,\r\n      description:\r\n        'Type text using the keyboard. Make sure the target input field is focused first (use click).',\r\n      schema: TypeSchema,\r\n    })\r\n  );\r\n\r\n  // computer_key\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY), {\r\n      name: EDesktopTools.KEY,\r\n      description:\r\n        'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',\r\n      schema: KeySchema,\r\n    })\r\n  );\r\n\r\n  // computer_key_combo\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.KEY_COMBO), {\r\n      name: EDesktopTools.KEY_COMBO,\r\n      description:\r\n        'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',\r\n      schema: KeyComboSchema,\r\n    })\r\n  );\r\n\r\n  // computer_scroll\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.SCROLL), {\r\n      name: EDesktopTools.SCROLL,\r\n      description:\r\n        'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',\r\n      schema: ScrollSchema,\r\n    })\r\n  );\r\n\r\n  // computer_drag\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.DRAG), {\r\n      name: EDesktopTools.DRAG,\r\n      description:\r\n        'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',\r\n      schema: DragSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_active_window\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {\r\n      name: EDesktopTools.GET_ACTIVE_WINDOW,\r\n      description:\r\n        'Get information about the currently active window (title, application name, position, size).',\r\n      schema: GetActiveWindowSchema,\r\n    })\r\n  );\r\n\r\n  // computer_get_mouse_position\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {\r\n      name: EDesktopTools.GET_MOUSE_POSITION,\r\n      description: 'Get the current mouse cursor position on screen.',\r\n      schema: GetMousePositionSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_read\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {\r\n      name: EDesktopTools.CLIPBOARD_READ,\r\n      description: 'Read the current contents of the system clipboard.',\r\n      schema: ClipboardReadSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_write\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {\r\n      name: EDesktopTools.CLIPBOARD_WRITE,\r\n      description: 'Write text to the system clipboard.',\r\n      schema: ClipboardWriteSchema,\r\n    })\r\n  );\r\n\r\n  // clipboard_paste\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {\r\n      name: EDesktopTools.CLIPBOARD_PASTE,\r\n      description:\r\n        'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',\r\n      schema: ClipboardPasteSchema,\r\n    })\r\n  );\r\n\r\n  // computer_wait\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.WAIT), {\r\n      name: EDesktopTools.WAIT,\r\n      description:\r\n        'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',\r\n      schema: WaitSchema,\r\n    })\r\n  );\r\n\r\n  // ============ Native UI Automation Tools (Windows) ============\r\n  // These are FASTER and MORE RELIABLE than screenshot-based automation\r\n  // They find elements by semantic properties (name, automationId, type)\r\n  // instead of relying on pixel coordinates from screenshots\r\n\r\n  // ui_find_element\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {\r\n      name: EDesktopTools.UI_FIND_ELEMENT,\r\n      description:\r\n        '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',\r\n      schema: UIFindElementSchema,\r\n    })\r\n  );\r\n\r\n  // ui_click_element\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {\r\n      name: EDesktopTools.UI_CLICK_ELEMENT,\r\n      description:\r\n        '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: \"OK\"}) or ui_click_element({controlType: \"Button\", name: \"Submit\"}). Windows only.',\r\n      schema: UIClickElementSchema,\r\n    })\r\n  );\r\n\r\n  // ui_get_window_tree\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {\r\n      name: EDesktopTools.UI_GET_WINDOW_TREE,\r\n      description:\r\n        'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',\r\n      schema: UIGetWindowTreeSchema,\r\n    })\r\n  );\r\n\r\n  // ui_find_buttons\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {\r\n      name: EDesktopTools.UI_FIND_BUTTONS,\r\n      description:\r\n        'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',\r\n      schema: UIFindButtonsSchema,\r\n    })\r\n  );\r\n\r\n  // ui_find_inputs\r\n  tools.push(\r\n    tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {\r\n      name: EDesktopTools.UI_FIND_INPUTS,\r\n      description:\r\n        'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',\r\n      schema: UIFindInputsSchema,\r\n    })\r\n  );\r\n\r\n  return tools;\r\n}\r\n\r\n/**\r\n * Get all desktop tool names\r\n */\r\nexport function getDesktopToolNames(): DesktopToolName[] {\r\n  return Object.values(EDesktopTools);\r\n}\r\n\r\n/**\r\n * Check if a tool name is a desktop tool\r\n */\r\nexport function isDesktopTool(name: string): name is DesktopToolName {\r\n  return Object.values(EDesktopTools).includes(name as DesktopToolName);\r\n}\r\n"],"names":[],"mappings":";;;AAGA;;;AAGG;AACU,MAAA,aAAa,GAAG;AAC3B,IAAA,UAAU,EAAE,qBAAqB;AACjC,IAAA,KAAK,EAAE,gBAAgB;AACvB,IAAA,YAAY,EAAE,uBAAuB;AACrC,IAAA,WAAW,EAAE,sBAAsB;AACnC,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,GAAG,EAAE,cAAc;AACnB,IAAA,SAAS,EAAE,oBAAoB;AAC/B,IAAA,MAAM,EAAE,iBAAiB;AACzB,IAAA,IAAI,EAAE,eAAe;AACrB,IAAA,iBAAiB,EAAE,4BAA4B;AAC/C,IAAA,kBAAkB,EAAE,6BAA6B;AACjD,IAAA,cAAc,EAAE,gBAAgB;AAChC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,IAAI,EAAE,eAAe;;AAErB,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,gBAAgB,EAAE,kBAAkB;AACpC,IAAA,kBAAkB,EAAE,oBAAoB;AACxC,IAAA,eAAe,EAAE,iBAAiB;AAClC,IAAA,cAAc,EAAE,gBAAgB;;AA2DlC;;;;;AAKG;AACG,SAAU,oBAAoB,CAAC,GAEpC,EAAA;AACC,IAAA,IAAI,CAAC,GAAG,EAAE,OAAO,EAAE;AACjB,QAAA,OAAO,KAAK;;IAGd,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,kBAAkB,CAAC;IAClD,MAAM,cAAc,GAAG,GAAG,CAAC,OAAO,CAAC,0BAA0B,CAAC;AAE9D,IAAA,OAAO,UAAU,KAAK,MAAM,IAAI,cAAc,KAAK,MAAM;AAC3D;AAEA;AACA,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAErC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAC3B,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IAC/C,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;AAChD,CAAA,CAAC;AAEF,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACjC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;IACtD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,8BAA8B,CAAC;AACvD,CAAA,CAAC;AAEF,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;IACrD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;AACtD,CAAA,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,cAAc,CAAC;AAC1C,CAAA,CAAC;AAEF,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;AACzB,IAAA,GAAG,EAAE;AACF,SAAA,MAAM;SACN,QAAQ,CACP,0KAA0K,CAC3K;AACJ,CAAA,CAAC;AAEF,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;AAC9B,IAAA,IAAI,EAAE;AACH,SAAA,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE;SAChB,QAAQ,CACP,qGAAqG,CACtG;AACJ,CAAA,CAAC;AAEF,MAAM,YAAY,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5B,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;IACnD,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,2BAA2B,CAAC;AACnD,IAAA,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;IAC3E,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iEAAiE,CAAC;AAC/F,CAAA,CAAC;AAEF,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,uBAAuB,CAAC;IACpD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;IAChD,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qBAAqB,CAAC;AACjD,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE1C,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAE3C,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAExC,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;AACxD,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAEzC,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC;IAC1B,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,sBAAsB,CAAC;AAChD,CAAA,CAAC;AAEF;AAEA,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC;AACnC,IAAA,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,2DAA2D,CAAC;AACjG,IAAA,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,8CAA8C,CAAC;AAC5F,IAAA,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mFAAmF,CAAC;AACjI,CAAA,CAAC;AAEF,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC;AACpC,IAAA,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,6BAA6B,CAAC;AACnE,IAAA,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,mCAAmC,CAAC;AACjF,IAAA,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAClF,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,4BAA4B,CAAC;AACjG,CAAA,CAAC;AAEF,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;AACrC,IAAA,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,wCAAwC,CAAC;AACnF,CAAA,CAAC;AAEF,MAAM,mBAAmB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAExC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;AAyBvC;;AAEG;AACH,SAAS,kBAAkB,CACzB,MAA2B,EAC3B,MAAc,EAAA;IAEd,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,KAAK,EAAE;AACnC,QAAA,OAAO,mBAAmB,MAAM,CAAA,UAAA,EAAa,MAAM,CAAC,KAAK,EAAE;;IAG7D,MAAM,KAAK,GAAa,EAAE;AAE1B,IAAA,IAAI,MAAM,CAAC,UAAU,EAAE;AACrB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,UAAU,CAAC,KAAK,CAAA,CAAA,EAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAA,CAAA,CAAG,CAC/E;;;AAIH,IAAA,IAAI,MAAM,CAAC,YAAY,EAAE;AACvB,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,kBAAA,CAAoB,CAAC;QAChC,KAAK,CAAC,IAAI,CAAC,CAAc,WAAA,EAAA,MAAM,CAAC,YAAY,CAAC,KAAK,CAAE,CAAA,CAAC;QACrD,KAAK,CAAC,IAAI,CAAC,CAAY,SAAA,EAAA,MAAM,CAAC,YAAY,CAAC,GAAG,CAAE,CAAA,CAAC;AACjD,QAAA,IAAI,MAAM,CAAC,YAAY,CAAC,MAAM,EAAE;AAC9B,YAAA,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,MAAM;AACpC,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,eAAA,EAAkB,CAAC,CAAC,CAAC,CAAA,EAAA,EAAK,CAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CAAC;AAC5C,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,CAAC,CAAC,KAAK,CAAA,CAAA,EAAI,CAAC,CAAC,MAAM,CAAA,CAAE,CAAC;;;AAIlD,IAAA,IAAI,MAAM,CAAC,aAAa,EAAE;AACxB,QAAA,KAAK,CAAC,IAAI,CACR,CAAwB,qBAAA,EAAA,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,EAAA,EAAK,MAAM,CAAC,aAAa,CAAC,CAAC,CAAA,CAAA,CAAG,CAC7E;;AAGH,IAAA,IAAI,MAAM,CAAC,SAAS,KAAK,SAAS,EAAE;QAClC,KAAK,CAAC,IAAI,CAAC,CAAA,uBAAA,EAA0B,MAAM,CAAC,SAAS,CAAE,CAAA,CAAC;;;AAI1D,IAAA,IAAI,MAAM,CAAC,SAAS,EAAE;AACpB,QAAA,MAAM,EAAE,GAAG,MAAM,CAAC,SAAS;AAC3B,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,qBAAA,CAAuB,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,CAAA,WAAA,EAAc,EAAE,CAAC,IAAI,CAAG,CAAA,CAAA,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,CAAA,mBAAA,EAAsB,EAAE,CAAC,YAAY,CAAG,CAAA,CAAA,CAAC;QACpD,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,EAAE,CAAC,WAAW,CAAE,CAAA,CAAC;AACzC,QAAA,IAAI,EAAE,CAAC,iBAAiB,EAAE;AACxB,YAAA,MAAM,CAAC,GAAG,EAAE,CAAC,iBAAiB;YAC9B,KAAK,CAAC,IAAI,CAAC,CAAA,aAAA,EAAgB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAK,EAAA,EAAA,CAAC,CAAC,KAAK,CAAA,CAAA,EAAI,CAAC,CAAC,MAAM,CAAE,CAAA,CAAC;AACjE,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,aAAA,EAAgB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAC,CAAC,CAAC,CAAA,EAAA,EAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,GAAC,CAAC,CAAC,CAAA,CAAA,CAAG,CAAC;;QAE7F,KAAK,CAAC,IAAI,CAAC,CAAA,aAAA,EAAgB,EAAE,CAAC,SAAS,CAAE,CAAA,CAAC;;AAG5C,IAAA,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE;QACrD,KAAK,CAAC,IAAI,CAAC,CAAwB,qBAAA,EAAA,MAAM,CAAC,UAAU,CAAC,MAAM,CAAM,IAAA,CAAA,CAAC;AAClE,QAAA,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE;AAC/C,YAAA,MAAM,MAAM,GAAG,EAAE,CAAC,iBAAiB;AACjC,gBAAA,CAAA,KAAA,EAAQ,EAAE,CAAC,iBAAiB,CAAC,CAAC,KAAK,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAG,CAAA,CAAA,GAAG,EAAE;AACnE,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,KAAA,EAAQ,EAAE,CAAC,WAAW,CAAA,GAAA,EAAM,EAAE,CAAC,IAAI,CAAA,CAAA,EAAI,EAAE,CAAC,YAAY,GAAG,CAAA,MAAA,EAAS,EAAE,CAAC,YAAY,CAAG,CAAA,CAAA,GAAG,EAAE,CAAA,EAAG,MAAM,CAAA,CAAE,CAAC;;QAElH,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,EAAE,EAAE;AACjC,YAAA,KAAK,CAAC,IAAI,CAAC,CAAA,UAAA,EAAa,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,EAAE,CAAA,KAAA,CAAO,CAAC;;;AAIjE,IAAA,IAAI,MAAM,CAAC,MAAM,EAAE;AACjB,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,YAAA,CAAc,CAAC;AAC1B,QAAA,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;AAClE,QAAA,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;;AAGnB,IAAA,IAAI,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;;AAE5E,QAAA,KAAK,CAAC,IAAI,CAAC,CAAA,WAAA,CAAa,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;;AAGjE,IAAA,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;AACtB,QAAA,KAAK,CAAC,IAAI,CAAC,mBAAmB,MAAM,CAAA,yBAAA,CAA2B,CAAC;;AAGlE,IAAA,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;AAC1B;AAEA;;;AAGG;AACa,SAAA,kBAAkB,CAChC,OAAA,GAAqC,EAAE,EAAA;AAEvC,IAAA,MAAM,EAAE,aAAa,EAAE,GAAG,OAAO;IACjC,MAAM,KAAK,GAA4B,EAAE;AAEzC;;;AAGG;AACH,IAAA,MAAM,kBAAkB,GAAG,CAAC,MAAc,KAAI;AAC5C,QAAA,OAAO,OACL,IAA6B,EAC7B,MAAuC,KACpB;AACnB,YAAA,MAAM,UAAU,GACd,MAAM,EAAE,QAAQ,EAAE,EAAE;gBACpB,CAAW,QAAA,EAAA,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;;AAGhE,YAAA,MAAM,MAAM,GAAwB;AAClC,gBAAA,wBAAwB,EAAE,IAAI;gBAC9B,MAAM;gBACN,IAAI;gBACJ,UAAU;aACX;;YAGD,IAAI,CAAC,aAAa,EAAE;AAClB,gBAAA,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC;;;AAI/B,YAAA,IAAI;gBACF,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC;AAC5D,gBAAA,OAAO,kBAAkB,CAAC,MAAM,EAAE,MAAM,CAAC;;YACzC,OAAO,KAAK,EAAE;AACd,gBAAA,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,GAAG,KAAK,CAAC,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC;AACxD,gBAAA,OAAO,CAAmB,gBAAA,EAAA,MAAM,CAAa,UAAA,EAAA,YAAY,EAAE;;AAE/D,SAAC;AACH,KAAC;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE;QACjD,IAAI,EAAE,aAAa,CAAC,UAAU;AAC9B,QAAA,WAAW,EACT,qGAAqG;AACvG,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE;QAC5C,IAAI,EAAE,aAAa,CAAC,KAAK;AACzB,QAAA,WAAW,EACT,4GAA4G;AAC9G,QAAA,MAAM,EAAE,WAAW;AACpB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,YAAY,CAAC,EAAE;QACnD,IAAI,EAAE,aAAa,CAAC,YAAY;AAChC,QAAA,WAAW,EACT,6DAA6D;AAC/D,QAAA,MAAM,EAAE,iBAAiB;AAC1B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE;QAClD,IAAI,EAAE,aAAa,CAAC,WAAW;AAC/B,QAAA,WAAW,EACT,kFAAkF;AACpF,QAAA,MAAM,EAAE,gBAAgB;AACzB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,GAAG,CAAC,EAAE;QAC1C,IAAI,EAAE,aAAa,CAAC,GAAG;AACvB,QAAA,WAAW,EACT,2FAA2F;AAC7F,QAAA,MAAM,EAAE,SAAS;AAClB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE;QAChD,IAAI,EAAE,aAAa,CAAC,SAAS;AAC7B,QAAA,WAAW,EACT,6FAA6F;AAC/F,QAAA,MAAM,EAAE,cAAc;AACvB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE;QAC7C,IAAI,EAAE,aAAa,CAAC,MAAM;AAC1B,QAAA,WAAW,EACT,wGAAwG;AAC1G,QAAA,MAAM,EAAE,YAAY;AACrB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,yFAAyF;AAC3F,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,iBAAiB,CAAC,EAAE;QACxD,IAAI,EAAE,aAAa,CAAC,iBAAiB;AACrC,QAAA,WAAW,EACT,8FAA8F;AAChG,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;QACzD,IAAI,EAAE,aAAa,CAAC,kBAAkB;AACtC,QAAA,WAAW,EAAE,kDAAkD;AAC/D,QAAA,MAAM,EAAE,sBAAsB;AAC/B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE;QACrD,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EAAE,oDAAoD;AACjE,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EAAE,qCAAqC;AAClD,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,oGAAoG;AACtG,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;QAC3C,IAAI,EAAE,aAAa,CAAC,IAAI;AACxB,QAAA,WAAW,EACT,+FAA+F;AACjG,QAAA,MAAM,EAAE,UAAU;AACnB,KAAA,CAAC,CACH;;;;;;IAQD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,oLAAoL;AACtL,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,gBAAgB,CAAC,EAAE;QACvD,IAAI,EAAE,aAAa,CAAC,gBAAgB;AACpC,QAAA,WAAW,EACT,mOAAmO;AACrO,QAAA,MAAM,EAAE,oBAAoB;AAC7B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,kBAAkB,CAAC,EAAE;QACzD,IAAI,EAAE,aAAa,CAAC,kBAAkB;AACtC,QAAA,WAAW,EACT,uLAAuL;AACzL,QAAA,MAAM,EAAE,qBAAqB;AAC9B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,eAAe,CAAC,EAAE;QACtD,IAAI,EAAE,aAAa,CAAC,eAAe;AACnC,QAAA,WAAW,EACT,iJAAiJ;AACnJ,QAAA,MAAM,EAAE,mBAAmB;AAC5B,KAAA,CAAC,CACH;;IAGD,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,kBAAkB,CAAC,aAAa,CAAC,cAAc,CAAC,EAAE;QACrD,IAAI,EAAE,aAAa,CAAC,cAAc;AAClC,QAAA,WAAW,EACT,2IAA2I;AAC7I,QAAA,MAAM,EAAE,kBAAkB;AAC3B,KAAA,CAAC,CACH;AAED,IAAA,OAAO,KAAK;AACd;AAEA;;AAEG;SACa,mBAAmB,GAAA;AACjC,IAAA,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC;AACrC;AAEA;;AAEG;AACG,SAAU,aAAa,CAAC,IAAY,EAAA;IACxC,OAAO,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,QAAQ,CAAC,IAAuB,CAAC;AACvE;;;;"}

package/dist/types/tools/DesktopTools.d.ts

@@ -1,134 +0,0 @@
-import { DynamicStructuredTool } from '@langchain/core/tools';
-/**
- * Desktop tool names - keep in sync with Ranger Desktop Electron app
- * These tools execute locally in the Electron app, NOT on the server
- */
-export declare const EDesktopTools: {
-    readonly SCREENSHOT: "computer_screenshot";
-    readonly CLICK: "computer_click";
-    readonly DOUBLE_CLICK: "computer_double_click";
-    readonly RIGHT_CLICK: "computer_right_click";
-    readonly TYPE: "computer_type";
-    readonly KEY: "computer_key";
-    readonly KEY_COMBO: "computer_key_combo";
-    readonly SCROLL: "computer_scroll";
-    readonly DRAG: "computer_drag";
-    readonly GET_ACTIVE_WINDOW: "computer_get_active_window";
-    readonly GET_MOUSE_POSITION: "computer_get_mouse_position";
-    readonly CLIPBOARD_READ: "clipboard_read";
-    readonly CLIPBOARD_WRITE: "clipboard_write";
-    readonly CLIPBOARD_PASTE: "clipboard_paste";
-    readonly WAIT: "computer_wait";
-    readonly UI_FIND_ELEMENT: "ui_find_element";
-    readonly UI_CLICK_ELEMENT: "ui_click_element";
-    readonly UI_GET_WINDOW_TREE: "ui_get_window_tree";
-    readonly UI_FIND_BUTTONS: "ui_find_buttons";
-    readonly UI_FIND_INPUTS: "ui_find_inputs";
-};
-export type DesktopToolName = (typeof EDesktopTools)[keyof typeof EDesktopTools];
-/**
- * Callback function type for waiting on desktop action results
- * This allows the server (Ranger) to provide a callback that waits for the Electron app
- * to POST results back to the server before returning to the LLM.
- *
- * @param action - The desktop action (click, type, screenshot, etc.)
- * @param args - Arguments for the action
- * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)
- * @returns Promise that resolves with the actual desktop result
- */
-export type DesktopToolCallback = (action: string, args: Record<string, unknown>, toolCallId: string) => Promise<DesktopActionResult>;
-/**
- * Result returned from desktop action execution
- */
-export interface DesktopActionResult {
-    success: boolean;
-    error?: string;
-    screenshot?: {
-        base64: string;
-        width: number;
-        height: number;
-    };
-    activeWindow?: {
-        title: string;
-        app: string;
-        bounds?: {
-            x: number;
-            y: number;
-            width: number;
-            height: number;
-        };
-    };
-    mousePosition?: {
-        x: number;
-        y: number;
-    };
-    clipboard?: string;
-    uiElement?: {
-        name: string;
-        automationId: string;
-        controlType: string;
-        boundingRectangle?: {
-            x: number;
-            y: number;
-            width: number;
-            height: number;
-        };
-        isEnabled: boolean;
-    };
-    uiElements?: Array<{
-        name: string;
-        automationId: string;
-        controlType: string;
-        boundingRectangle?: {
-            x: number;
-            y: number;
-            width: number;
-            height: number;
-        };
-    }>;
-    uiTree?: unknown;
-    data?: unknown;
-}
-/**
- * Check if desktop capability is available based on request headers or context
- * The Ranger Desktop Electron app sets these headers when connected:
- * - X-Ranger-Desktop: true
- * - X-Ranger-Desktop-Capable: true
- */
-export declare function hasDesktopCapability(req?: {
-    headers?: Record<string, string | string[] | undefined>;
-}): boolean;
-/**
- * Desktop tool response interface
- * This is what the Electron app returns after executing the action
- */
-export interface DesktopToolResponse {
-    requiresDesktopExecution: true;
-    action: string;
-    args: Record<string, unknown>;
-    toolCallId?: string;
-}
-/**
- * Options for creating desktop tools
- */
-export interface CreateDesktopToolsOptions {
-    /**
-     * Optional callback that waits for desktop action results.
-     * When provided, tools will await this callback to get actual results from the Electron app.
-     * When not provided, tools return markers immediately (for non-server contexts).
-     */
-    waitForResult?: DesktopToolCallback;
-}
-/**
- * Create desktop automation tools for the agent
- * These tools allow AI to control the user's desktop when Ranger Desktop is running
- */
-export declare function createDesktopTools(options?: CreateDesktopToolsOptions): DynamicStructuredTool[];
-/**
- * Get all desktop tool names
- */
-export declare function getDesktopToolNames(): DesktopToolName[];
-/**
- * Check if a tool name is a desktop tool
- */
-export declare function isDesktopTool(name: string): name is DesktopToolName;