illuma-agents 1.0.45 → 1.0.47

package/src/tools/DesktopTools.ts

@@ -1,574 +0,0 @@
-import { z } from 'zod';
-import { tool, DynamicStructuredTool } from '@langchain/core/tools';
-
-/**
- * Desktop tool names - keep in sync with Ranger Desktop Electron app
- * These tools execute locally in the Electron app, NOT on the server
- */
-export const EDesktopTools = {
-  SCREENSHOT: 'computer_screenshot',
-  CLICK: 'computer_click',
-  DOUBLE_CLICK: 'computer_double_click',
-  RIGHT_CLICK: 'computer_right_click',
-  TYPE: 'computer_type',
-  KEY: 'computer_key',
-  KEY_COMBO: 'computer_key_combo',
-  SCROLL: 'computer_scroll',
-  DRAG: 'computer_drag',
-  GET_ACTIVE_WINDOW: 'computer_get_active_window',
-  GET_MOUSE_POSITION: 'computer_get_mouse_position',
-  CLIPBOARD_READ: 'clipboard_read',
-  CLIPBOARD_WRITE: 'clipboard_write',
-  CLIPBOARD_PASTE: 'clipboard_paste',
-  WAIT: 'computer_wait',
-  // Native UI Automation tools (Windows) - faster and more reliable than screenshot-based
-  UI_FIND_ELEMENT: 'ui_find_element',
-  UI_CLICK_ELEMENT: 'ui_click_element',
-  UI_GET_WINDOW_TREE: 'ui_get_window_tree',
-  UI_FIND_BUTTONS: 'ui_find_buttons',
-  UI_FIND_INPUTS: 'ui_find_inputs',
-} as const;
-
-export type DesktopToolName =
-  (typeof EDesktopTools)[keyof typeof EDesktopTools];
-
-/**
- * Callback function type for waiting on desktop action results
- * This allows the server (Ranger) to provide a callback that waits for the Electron app
- * to POST results back to the server before returning to the LLM.
- *
- * @param action - The desktop action (click, type, screenshot, etc.)
- * @param args - Arguments for the action
- * @param toolCallId - Unique ID for this tool call (from config.toolCall.id)
- * @returns Promise that resolves with the actual desktop result
- */
-export type DesktopToolCallback = (
-  action: string,
-  args: Record<string, unknown>,
-  toolCallId: string
-) => Promise<DesktopActionResult>;
-
-/**
- * Result returned from desktop action execution
- */
-export interface DesktopActionResult {
-  success: boolean;
-  error?: string;
-  screenshot?: {
-    base64: string;
-    width: number;
-    height: number;
-  };
-  activeWindow?: {
-    title: string;
-    app: string;
-    bounds?: { x: number; y: number; width: number; height: number };
-  };
-  mousePosition?: { x: number; y: number };
-  clipboard?: string;
-  // UI Automation results
-  uiElement?: {
-    name: string;
-    automationId: string;
-    controlType: string;
-    boundingRectangle?: { x: number; y: number; width: number; height: number };
-    isEnabled: boolean;
-  };
-  uiElements?: Array<{
-    name: string;
-    automationId: string;
-    controlType: string;
-    boundingRectangle?: { x: number; y: number; width: number; height: number };
-  }>;
-  uiTree?: unknown;
-  // Generic data for extended results
-  data?: unknown;
-}
-
-/**
- * Check if desktop capability is available based on request headers or context
- * The Ranger Desktop Electron app sets these headers when connected:
- * - X-Ranger-Desktop: true
- * - X-Ranger-Desktop-Capable: true
- */
-export function hasDesktopCapability(req?: {
-  headers?: Record<string, string | string[] | undefined>;
-}): boolean {
-  if (!req?.headers) {
-    return false;
-  }
-
-  const desktopApp = req.headers['x-ranger-desktop'];
-  const desktopCapable = req.headers['x-ranger-desktop-capable'];
-
-  return desktopApp === 'true' || desktopCapable === 'true';
-}
-
-// Tool schemas
-const ScreenshotSchema = z.object({});
-
-const ClickSchema = z.object({
-  x: z.number().describe('X coordinate to click'),
-  y: z.number().describe('Y coordinate to click'),
-});
-
-const DoubleClickSchema = z.object({
-  x: z.number().describe('X coordinate to double-click'),
-  y: z.number().describe('Y coordinate to double-click'),
-});
-
-const RightClickSchema = z.object({
-  x: z.number().describe('X coordinate to right-click'),
-  y: z.number().describe('Y coordinate to right-click'),
-});
-
-const TypeSchema = z.object({
-  text: z.string().describe('Text to type'),
-});
-
-const KeySchema = z.object({
-  key: z
-    .string()
-    .describe(
-      'Key to press (e.g., "Enter", "Tab", "Escape", "Backspace", "Delete", "ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight", "Home", "End", "PageUp", "PageDown", "F1"-"F12")'
-    ),
-});
-
-const KeyComboSchema = z.object({
-  keys: z
-    .array(z.string())
-    .describe(
-      'Array of keys to press together (e.g., ["Control", "c"] for copy, ["Alt", "Tab"] for window switch)'
-    ),
-});
-
-const ScrollSchema = z.object({
-  x: z.number().describe('X coordinate to scroll at'),
-  y: z.number().describe('Y coordinate to scroll at'),
-  deltaX: z.number().optional().describe('Horizontal scroll amount (pixels)'),
-  deltaY: z.number().describe('Vertical scroll amount (pixels, negative = up, positive = down)'),
-});
-
-const DragSchema = z.object({
-  startX: z.number().describe('Starting X coordinate'),
-  startY: z.number().describe('Starting Y coordinate'),
-  endX: z.number().describe('Ending X coordinate'),
-  endY: z.number().describe('Ending Y coordinate'),
-});
-
-const GetActiveWindowSchema = z.object({});
-
-const GetMousePositionSchema = z.object({});
-
-const ClipboardReadSchema = z.object({});
-
-const ClipboardWriteSchema = z.object({
-  text: z.string().describe('Text to write to clipboard'),
-});
-
-const ClipboardPasteSchema = z.object({});
-
-const WaitSchema = z.object({
-  ms: z.number().describe('Milliseconds to wait'),
-});
-
-// ============ Native UI Automation Schemas (Windows) ============
-
-const UIFindElementSchema = z.object({
-  name: z.string().optional().describe('Element name/label to find (e.g., "Submit", "OK", "File")'),
-  automationId: z.string().optional().describe('Automation ID of element (unique identifier)'),
-  controlType: z.string().optional().describe('Type of control: Button, Edit, Text, ComboBox, List, Menu, MenuItem, Window, etc.'),
-});
-
-const UIClickElementSchema = z.object({
-  name: z.string().optional().describe('Element name/label to click'),
-  automationId: z.string().optional().describe('Automation ID of element to click'),
-  controlType: z.string().optional().describe('Type of control: Button, Edit, etc.'),
-  clickType: z.enum(['left', 'right', 'double']).optional().describe('Click type (default: left)'),
-});
-
-const UIGetWindowTreeSchema = z.object({
-  maxDepth: z.number().optional().describe('Maximum depth to traverse (default: 3)'),
-});
-
-const UIFindButtonsSchema = z.object({});
-
-const UIFindInputsSchema = z.object({});
-
-/**
- * Desktop tool response interface
- * This is what the Electron app returns after executing the action
- */
-export interface DesktopToolResponse {
-  requiresDesktopExecution: true;
-  action: string;
-  args: Record<string, unknown>;
-  toolCallId?: string;
-}
-
-/**
- * Options for creating desktop tools
- */
-export interface CreateDesktopToolsOptions {
-  /**
-   * Optional callback that waits for desktop action results.
-   * When provided, tools will await this callback to get actual results from the Electron app.
-   * When not provided, tools return markers immediately (for non-server contexts).
-   */
-  waitForResult?: DesktopToolCallback;
-}
-
-/**
- * Format desktop action result for LLM consumption
- */
-function formatResultForLLM(
-  result: DesktopActionResult,
-  action: string
-): string {
-  if (!result.success && result.error) {
-    return `Desktop action "${action}" failed: ${result.error}`;
-  }
-
-  const parts: string[] = [];
-
-  if (result.screenshot) {
-    parts.push(
-      `Screenshot captured (${result.screenshot.width}x${result.screenshot.height})`
-    );
-    // The base64 image will be handled separately by the message formatter
-  }
-
-  if (result.activeWindow) {
-    parts.push(`**Active Window:**`);
-    parts.push(`  - Title: ${result.activeWindow.title}`);
-    parts.push(`  - App: ${result.activeWindow.app}`);
-    if (result.activeWindow.bounds) {
-      const b = result.activeWindow.bounds;
-      parts.push(`  - Position: (${b.x}, ${b.y})`);
-      parts.push(`  - Size: ${b.width}x${b.height}`);
-    }
-  }
-
-  if (result.mousePosition) {
-    parts.push(
-      `**Mouse Position:** (${result.mousePosition.x}, ${result.mousePosition.y})`
-    );
-  }
-
-  if (result.clipboard !== undefined) {
-    parts.push(`**Clipboard Content:** ${result.clipboard}`);
-  }
-
-  // UI Automation results
-  if (result.uiElement) {
-    const el = result.uiElement;
-    parts.push(`**UI Element Found:**`);
-    parts.push(`  - Name: "${el.name}"`);
-    parts.push(`  - AutomationId: "${el.automationId}"`);
-    parts.push(`  - Type: ${el.controlType}`);
-    if (el.boundingRectangle) {
-      const b = el.boundingRectangle;
-      parts.push(`  - Bounds: (${b.x}, ${b.y}) ${b.width}x${b.height}`);
-      parts.push(`  - Center: (${Math.round(b.x + b.width/2)}, ${Math.round(b.y + b.height/2)})`);
-    }
-    parts.push(`  - Enabled: ${el.isEnabled}`);
-  }
-
-  if (result.uiElements && result.uiElements.length > 0) {
-    parts.push(`**UI Elements Found (${result.uiElements.length}):**`);
-    for (const el of result.uiElements.slice(0, 20)) { // Limit to 20
-      const bounds = el.boundingRectangle ? 
-        ` at (${el.boundingRectangle.x}, ${el.boundingRectangle.y})` : '';
-      parts.push(`  - [${el.controlType}] "${el.name}"${el.automationId ? ` (id: ${el.automationId})` : ''}${bounds}`);
-    }
-    if (result.uiElements.length > 20) {
-      parts.push(`  ... and ${result.uiElements.length - 20} more`);
-    }
-  }
-
-  if (result.uiTree) {
-    parts.push(`**UI Tree:**`);
-    parts.push('```json');
-    parts.push(JSON.stringify(result.uiTree, null, 2).slice(0, 3000)); // Limit size
-    parts.push('```');
-  }
-
-  if (result.data && !result.uiElement && !result.uiElements && !result.uiTree) {
-    // Generic data fallback
-    parts.push(`**Result:**`);
-    parts.push(JSON.stringify(result.data, null, 2).slice(0, 2000));
-  }
-
-  if (parts.length === 0) {
-    parts.push(`Desktop action "${action}" completed successfully.`);
-  }
-
-  return parts.join('\\n');
-}
-
-/**
- * Create desktop automation tools for the agent
- * These tools allow AI to control the user's desktop when Ranger Desktop is running
- */
-export function createDesktopTools(
-  options: CreateDesktopToolsOptions = {}
-): DynamicStructuredTool[] {
-  const { waitForResult } = options;
-  const tools: DynamicStructuredTool[] = [];
-
-  /**
-   * Helper to create tool function that optionally waits for results
-   * The toolCallId is extracted from the RunnableConfig passed by LangChain
-   */
-  const createToolFunction = (action: string) => {
-    return async (
-      args: Record<string, unknown>,
-      config?: { toolCall?: { id?: string } }
-    ): Promise<string> => {
-      const toolCallId =
-        config?.toolCall?.id ??
-        `desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;
-
-      // Create marker for Electron app
-      const marker: DesktopToolResponse = {
-        requiresDesktopExecution: true,
-        action,
-        args,
-        toolCallId,
-      };
-
-      // If no callback, return marker immediately (Electron handles via SSE interception)
-      if (!waitForResult) {
-        return JSON.stringify(marker);
-      }
-
-      // With callback: wait for actual results from Electron app
-      try {
-        const result = await waitForResult(action, args, toolCallId);
-        return formatResultForLLM(result, action);
-      } catch (error) {
-        const errorMessage =
-          error instanceof Error ? error.message : String(error);
-        return `Desktop action "${action}" failed: ${errorMessage}`;
-      }
-    };
-  };
-
-  // computer_screenshot
-  tools.push(
-    tool(createToolFunction(EDesktopTools.SCREENSHOT), {
-      name: EDesktopTools.SCREENSHOT,
-      description:
-        'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',
-      schema: ScreenshotSchema,
-    })
-  );
-
-  // computer_click
-  tools.push(
-    tool(createToolFunction(EDesktopTools.CLICK), {
-      name: EDesktopTools.CLICK,
-      description:
-        'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',
-      schema: ClickSchema,
-    })
-  );
-
-  // computer_double_click
-  tools.push(
-    tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {
-      name: EDesktopTools.DOUBLE_CLICK,
-      description:
-        'Double-click the mouse at the specified screen coordinates.',
-      schema: DoubleClickSchema,
-    })
-  );
-
-  // computer_right_click
-  tools.push(
-    tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {
-      name: EDesktopTools.RIGHT_CLICK,
-      description:
-        'Right-click the mouse at the specified screen coordinates to open context menus.',
-      schema: RightClickSchema,
-    })
-  );
-
-  // computer_type
-  tools.push(
-    tool(createToolFunction(EDesktopTools.TYPE), {
-      name: EDesktopTools.TYPE,
-      description:
-        'Type text using the keyboard. Make sure the target input field is focused first (use click).',
-      schema: TypeSchema,
-    })
-  );
-
-  // computer_key
-  tools.push(
-    tool(createToolFunction(EDesktopTools.KEY), {
-      name: EDesktopTools.KEY,
-      description:
-        'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',
-      schema: KeySchema,
-    })
-  );
-
-  // computer_key_combo
-  tools.push(
-    tool(createToolFunction(EDesktopTools.KEY_COMBO), {
-      name: EDesktopTools.KEY_COMBO,
-      description:
-        'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',
-      schema: KeyComboSchema,
-    })
-  );
-
-  // computer_scroll
-  tools.push(
-    tool(createToolFunction(EDesktopTools.SCROLL), {
-      name: EDesktopTools.SCROLL,
-      description:
-        'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',
-      schema: ScrollSchema,
-    })
-  );
-
-  // computer_drag
-  tools.push(
-    tool(createToolFunction(EDesktopTools.DRAG), {
-      name: EDesktopTools.DRAG,
-      description:
-        'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',
-      schema: DragSchema,
-    })
-  );
-
-  // computer_get_active_window
-  tools.push(
-    tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {
-      name: EDesktopTools.GET_ACTIVE_WINDOW,
-      description:
-        'Get information about the currently active window (title, application name, position, size).',
-      schema: GetActiveWindowSchema,
-    })
-  );
-
-  // computer_get_mouse_position
-  tools.push(
-    tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {
-      name: EDesktopTools.GET_MOUSE_POSITION,
-      description: 'Get the current mouse cursor position on screen.',
-      schema: GetMousePositionSchema,
-    })
-  );
-
-  // clipboard_read
-  tools.push(
-    tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {
-      name: EDesktopTools.CLIPBOARD_READ,
-      description: 'Read the current contents of the system clipboard.',
-      schema: ClipboardReadSchema,
-    })
-  );
-
-  // clipboard_write
-  tools.push(
-    tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {
-      name: EDesktopTools.CLIPBOARD_WRITE,
-      description: 'Write text to the system clipboard.',
-      schema: ClipboardWriteSchema,
-    })
-  );
-
-  // clipboard_paste
-  tools.push(
-    tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {
-      name: EDesktopTools.CLIPBOARD_PASTE,
-      description:
-        'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',
-      schema: ClipboardPasteSchema,
-    })
-  );
-
-  // computer_wait
-  tools.push(
-    tool(createToolFunction(EDesktopTools.WAIT), {
-      name: EDesktopTools.WAIT,
-      description:
-        'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',
-      schema: WaitSchema,
-    })
-  );
-
-  // ============ Native UI Automation Tools (Windows) ============
-  // These are FASTER and MORE RELIABLE than screenshot-based automation
-  // They find elements by semantic properties (name, automationId, type)
-  // instead of relying on pixel coordinates from screenshots
-
-  // ui_find_element
-  tools.push(
-    tool(createToolFunction(EDesktopTools.UI_FIND_ELEMENT), {
-      name: EDesktopTools.UI_FIND_ELEMENT,
-      description:
-        '🚀 PREFERRED: Find a UI element by semantic properties (name, automationId, controlType). MUCH FASTER than screenshot analysis. Returns element bounds for clicking. Windows only.',
-      schema: UIFindElementSchema,
-    })
-  );
-
-  // ui_click_element
-  tools.push(
-    tool(createToolFunction(EDesktopTools.UI_CLICK_ELEMENT), {
-      name: EDesktopTools.UI_CLICK_ELEMENT,
-      description:
-        '🚀 PREFERRED: Find and click a UI element by name/automationId. More reliable than coordinate-based clicking. Example: ui_click_element({name: "OK"}) or ui_click_element({controlType: "Button", name: "Submit"}). Windows only.',
-      schema: UIClickElementSchema,
-    })
-  );
-
-  // ui_get_window_tree
-  tools.push(
-    tool(createToolFunction(EDesktopTools.UI_GET_WINDOW_TREE), {
-      name: EDesktopTools.UI_GET_WINDOW_TREE,
-      description:
-        'Get the UI element tree of the active window. Shows all buttons, inputs, menus, etc. with their names and automationIds. Use this to discover elements before clicking. Windows only.',
-      schema: UIGetWindowTreeSchema,
-    })
-  );
-
-  // ui_find_buttons
-  tools.push(
-    tool(createToolFunction(EDesktopTools.UI_FIND_BUTTONS), {
-      name: EDesktopTools.UI_FIND_BUTTONS,
-      description:
-        'Find all clickable buttons in the active window. Returns list with names and positions. Useful for discovering available actions. Windows only.',
-      schema: UIFindButtonsSchema,
-    })
-  );
-
-  // ui_find_inputs
-  tools.push(
-    tool(createToolFunction(EDesktopTools.UI_FIND_INPUTS), {
-      name: EDesktopTools.UI_FIND_INPUTS,
-      description:
-        'Find all text input fields in the active window. Returns list with names and positions. Useful for discovering form fields. Windows only.',
-      schema: UIFindInputsSchema,
-    })
-  );
-
-  return tools;
-}
-
-/**
- * Get all desktop tool names
- */
-export function getDesktopToolNames(): DesktopToolName[] {
-  return Object.values(EDesktopTools);
-}
-
-/**
- * Check if a tool name is a desktop tool
- */
-export function isDesktopTool(name: string): name is DesktopToolName {
-  return Object.values(EDesktopTools).includes(name as DesktopToolName);
-}