illuma-agents 1.0.43 → 1.0.44

package/src/tools/DesktopTools.ts

@@ -1,11 +1,6 @@
 import { z } from 'zod';
 import { tool, DynamicStructuredTool } from '@langchain/core/tools';
 
-/**
- * Type for tool configuration passed by LangChain runtime
- */
-type ToolCallConfig = { toolCall?: { id?: string } };
-
 /**
  * Desktop tool names - keep in sync with Ranger Desktop Electron app
  * These tools execute locally in the Electron app, NOT on the server
@@ -225,39 +220,6 @@ function formatResultForLLM(
   return parts.join('\n');
 }
 
-/**
- * Create a tool result (either wait for callback or return marker)
- */
-async function createToolResult(
-  action: string,
-  args: Record<string, unknown>,
-  config: ToolCallConfig | undefined,
-  waitForResult?: DesktopToolCallback
-): Promise<string> {
-  const toolCallId = config?.toolCall?.id || `desktop-${Date.now()}`;
-
-  if (waitForResult) {
-    // Server context: wait for actual result from Electron app
-    try {
-      const result = await waitForResult(action, args, toolCallId);
-      return formatResultForLLM(result, action);
-    } catch (error) {
-      const errorMessage =
-        error instanceof Error ? error.message : String(error);
-      return `Desktop action "${action}" failed: ${errorMessage}`;
-    }
-  }
-
-  // Non-server context: return marker for later processing
-  const response: DesktopToolResponse = {
-    requiresDesktopExecution: true,
-    action,
-    args,
-    toolCallId,
-  };
-  return JSON.stringify(response);
-}
-
 /**
  * Create desktop automation tools for the agent
  * These tools allow AI to control the user's desktop when Ranger Desktop is running
@@ -266,275 +228,194 @@ export function createDesktopTools(
   options: CreateDesktopToolsOptions = {}
 ): DynamicStructuredTool[] {
   const { waitForResult } = options;
+  const tools: DynamicStructuredTool[] = [];
 
-  return [
-    // computer_screenshot
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.SCREENSHOT,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.SCREENSHOT,
-        description:
-          'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',
-        schema: ScreenshotSchema,
-      }
-    ),
-
-    // computer_click
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.CLICK,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.CLICK,
-        description:
-          'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',
-        schema: ClickSchema,
-      }
-    ),
-
-    // computer_double_click
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.DOUBLE_CLICK,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.DOUBLE_CLICK,
-        description:
-          'Double-click the mouse at the specified screen coordinates.',
-        schema: DoubleClickSchema,
-      }
-    ),
-
-    // computer_right_click
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.RIGHT_CLICK,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.RIGHT_CLICK,
-        description:
-          'Right-click the mouse at the specified screen coordinates to open context menus.',
-        schema: RightClickSchema,
-      }
-    ),
-
-    // computer_type
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.TYPE,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.TYPE,
-        description:
-          'Type text using the keyboard. Make sure the target input field is focused first (use click).',
-        schema: TypeSchema,
-      }
-    ),
-
-    // computer_key
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.KEY,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.KEY,
-        description:
-          'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',
-        schema: KeySchema,
-      }
-    ),
-
-    // computer_key_combo
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.KEY_COMBO,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.KEY_COMBO,
-        description:
-          'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',
-        schema: KeyComboSchema,
-      }
-    ),
-
-    // computer_scroll
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.SCROLL,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.SCROLL,
-        description:
-          'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',
-        schema: ScrollSchema,
-      }
-    ),
-
-    // computer_drag
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.DRAG,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.DRAG,
-        description:
-          'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',
-        schema: DragSchema,
-      }
-    ),
-
-    // computer_get_active_window
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.GET_ACTIVE_WINDOW,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.GET_ACTIVE_WINDOW,
-        description:
-          'Get information about the currently active window (title, application name, position, size).',
-        schema: GetActiveWindowSchema,
-      }
-    ),
-
-    // computer_get_mouse_position
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.GET_MOUSE_POSITION,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.GET_MOUSE_POSITION,
-        description: 'Get the current mouse cursor position on screen.',
-        schema: GetMousePositionSchema,
-      }
-    ),
-
-    // clipboard_read
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.CLIPBOARD_READ,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.CLIPBOARD_READ,
-        description: 'Read the current contents of the system clipboard.',
-        schema: ClipboardReadSchema,
-      }
-    ),
-
-    // clipboard_write
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.CLIPBOARD_WRITE,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.CLIPBOARD_WRITE,
-        description: 'Write text to the system clipboard.',
-        schema: ClipboardWriteSchema,
+  /**
+   * Helper to create tool function that optionally waits for results
+   * The toolCallId is extracted from the RunnableConfig passed by LangChain
+   */
+  const createToolFunction = (action: string) => {
+    return async (
+      args: Record<string, unknown>,
+      config?: { toolCall?: { id?: string } }
+    ): Promise<string> => {
+      const toolCallId =
+        config?.toolCall?.id ??
+        `desktop_${Date.now()}_${Math.random().toString(36).slice(2)}`;
+
+      // Create marker for Electron app
+      const marker: DesktopToolResponse = {
+        requiresDesktopExecution: true,
+        action,
+        args,
+        toolCallId,
+      };
+
+      // If no callback, return marker immediately (Electron handles via SSE interception)
+      if (!waitForResult) {
+        return JSON.stringify(marker);
       }
-    ),
 
-    // clipboard_paste
-    tool(
-      async (_args, config) => {
-        return createToolResult(
-          EDesktopTools.CLIPBOARD_PASTE,
-          {},
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.CLIPBOARD_PASTE,
-        description:
-          'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',
-        schema: ClipboardPasteSchema,
+      // With callback: wait for actual results from Electron app
+      try {
+        const result = await waitForResult(action, args, toolCallId);
+        return formatResultForLLM(result, action);
+      } catch (error) {
+        const errorMessage =
+          error instanceof Error ? error.message : String(error);
+        return `Desktop action "${action}" failed: ${errorMessage}`;
       }
-    ),
+    };
+  };
 
-    // computer_wait
-    tool(
-      async (args, config) => {
-        return createToolResult(
-          EDesktopTools.WAIT,
-          args,
-          config as ToolCallConfig,
-          waitForResult
-        );
-      },
-      {
-        name: EDesktopTools.WAIT,
-        description:
-          'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',
-        schema: WaitSchema,
-      }
-    ),
-  ];
+  // computer_screenshot
+  tools.push(
+    tool(createToolFunction(EDesktopTools.SCREENSHOT), {
+      name: EDesktopTools.SCREENSHOT,
+      description:
+        'Take a screenshot of the entire screen. Use this to see what is currently displayed on the desktop.',
+      schema: ScreenshotSchema,
+    })
+  );
+
+  // computer_click
+  tools.push(
+    tool(createToolFunction(EDesktopTools.CLICK), {
+      name: EDesktopTools.CLICK,
+      description:
+        'Click the mouse at the specified screen coordinates. Use screenshot first to identify the target location.',
+      schema: ClickSchema,
+    })
+  );
+
+  // computer_double_click
+  tools.push(
+    tool(createToolFunction(EDesktopTools.DOUBLE_CLICK), {
+      name: EDesktopTools.DOUBLE_CLICK,
+      description:
+        'Double-click the mouse at the specified screen coordinates.',
+      schema: DoubleClickSchema,
+    })
+  );
+
+  // computer_right_click
+  tools.push(
+    tool(createToolFunction(EDesktopTools.RIGHT_CLICK), {
+      name: EDesktopTools.RIGHT_CLICK,
+      description:
+        'Right-click the mouse at the specified screen coordinates to open context menus.',
+      schema: RightClickSchema,
+    })
+  );
+
+  // computer_type
+  tools.push(
+    tool(createToolFunction(EDesktopTools.TYPE), {
+      name: EDesktopTools.TYPE,
+      description:
+        'Type text using the keyboard. Make sure the target input field is focused first (use click).',
+      schema: TypeSchema,
+    })
+  );
+
+  // computer_key
+  tools.push(
+    tool(createToolFunction(EDesktopTools.KEY), {
+      name: EDesktopTools.KEY,
+      description:
+        'Press a single key on the keyboard (Enter, Tab, Escape, arrow keys, function keys, etc.).',
+      schema: KeySchema,
+    })
+  );
+
+  // computer_key_combo
+  tools.push(
+    tool(createToolFunction(EDesktopTools.KEY_COMBO), {
+      name: EDesktopTools.KEY_COMBO,
+      description:
+        'Press a key combination (e.g., Ctrl+C to copy, Ctrl+V to paste, Alt+Tab to switch windows).',
+      schema: KeyComboSchema,
+    })
+  );
+
+  // computer_scroll
+  tools.push(
+    tool(createToolFunction(EDesktopTools.SCROLL), {
+      name: EDesktopTools.SCROLL,
+      description:
+        'Scroll at the specified screen coordinates. Use negative deltaY to scroll up, positive to scroll down.',
+      schema: ScrollSchema,
+    })
+  );
+
+  // computer_drag
+  tools.push(
+    tool(createToolFunction(EDesktopTools.DRAG), {
+      name: EDesktopTools.DRAG,
+      description:
+        'Drag the mouse from one position to another (for moving windows, selecting text, etc.).',
+      schema: DragSchema,
+    })
+  );
+
+  // computer_get_active_window
+  tools.push(
+    tool(createToolFunction(EDesktopTools.GET_ACTIVE_WINDOW), {
+      name: EDesktopTools.GET_ACTIVE_WINDOW,
+      description:
+        'Get information about the currently active window (title, application name, position, size).',
+      schema: GetActiveWindowSchema,
+    })
+  );
+
+  // computer_get_mouse_position
+  tools.push(
+    tool(createToolFunction(EDesktopTools.GET_MOUSE_POSITION), {
+      name: EDesktopTools.GET_MOUSE_POSITION,
+      description: 'Get the current mouse cursor position on screen.',
+      schema: GetMousePositionSchema,
+    })
+  );
+
+  // clipboard_read
+  tools.push(
+    tool(createToolFunction(EDesktopTools.CLIPBOARD_READ), {
+      name: EDesktopTools.CLIPBOARD_READ,
+      description: 'Read the current contents of the system clipboard.',
+      schema: ClipboardReadSchema,
+    })
+  );
+
+  // clipboard_write
+  tools.push(
+    tool(createToolFunction(EDesktopTools.CLIPBOARD_WRITE), {
+      name: EDesktopTools.CLIPBOARD_WRITE,
+      description: 'Write text to the system clipboard.',
+      schema: ClipboardWriteSchema,
+    })
+  );
+
+  // clipboard_paste
+  tools.push(
+    tool(createToolFunction(EDesktopTools.CLIPBOARD_PASTE), {
+      name: EDesktopTools.CLIPBOARD_PASTE,
+      description:
+        'Paste the clipboard contents (equivalent to Ctrl+V). Use clipboard_write first to set the content.',
+      schema: ClipboardPasteSchema,
+    })
+  );
+
+  // computer_wait
+  tools.push(
+    tool(createToolFunction(EDesktopTools.WAIT), {
+      name: EDesktopTools.WAIT,
+      description:
+        'Wait for the specified number of milliseconds. Use this to wait for UI animations or loading.',
+      schema: WaitSchema,
+    })
+  );
+
+  return tools;
 }
 
 /**