openclaw-cascade-plugin 1.0.0

package/src/tools/desktop-automation.test.ts

@@ -0,0 +1,305 @@
+/**
+ * Tests for Desktop Automation Tools
+ */
+
+import { registerDesktopTools } from './desktop-automation';
+import { ToolRegistry } from './tool-registry';
+import { MockCascadeMcpClient } from '../test-utils';
+
+describe('Desktop Automation Tools', () => {
+  let registry: ToolRegistry;
+  let mockClient: MockCascadeMcpClient;
+
+  beforeEach(() => {
+    registry = new ToolRegistry();
+    mockClient = new MockCascadeMcpClient();
+    registerDesktopTools(registry, mockClient as any);
+  });
+
+  describe('cascade_click_element', () => {
+    test('should be registered', () => {
+      expect(registry.has('cascade_click_element')).toBe(true);
+    });
+
+    test('should click element by ID', async () => {
+      // Arrange
+      mockClient.registerMockTool('click_element', () => ({ success: true }));
+
+      // Act
+      const result = await registry.call('cascade_click_element', {
+        platform_source: 'WINDOWS',
+        id: 'button1'
+      });
+
+      // Assert
+      expect(mockClient.callTool).toHaveBeenCalledWith('click_element', {
+        platform_source: 'WINDOWS',
+        id: 'button1'
+      });
+      expect(result.content[0].text).toContain('success');
+    });
+
+    test('should handle click failure', async () => {
+      // Arrange
+      mockClient.simulateError(new Error('Element not found'));
+
+      // Act
+      const result = await registry.call('cascade_click_element', {
+        platform_source: 'WINDOWS',
+        id: 'nonexistent'
+      });
+
+      // Assert
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain('Element not found');
+    });
+
+    test('should require platform_source', async () => {
+      // Act
+      const result = await registry.call('cascade_click_element', {
+        id: 'button1'
+      });
+
+      // Assert
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain('platform_source is required');
+    });
+  });
+
+  describe('cascade_type_text', () => {
+    test('should be registered', () => {
+      expect(registry.has('cascade_type_text')).toBe(true);
+    });
+
+    test('should type text into element', async () => {
+      // Arrange
+      mockClient.registerMockTool('type_text', () => ({ success: true }));
+
+      // Act
+      await registry.call('cascade_type_text', {
+        selector: { platform_source: 'WINDOWS', id: 'input1' },
+        text: 'Hello World'
+      });
+
+      // Assert
+      expect(mockClient.callTool).toHaveBeenCalledWith('type_text', {
+        selector: { platform_source: 'WINDOWS', id: 'input1' },
+        text: 'Hello World'
+      });
+    });
+
+    test('should require selector', async () => {
+      // Act
+      const result = await registry.call('cascade_type_text', {
+        text: 'Hello'
+      });
+
+      // Assert
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain('selector');
+    });
+
+    test('should require text', async () => {
+      // Act
+      const result = await registry.call('cascade_type_text', {
+        selector: { platform_source: 'WINDOWS' }
+      });
+
+      // Assert
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain('text is required');
+    });
+  });
+
+  describe('cascade_get_semantic_tree', () => {
+    test('should be registered', () => {
+      expect(registry.has('cascade_get_semantic_tree')).toBe(true);
+    });
+
+    test('should return semantic tree', async () => {
+      // Arrange
+      mockClient.registerMockTool('get_semantic_tree', () => ({
+        elements: [
+          { id: '1', name: 'Button', control_type: 'BUTTON' },
+          { id: '2', name: 'Input', control_type: 'INPUT' }
+        ]
+      }));
+
+      // Act
+      const result = await registry.call('cascade_get_semantic_tree', {});
+
+      // Assert
+      expect(mockClient.callTool).toHaveBeenCalledWith('get_semantic_tree', {});
+      expect(result.content[0].text).toContain('Button');
+      expect(result.content[0].text).toContain('Input');
+    });
+
+    test('should handle empty tree', async () => {
+      // Arrange
+      mockClient.registerMockTool('get_semantic_tree', () => ({
+        elements: []
+      }));
+
+      // Act
+      const result = await registry.call('cascade_get_semantic_tree', {});
+
+      // Assert
+      expect(result.content[0].text).toContain('elements');
+    });
+  });
+
+  describe('cascade_get_screenshot', () => {
+    test('should be registered', () => {
+      expect(registry.has('cascade_get_screenshot')).toBe(true);
+    });
+
+    test('should return embedded screenshot when small', async () => {
+      // Arrange
+      const smallImage = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==';
+      mockClient.registerMockTool('get_screenshot', () => ({
+        image: smallImage,
+        format: 'PNG',
+        marks: [{ element_id: '1', label: 'A' }]
+      }));
+
+      // Act
+      const result = await registry.call('cascade_get_screenshot', {});
+
+      // Assert
+      expect(result.content[0].type).toBe('image');
+      expect(result.content[0].source.data).toBe(smallImage);
+    });
+
+    test('should save large screenshot to disk', async () => {
+      // Arrange
+      const largeImage = 'A'.repeat(5000000); // 5MB base64 = ~3.75MB actual
+      mockClient.registerMockTool('get_screenshot', () => ({
+        image: largeImage,
+        format: 'PNG',
+        marks: []
+      }));
+
+      // Act
+      const result = await registry.call('cascade_get_screenshot', {});
+
+      // Assert
+      expect(result.content[0].type).toBe('text');
+      expect(result.content[0].text).toContain('saved');
+    });
+  });
+
+  describe('cascade_start_app', () => {
+    test('should be registered', () => {
+      expect(registry.has('cascade_start_app')).toBe(true);
+    });
+
+    test('should start application', async () => {
+      // Arrange
+      mockClient.registerMockTool('start_app', () => ({ success: true }));
+
+      // Act
+      const result = await registry.call('cascade_start_app', {
+        app_name: 'notepad'
+      });
+
+      // Assert
+      expect(mockClient.callTool).toHaveBeenCalledWith('start_app', {
+        app_name: 'notepad'
+      });
+      expect(result.content[0].text).toContain('success');
+    });
+
+    test('should require app_name', async () => {
+      // Act
+      const result = await registry.call('cascade_start_app', {});
+
+      // Assert
+      expect(result.isError).toBe(true);
+      expect(result.content[0].text).toContain('app_name is required');
+    });
+  });
+
+  describe('cascade_hover_element', () => {
+    test('should be registered', () => {
+      expect(registry.has('cascade_hover_element')).toBe(true);
+    });
+
+    test('should hover over element', async () => {
+      // Arrange
+      mockClient.registerMockTool('hover_element', () => ({ success: true }));
+
+      // Act
+      await registry.call('cascade_hover_element', {
+        selector: { platform_source: 'WINDOWS', id: 'button1' }
+      });
+
+      // Assert
+      expect(mockClient.callTool).toHaveBeenCalledWith('hover_element', {
+        selector: { platform_source: 'WINDOWS', id: 'button1' }
+      });
+    });
+  });
+
+  describe('cascade_focus_element', () => {
+    test('should be registered', () => {
+      expect(registry.has('cascade_focus_element')).toBe(true);
+    });
+
+    test('should focus on element', async () => {
+      // Arrange
+      mockClient.registerMockTool('focus_element', () => ({ success: true }));
+
+      // Act
+      await registry.call('cascade_focus_element', {
+        selector: { platform_source: 'WINDOWS', id: 'input1' }
+      });
+
+      // Assert
+      expect(mockClient.callTool).toHaveBeenCalledWith('focus_element', {
+        selector: { platform_source: 'WINDOWS', id: 'input1' }
+      });
+    });
+  });
+
+  describe('cascade_scroll_element', () => {
+    test('should be registered', () => {
+      expect(registry.has('cascade_scroll_element')).toBe(true);
+    });
+
+    test('should scroll element', async () => {
+      // Arrange
+      mockClient.registerMockTool('scroll_element', () => ({ success: true }));
+
+      // Act
+      await registry.call('cascade_scroll_element', {
+        selector: { platform_source: 'WINDOWS', id: 'scrollable' }
+      });
+
+      // Assert
+      expect(mockClient.callTool).toHaveBeenCalledWith('scroll_element', {
+        selector: { platform_source: 'WINDOWS', id: 'scrollable' }
+      });
+    });
+  });
+
+  describe('cascade_wait_visible', () => {
+    test('should be registered', () => {
+      expect(registry.has('cascade_wait_visible')).toBe(true);
+    });
+
+    test('should wait for element visibility', async () => {
+      // Arrange
+      mockClient.registerMockTool('wait_visible', () => ({ success: true }));
+
+      // Act
+      await registry.call('cascade_wait_visible', {
+        selector: { platform_source: 'WINDOWS', id: 'loading' }
+      });
+
+      // Assert
+      expect(mockClient.callTool).toHaveBeenCalledWith('wait_visible', {
+        selector: { platform_source: 'WINDOWS', id: 'loading' }
+      });
+    });
+  });
+});

package/src/tools/desktop-automation.ts

@@ -0,0 +1,366 @@
+/**
+ * Desktop Automation Tools
+ * 
+ * 9 tools for controlling Windows desktop applications
+ */
+
+import { ToolRegistry } from './tool-registry';
+import { CascadeMcpClient } from '../cascade-client';
+import { ToolResponse, CascadePluginConfig } from '../types';
+import { errorResponse, formatSuccess, imageResponse } from './response-helpers';
+import { writeFile, mkdir } from 'fs/promises';
+import { join } from 'path';
+import { homedir } from 'os';
+
+// Max size for embedded images (4MB)
+const MAX_EMBED_SIZE = 4 * 1024 * 1024;
+
+export function registerDesktopTools(
+  registry: ToolRegistry,
+  client: CascadeMcpClient,
+  config?: CascadePluginConfig
+): void {
+  const screenshotMode = config?.screenshotMode || 'auto';
+  const screenshotDir = config?.screenshotDir || join(homedir(), '.openclaw', 'screenshots');
+
+  // 1. cascade_click_element
+  registry.register({
+    name: 'cascade_click_element',
+    description: 'Click on a UI element on the desktop or in an application',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        platform_source: {
+          type: 'string',
+          enum: ['WINDOWS', 'JAVA', 'WEB'],
+          description: 'Platform type (Windows desktop, Java app, or web)'
+        },
+        id: { type: 'string', description: 'Element ID if known' },
+        name: { type: 'string', description: 'Element name/label' },
+        control_type: {
+          type: 'string',
+          enum: ['BUTTON', 'INPUT', 'COMBO', 'MENU', 'TREE', 'TABLE', 'CUSTOM'],
+          description: 'Control type'
+        },
+        path: {
+          type: 'array',
+          items: { type: 'string' },
+          description: 'Path components for navigation'
+        },
+        index: { type: 'integer', description: 'Index if multiple matches' },
+        text_hint: { type: 'string', description: 'Text hint for element' }
+      },
+      required: ['platform_source']
+    },
+    handler: async (args): Promise<ToolResponse> => {
+      try {
+        if (!args.platform_source) {
+          return errorResponse('platform_source is required');
+        }
+
+        const result = await client.callTool('click_element', args);
+        return formatSuccess(result);
+      } catch (error) {
+        return errorResponse(
+          error instanceof Error ? error.message : 'Failed to click element',
+          'Try using cascade_get_semantic_tree first to identify the correct element'
+        );
+      }
+    }
+  });
+
+  // 2. cascade_type_text
+  registry.register({
+    name: 'cascade_type_text',
+    description: 'Type text into a UI element (input field, text area, etc.)',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        selector: {
+          type: 'object',
+          description: 'Element selector',
+          properties: {
+            platform_source: {
+              type: 'string',
+              enum: ['WINDOWS', 'JAVA', 'WEB']
+            },
+            id: { type: 'string' },
+            name: { type: 'string' },
+            control_type: {
+              type: 'string',
+              enum: ['BUTTON', 'INPUT', 'COMBO', 'MENU', 'TREE', 'TABLE', 'CUSTOM']
+            },
+            path: { type: 'array', items: { type: 'string' } },
+            index: { type: 'integer' },
+            text_hint: { type: 'string' }
+          },
+          required: ['platform_source']
+        },
+        text: {
+          type: 'string',
+          description: 'Text to type'
+        }
+      },
+      required: ['selector', 'text']
+    },
+    handler: async (args): Promise<ToolResponse> => {
+      try {
+        if (!args.selector) {
+          return errorResponse('selector is required');
+        }
+        if (!args.text) {
+          return errorResponse('text is required');
+        }
+
+        const result = await client.callTool('type_text', args);
+        return formatSuccess(result);
+      } catch (error) {
+        return errorResponse(
+          error instanceof Error ? error.message : 'Failed to type text'
+        );
+      }
+    }
+  });
+
+  // 3. cascade_get_semantic_tree
+  registry.register({
+    name: 'cascade_get_semantic_tree',
+    description: 'Get the UI element structure of the current window or application',
+    inputSchema: {
+      type: 'object',
+      properties: {}
+    },
+    handler: async (): Promise<ToolResponse> => {
+      try {
+        const result = await client.callTool('get_semantic_tree', {});
+        return formatSuccess(result);
+      } catch (error) {
+        return errorResponse(
+          error instanceof Error ? error.message : 'Failed to get semantic tree'
+        );
+      }
+    }
+  });
+
+  // 4. cascade_get_screenshot
+  registry.register({
+    name: 'cascade_get_screenshot',
+    description: 'Capture a screenshot of the current screen with element annotations',
+    inputSchema: {
+      type: 'object',
+      properties: {}
+    },
+    handler: async (): Promise<ToolResponse> => {
+      try {
+        const result = await client.callTool('get_screenshot', {});
+        
+        if (!result.image) {
+          return errorResponse('No image data received');
+        }
+
+        const imageSize = Buffer.from(result.image, 'base64').length;
+        const shouldEmbed = screenshotMode === 'embed' || 
+                           (screenshotMode === 'auto' && imageSize < MAX_EMBED_SIZE);
+
+        if (shouldEmbed) {
+          return imageResponse(
+            result.image,
+            result.format?.toLowerCase() === 'jpeg' ? 'jpeg' : 'png',
+            `Screenshot captured (${(imageSize / 1024).toFixed(1)} KB)`
+          );
+        } else {
+          // Save to disk
+          await mkdir(screenshotDir, { recursive: true });
+          const filename = `cascade-${Date.now()}.${result.format?.toLowerCase() || 'png'}`;
+          const filepath = join(screenshotDir, filename);
+          await writeFile(filepath, Buffer.from(result.image, 'base64'));
+          
+          return formatSuccess({
+            saved_to: filepath,
+            size_kb: (imageSize / 1024).toFixed(1),
+            marks: result.marks
+          });
+        }
+      } catch (error) {
+        return errorResponse(
+          error instanceof Error ? error.message : 'Failed to capture screenshot'
+        );
+      }
+    }
+  });
+
+  // 5. cascade_start_app
+  registry.register({
+    name: 'cascade_start_app',
+    description: 'Start an application by name',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        app_name: {
+          type: 'string',
+          description: 'Application name or executable (e.g., notepad, calc, excel)'
+        }
+      },
+      required: ['app_name']
+    },
+    handler: async (args): Promise<ToolResponse> => {
+      try {
+        if (!args.app_name) {
+          return errorResponse('app_name is required');
+        }
+
+        const result = await client.callTool('start_app', args);
+        return formatSuccess(result);
+      } catch (error) {
+        return errorResponse(
+          error instanceof Error ? error.message : 'Failed to start application',
+          'Verify the app name is correct and the application is installed'
+        );
+      }
+    }
+  });
+
+  // 6. cascade_hover_element
+  registry.register({
+    name: 'cascade_hover_element',
+    description: 'Hover over a UI element',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        selector: {
+          type: 'object',
+          description: 'Element selector',
+          properties: {
+            platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'] },
+            id: { type: 'string' },
+            name: { type: 'string' },
+            control_type: { type: 'string' },
+            path: { type: 'array', items: { type: 'string' } },
+            index: { type: 'integer' },
+            text_hint: { type: 'string' }
+          },
+          required: ['platform_source']
+        }
+      },
+      required: ['selector']
+    },
+    handler: async (args): Promise<ToolResponse> => {
+      try {
+        const result = await client.callTool('hover_element', args);
+        return formatSuccess(result);
+      } catch (error) {
+        return errorResponse(
+          error instanceof Error ? error.message : 'Failed to hover element'
+        );
+      }
+    }
+  });
+
+  // 7. cascade_focus_element
+  registry.register({
+    name: 'cascade_focus_element',
+    description: 'Focus on a UI element',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        selector: {
+          type: 'object',
+          description: 'Element selector',
+          properties: {
+            platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'] },
+            id: { type: 'string' },
+            name: { type: 'string' },
+            control_type: { type: 'string' },
+            path: { type: 'array', items: { type: 'string' } },
+            index: { type: 'integer' },
+            text_hint: { type: 'string' }
+          },
+          required: ['platform_source']
+        }
+      },
+      required: ['selector']
+    },
+    handler: async (args): Promise<ToolResponse> => {
+      try {
+        const result = await client.callTool('focus_element', args);
+        return formatSuccess(result);
+      } catch (error) {
+        return errorResponse(
+          error instanceof Error ? error.message : 'Failed to focus element'
+        );
+      }
+    }
+  });
+
+  // 8. cascade_scroll_element
+  registry.register({
+    name: 'cascade_scroll_element',
+    description: 'Scroll a UI element',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        selector: {
+          type: 'object',
+          description: 'Element selector',
+          properties: {
+            platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'] },
+            id: { type: 'string' },
+            name: { type: 'string' },
+            control_type: { type: 'string' },
+            path: { type: 'array', items: { type: 'string' } },
+            index: { type: 'integer' },
+            text_hint: { type: 'string' }
+          },
+          required: ['platform_source']
+        }
+      },
+      required: ['selector']
+    },
+    handler: async (args): Promise<ToolResponse> => {
+      try {
+        const result = await client.callTool('scroll_element', args);
+        return formatSuccess(result);
+      } catch (error) {
+        return errorResponse(
+          error instanceof Error ? error.message : 'Failed to scroll element'
+        );
+      }
+    }
+  });
+
+  // 9. cascade_wait_visible
+  registry.register({
+    name: 'cascade_wait_visible',
+    description: 'Wait for a UI element to become visible',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        selector: {
+          type: 'object',
+          description: 'Element selector',
+          properties: {
+            platform_source: { type: 'string', enum: ['WINDOWS', 'JAVA', 'WEB'] },
+            id: { type: 'string' },
+            name: { type: 'string' },
+            control_type: { type: 'string' },
+            path: { type: 'array', items: { type: 'string' } },
+            index: { type: 'integer' },
+            text_hint: { type: 'string' }
+          },
+          required: ['platform_source']
+        }
+      },
+      required: ['selector']
+    },
+    handler: async (args): Promise<ToolResponse> => {
+      try {
+        const result = await client.callTool('wait_visible', args);
+        return formatSuccess(result);
+      } catch (error) {
+        return errorResponse(
+          error instanceof Error ? error.message : 'Failed to wait for element'
+        );
+      }
+    }
+  });
+}

package/src/tools/index.ts

@@ -0,0 +1,13 @@
+/**
+ * Tools Index
+ * 
+ * Export all tool registration functions
+ */
+
+export { registerDesktopTools } from './desktop-automation';
+export { registerWebTools } from './web-automation';
+export { registerApiTools } from './api-tools';
+export { registerSandboxTools } from './sandbox-tools';
+export { registerA2ATools } from './a2a-tools';
+// Note: ToolRegistry class is not exported from main index to avoid conflict with ToolRegistry interface
+export * from './response-helpers';