claude-chrome-parallel 1.0.0

package/dist/tools/computer.js

@@ -0,0 +1,638 @@
+"use strict";
+/**
+ * Computer Tool - Mouse, keyboard, and screenshot actions
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerComputerTool = registerComputerTool;
+const session_manager_1 = require("../session-manager");
+const ref_id_manager_1 = require("../utils/ref-id-manager");
+const screenshot_scheduler_1 = require("../cdp/screenshot-scheduler");
+// Maximum screenshot dimensions to avoid API errors (2000px limit)
+const MAX_SCREENSHOT_WIDTH = 1920;
+const MAX_SCREENSHOT_HEIGHT = 1080;
+const definition = {
+    name: 'computer',
+    description: 'Use mouse and keyboard to interact with a web browser, and take screenshots.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            tabId: {
+                type: 'string',
+                description: 'Tab ID to execute the action on',
+            },
+            action: {
+                type: 'string',
+                enum: [
+                    'left_click',
+                    'right_click',
+                    'double_click',
+                    'triple_click',
+                    'type',
+                    'screenshot',
+                    'wait',
+                    'scroll',
+                    'key',
+                    'scroll_to',
+                    'hover',
+                ],
+                description: 'The action to perform',
+            },
+            coordinate: {
+                type: 'array',
+                items: { type: 'number' },
+                description: '(x, y) coordinates for click/scroll actions',
+            },
+            text: {
+                type: 'string',
+                description: 'Text to type or key to press',
+            },
+            duration: {
+                type: 'number',
+                description: 'Wait duration in seconds',
+            },
+            scroll_direction: {
+                type: 'string',
+                enum: ['up', 'down', 'left', 'right'],
+                description: 'Scroll direction',
+            },
+            scroll_amount: {
+                type: 'number',
+                description: 'Number of scroll wheel ticks',
+            },
+            ref: {
+                type: 'string',
+                description: 'Element reference ID. Can be used for scroll_to or as an alternative to coordinate for click actions.',
+            },
+        },
+        required: ['action', 'tabId'],
+    },
+};
+const handler = async (sessionId, args) => {
+    const tabId = args.tabId;
+    const action = args.action;
+    const coordinate = args.coordinate;
+    const text = args.text;
+    const duration = args.duration;
+    const scrollDirection = args.scroll_direction;
+    const scrollAmount = args.scroll_amount || 3;
+    const ref = args.ref;
+    const sessionManager = (0, session_manager_1.getSessionManager)();
+    if (!tabId) {
+        return {
+            content: [{ type: 'text', text: 'Error: tabId is required' }],
+            isError: true,
+        };
+    }
+    try {
+        const page = await sessionManager.getPage(sessionId, tabId, undefined, 'computer');
+        if (!page) {
+            return {
+                content: [{ type: 'text', text: `Error: Tab ${tabId} not found` }],
+                isError: true,
+            };
+        }
+        switch (action) {
+            case 'screenshot': {
+                // Save original viewport to restore later
+                const originalViewport = page.viewport();
+                // ALWAYS ensure viewport is within limits before screenshot
+                // This prevents the 2000px API error regardless of current state
+                const currentWidth = originalViewport?.width ?? 1920;
+                const currentHeight = originalViewport?.height ?? 1080;
+                const needsResize = !originalViewport ||
+                    currentWidth > MAX_SCREENSHOT_WIDTH ||
+                    currentHeight > MAX_SCREENSHOT_HEIGHT;
+                if (needsResize) {
+                    // Set viewport to max allowed size
+                    const newWidth = Math.min(currentWidth, MAX_SCREENSHOT_WIDTH);
+                    const newHeight = Math.min(currentHeight, MAX_SCREENSHOT_HEIGHT);
+                    await page.setViewport({
+                        width: newWidth,
+                        height: newHeight,
+                        deviceScaleFactor: 1, // Ensure 1:1 pixel ratio
+                    });
+                }
+                // Use ScreenshotScheduler for concurrency-controlled capture
+                const cdpClient = (0, session_manager_1.getSessionManager)().getCDPClient();
+                const scheduler = (0, screenshot_scheduler_1.getScreenshotScheduler)();
+                const { data: screenshot } = await scheduler.capture(page, cdpClient, {
+                    format: 'webp',
+                    quality: 60,
+                    clip: {
+                        x: 0,
+                        y: 0,
+                        width: Math.min(currentWidth, MAX_SCREENSHOT_WIDTH),
+                        height: Math.min(currentHeight, MAX_SCREENSHOT_HEIGHT),
+                        scale: 1,
+                    },
+                    optimizeForSpeed: true,
+                });
+                // Restore original viewport if we changed it
+                if (needsResize && originalViewport) {
+                    await page.setViewport(originalViewport);
+                }
+                return {
+                    content: [
+                        {
+                            type: 'image',
+                            data: screenshot,
+                            mimeType: 'image/webp',
+                        },
+                    ],
+                };
+            }
+            case 'left_click': {
+                // Support ref-based clicking
+                let clickCoord = coordinate || null;
+                let refInfo = '';
+                if (ref && !coordinate) {
+                    const refIdManager = (0, ref_id_manager_1.getRefIdManager)();
+                    const refEntry = refIdManager.getRef(sessionId, tabId, ref);
+                    if (!refEntry) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Element ${ref} not found` }],
+                            isError: true,
+                        };
+                    }
+                    // Scroll into view and get coordinates
+                    const cdpClient = sessionManager.getCDPClient();
+                    try {
+                        await cdpClient.send(page, 'DOM.scrollIntoViewIfNeeded', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        const { model } = await cdpClient.send(page, 'DOM.getBoxModel', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        // content is [x1,y1, x2,y1, x2,y2, x1,y2] - get center
+                        const x = (model.content[0] + model.content[2]) / 2;
+                        const y = (model.content[1] + model.content[5]) / 2;
+                        clickCoord = [Math.round(x), Math.round(y)];
+                        refInfo = ` [${ref}]`;
+                    }
+                    catch (e) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Could not get position for ${ref}: ${e instanceof Error ? e.message : String(e)}` }],
+                            isError: true,
+                        };
+                    }
+                }
+                if (!clickCoord) {
+                    return {
+                        content: [{ type: 'text', text: 'Error: coordinate or ref is required for left_click' }],
+                        isError: true,
+                    };
+                }
+                const leftClickValidation = await validateCoordinates(page, clickCoord[0], clickCoord[1]);
+                if (!leftClickValidation.valid) {
+                    return {
+                        content: [{ type: 'text', text: `Error: ${leftClickValidation.warning}` }],
+                        isError: true,
+                    };
+                }
+                await page.mouse.click(clickCoord[0], clickCoord[1]);
+                const resultText = leftClickValidation.warning
+                    ? `Clicked at (${clickCoord[0]}, ${clickCoord[1]})${refInfo}. Warning: ${leftClickValidation.warning}`
+                    : `Clicked at (${clickCoord[0]}, ${clickCoord[1]})${refInfo}`;
+                return {
+                    content: [{ type: 'text', text: resultText }],
+                };
+            }
+            case 'right_click': {
+                // Support ref-based clicking
+                let clickCoord = coordinate || null;
+                let refInfo = '';
+                if (ref && !coordinate) {
+                    const refIdManager = (0, ref_id_manager_1.getRefIdManager)();
+                    const refEntry = refIdManager.getRef(sessionId, tabId, ref);
+                    if (!refEntry) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Element ${ref} not found` }],
+                            isError: true,
+                        };
+                    }
+                    const cdpClient = sessionManager.getCDPClient();
+                    try {
+                        await cdpClient.send(page, 'DOM.scrollIntoViewIfNeeded', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        const { model } = await cdpClient.send(page, 'DOM.getBoxModel', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        const x = (model.content[0] + model.content[2]) / 2;
+                        const y = (model.content[1] + model.content[5]) / 2;
+                        clickCoord = [Math.round(x), Math.round(y)];
+                        refInfo = ` [${ref}]`;
+                    }
+                    catch (e) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Could not get position for ${ref}: ${e instanceof Error ? e.message : String(e)}` }],
+                            isError: true,
+                        };
+                    }
+                }
+                if (!clickCoord) {
+                    return {
+                        content: [{ type: 'text', text: 'Error: coordinate or ref is required for right_click' }],
+                        isError: true,
+                    };
+                }
+                const rightClickValidation = await validateCoordinates(page, clickCoord[0], clickCoord[1]);
+                if (!rightClickValidation.valid) {
+                    return {
+                        content: [{ type: 'text', text: `Error: ${rightClickValidation.warning}` }],
+                        isError: true,
+                    };
+                }
+                await page.mouse.click(clickCoord[0], clickCoord[1], { button: 'right' });
+                const rightClickText = rightClickValidation.warning
+                    ? `Right-clicked at (${clickCoord[0]}, ${clickCoord[1]})${refInfo}. Warning: ${rightClickValidation.warning}`
+                    : `Right-clicked at (${clickCoord[0]}, ${clickCoord[1]})${refInfo}`;
+                return {
+                    content: [{ type: 'text', text: rightClickText }],
+                };
+            }
+            case 'double_click': {
+                // Support ref-based clicking
+                let clickCoord = coordinate || null;
+                let refInfo = '';
+                if (ref && !coordinate) {
+                    const refIdManager = (0, ref_id_manager_1.getRefIdManager)();
+                    const refEntry = refIdManager.getRef(sessionId, tabId, ref);
+                    if (!refEntry) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Element ${ref} not found` }],
+                            isError: true,
+                        };
+                    }
+                    const cdpClient = sessionManager.getCDPClient();
+                    try {
+                        await cdpClient.send(page, 'DOM.scrollIntoViewIfNeeded', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        const { model } = await cdpClient.send(page, 'DOM.getBoxModel', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        const x = (model.content[0] + model.content[2]) / 2;
+                        const y = (model.content[1] + model.content[5]) / 2;
+                        clickCoord = [Math.round(x), Math.round(y)];
+                        refInfo = ` [${ref}]`;
+                    }
+                    catch (e) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Could not get position for ${ref}: ${e instanceof Error ? e.message : String(e)}` }],
+                            isError: true,
+                        };
+                    }
+                }
+                if (!clickCoord) {
+                    return {
+                        content: [
+                            { type: 'text', text: 'Error: coordinate or ref is required for double_click' },
+                        ],
+                        isError: true,
+                    };
+                }
+                const doubleClickValidation = await validateCoordinates(page, clickCoord[0], clickCoord[1]);
+                if (!doubleClickValidation.valid) {
+                    return {
+                        content: [{ type: 'text', text: `Error: ${doubleClickValidation.warning}` }],
+                        isError: true,
+                    };
+                }
+                await page.mouse.click(clickCoord[0], clickCoord[1], { clickCount: 2 });
+                const doubleClickText = doubleClickValidation.warning
+                    ? `Double-clicked at (${clickCoord[0]}, ${clickCoord[1]})${refInfo}. Warning: ${doubleClickValidation.warning}`
+                    : `Double-clicked at (${clickCoord[0]}, ${clickCoord[1]})${refInfo}`;
+                return {
+                    content: [{ type: 'text', text: doubleClickText }],
+                };
+            }
+            case 'triple_click': {
+                // Support ref-based clicking
+                let clickCoord = coordinate || null;
+                let refInfo = '';
+                if (ref && !coordinate) {
+                    const refIdManager = (0, ref_id_manager_1.getRefIdManager)();
+                    const refEntry = refIdManager.getRef(sessionId, tabId, ref);
+                    if (!refEntry) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Element ${ref} not found` }],
+                            isError: true,
+                        };
+                    }
+                    const cdpClient = sessionManager.getCDPClient();
+                    try {
+                        await cdpClient.send(page, 'DOM.scrollIntoViewIfNeeded', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        const { model } = await cdpClient.send(page, 'DOM.getBoxModel', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        const x = (model.content[0] + model.content[2]) / 2;
+                        const y = (model.content[1] + model.content[5]) / 2;
+                        clickCoord = [Math.round(x), Math.round(y)];
+                        refInfo = ` [${ref}]`;
+                    }
+                    catch (e) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Could not get position for ${ref}: ${e instanceof Error ? e.message : String(e)}` }],
+                            isError: true,
+                        };
+                    }
+                }
+                if (!clickCoord) {
+                    return {
+                        content: [
+                            { type: 'text', text: 'Error: coordinate or ref is required for triple_click' },
+                        ],
+                        isError: true,
+                    };
+                }
+                const tripleClickValidation = await validateCoordinates(page, clickCoord[0], clickCoord[1]);
+                if (!tripleClickValidation.valid) {
+                    return {
+                        content: [{ type: 'text', text: `Error: ${tripleClickValidation.warning}` }],
+                        isError: true,
+                    };
+                }
+                await page.mouse.click(clickCoord[0], clickCoord[1], { clickCount: 3 });
+                const tripleClickText = tripleClickValidation.warning
+                    ? `Triple-clicked at (${clickCoord[0]}, ${clickCoord[1]})${refInfo}. Warning: ${tripleClickValidation.warning}`
+                    : `Triple-clicked at (${clickCoord[0]}, ${clickCoord[1]})${refInfo}`;
+                return {
+                    content: [{ type: 'text', text: tripleClickText }],
+                };
+            }
+            case 'hover': {
+                // Support ref-based hovering
+                let hoverCoord = coordinate || null;
+                let refInfo = '';
+                if (ref && !coordinate) {
+                    const refIdManager = (0, ref_id_manager_1.getRefIdManager)();
+                    const refEntry = refIdManager.getRef(sessionId, tabId, ref);
+                    if (!refEntry) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Element ${ref} not found` }],
+                            isError: true,
+                        };
+                    }
+                    const cdpClient = sessionManager.getCDPClient();
+                    try {
+                        await cdpClient.send(page, 'DOM.scrollIntoViewIfNeeded', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        const { model } = await cdpClient.send(page, 'DOM.getBoxModel', {
+                            backendNodeId: refEntry.backendDOMNodeId,
+                        });
+                        const x = (model.content[0] + model.content[2]) / 2;
+                        const y = (model.content[1] + model.content[5]) / 2;
+                        hoverCoord = [Math.round(x), Math.round(y)];
+                        refInfo = ` [${ref}]`;
+                    }
+                    catch (e) {
+                        return {
+                            content: [{ type: 'text', text: `Error: Could not get position for ${ref}: ${e instanceof Error ? e.message : String(e)}` }],
+                            isError: true,
+                        };
+                    }
+                }
+                if (!hoverCoord) {
+                    return {
+                        content: [{ type: 'text', text: 'Error: coordinate or ref is required for hover' }],
+                        isError: true,
+                    };
+                }
+                const hoverValidation = await validateCoordinates(page, hoverCoord[0], hoverCoord[1]);
+                if (!hoverValidation.valid) {
+                    return {
+                        content: [{ type: 'text', text: `Error: ${hoverValidation.warning}` }],
+                        isError: true,
+                    };
+                }
+                await page.mouse.move(hoverCoord[0], hoverCoord[1]);
+                const hoverText = hoverValidation.warning
+                    ? `Hovered at (${hoverCoord[0]}, ${hoverCoord[1]})${refInfo}. Warning: ${hoverValidation.warning}`
+                    : `Hovered at (${hoverCoord[0]}, ${hoverCoord[1]})${refInfo}`;
+                return {
+                    content: [{ type: 'text', text: hoverText }],
+                };
+            }
+            case 'type': {
+                if (!text) {
+                    return {
+                        content: [{ type: 'text', text: 'Error: text is required for type action' }],
+                        isError: true,
+                    };
+                }
+                await page.keyboard.type(text);
+                return {
+                    content: [{ type: 'text', text: `Typed: ${text}` }],
+                };
+            }
+            case 'key': {
+                if (!text) {
+                    return {
+                        content: [{ type: 'text', text: 'Error: text is required for key action' }],
+                        isError: true,
+                    };
+                }
+                // Handle multiple keys separated by space
+                const keys = text.split(' ');
+                for (const key of keys) {
+                    if (key.includes('+')) {
+                        // Handle modifier keys like ctrl+a
+                        const parts = key.split('+');
+                        const modifiers = parts.slice(0, -1);
+                        const mainKey = parts[parts.length - 1];
+                        for (const mod of modifiers) {
+                            await page.keyboard.down(normalizeKey(mod));
+                        }
+                        await page.keyboard.press(normalizeKey(mainKey));
+                        for (const mod of modifiers.reverse()) {
+                            await page.keyboard.up(normalizeKey(mod));
+                        }
+                    }
+                    else {
+                        await page.keyboard.press(normalizeKey(key));
+                    }
+                }
+                return {
+                    content: [{ type: 'text', text: `Pressed: ${text}` }],
+                };
+            }
+            case 'wait': {
+                // Validate duration
+                if (duration !== undefined && duration < 0) {
+                    return {
+                        content: [{ type: 'text', text: 'Error: duration cannot be negative' }],
+                        isError: true,
+                    };
+                }
+                const waitTime = Math.min(Math.max((duration || 1) * 1000, 0), 30000);
+                await new Promise((resolve) => setTimeout(resolve, waitTime));
+                return {
+                    content: [{ type: 'text', text: `Waited ${waitTime / 1000} seconds` }],
+                };
+            }
+            case 'scroll': {
+                if (!coordinate) {
+                    return {
+                        content: [{ type: 'text', text: 'Error: coordinate is required for scroll' }],
+                        isError: true,
+                    };
+                }
+                if (!scrollDirection) {
+                    return {
+                        content: [
+                            { type: 'text', text: 'Error: scroll_direction is required for scroll' },
+                        ],
+                        isError: true,
+                    };
+                }
+                const scrollValidation = await validateCoordinates(page, coordinate[0], coordinate[1]);
+                if (!scrollValidation.valid) {
+                    return {
+                        content: [{ type: 'text', text: `Error: ${scrollValidation.warning}` }],
+                        isError: true,
+                    };
+                }
+                const deltaMultiplier = 100;
+                let deltaX = 0;
+                let deltaY = 0;
+                switch (scrollDirection) {
+                    case 'up':
+                        deltaY = -scrollAmount * deltaMultiplier;
+                        break;
+                    case 'down':
+                        deltaY = scrollAmount * deltaMultiplier;
+                        break;
+                    case 'left':
+                        deltaX = -scrollAmount * deltaMultiplier;
+                        break;
+                    case 'right':
+                        deltaX = scrollAmount * deltaMultiplier;
+                        break;
+                }
+                // Try CDP mouse wheel first, fallback to JavaScript scroll if it times out
+                let usedFallback = false;
+                try {
+                    await page.mouse.move(coordinate[0], coordinate[1]);
+                    await page.mouse.wheel({ deltaX, deltaY });
+                }
+                catch (scrollError) {
+                    // Fallback to JavaScript scroll if mouse.wheel fails (timeout, etc.)
+                    usedFallback = true;
+                    await page.evaluate((dx, dy) => {
+                        window.scrollBy(dx, dy);
+                    }, deltaX, deltaY);
+                }
+                const scrollMethod = usedFallback ? ' (via JavaScript fallback)' : '';
+                return {
+                    content: [
+                        {
+                            type: 'text',
+                            text: `Scrolled ${scrollDirection} at (${coordinate[0]}, ${coordinate[1]})${scrollMethod}`,
+                        },
+                    ],
+                };
+            }
+            case 'scroll_to': {
+                if (!ref) {
+                    return {
+                        content: [{ type: 'text', text: 'Error: ref is required for scroll_to' }],
+                        isError: true,
+                    };
+                }
+                const refIdManager = (0, ref_id_manager_1.getRefIdManager)();
+                const refEntry = refIdManager.getRef(sessionId, tabId, ref);
+                if (!refEntry) {
+                    return {
+                        content: [{ type: 'text', text: `Error: Element ${ref} not found` }],
+                        isError: true,
+                    };
+                }
+                // Use CDP to scroll element into view
+                const cdpClient = sessionManager.getCDPClient();
+                await cdpClient.send(page, 'DOM.scrollIntoViewIfNeeded', {
+                    backendNodeId: refEntry.backendDOMNodeId,
+                });
+                return {
+                    content: [{ type: 'text', text: `Scrolled to ${ref}` }],
+                };
+            }
+            default:
+                return {
+                    content: [{ type: 'text', text: `Error: Unknown action: ${action}` }],
+                    isError: true,
+                };
+        }
+    }
+    catch (error) {
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Computer action error: ${error instanceof Error ? error.message : String(error)}`,
+                },
+            ],
+            isError: true,
+        };
+    }
+};
+/**
+ * Validate and check coordinates against viewport bounds
+ */
+async function validateCoordinates(page, x, y) {
+    // Check for negative coordinates
+    if (x < 0 || y < 0) {
+        return {
+            valid: false,
+            warning: `Negative coordinates (${x}, ${y}) are not allowed`,
+        };
+    }
+    try {
+        // Get viewport dimensions
+        const viewport = page.viewport();
+        if (viewport) {
+            const { width, height } = viewport;
+            if (x > width || y > height) {
+                return {
+                    valid: true,
+                    warning: `Coordinates (${x}, ${y}) are outside visible viewport (${width}x${height}). The click may not hit the intended target.`,
+                };
+            }
+        }
+    }
+    catch {
+        // If we can't get viewport, just allow the operation
+    }
+    return { valid: true };
+}
+function normalizeKey(key) {
+    const keyMap = {
+        ctrl: 'Control',
+        cmd: 'Meta',
+        meta: 'Meta',
+        alt: 'Alt',
+        shift: 'Shift',
+        enter: 'Enter',
+        tab: 'Tab',
+        escape: 'Escape',
+        esc: 'Escape',
+        backspace: 'Backspace',
+        delete: 'Delete',
+        up: 'ArrowUp',
+        down: 'ArrowDown',
+        left: 'ArrowLeft',
+        right: 'ArrowRight',
+        home: 'Home',
+        end: 'End',
+        pageup: 'PageUp',
+        pagedown: 'PageDown',
+    };
+    return keyMap[key.toLowerCase()] || key;
+}
+function registerComputerTool(server) {
+    server.registerTool('computer', handler, definition);
+}
+//# sourceMappingURL=computer.js.map