cowork-os 0.3.21

package/dist/electron/electron/agent/tools/node-tools.js

@@ -0,0 +1,476 @@
+"use strict";
+/**
+ * Node Tools for AI Agent
+ *
+ * Tools that enable the AI agent to interact with mobile companion nodes.
+ * These tools allow the agent to:
+ * - List connected nodes
+ * - Take photos using node cameras
+ * - Get location from nodes
+ * - Record screen from nodes
+ * - Send SMS (Android only)
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.nodeTools = exports.nodeSmsSendTool = exports.nodeScreenRecordTool = exports.nodeLocationTool = exports.nodeCameraSnapTool = exports.nodeDescribeTool = exports.nodeListTool = void 0;
+exports.registerNodeTools = registerNodeTools;
+const control_plane_1 = require("../../control-plane");
+/**
+ * List connected mobile companion nodes
+ */
+exports.nodeListTool = {
+    name: 'node_list',
+    description: 'List connected mobile companion devices (nodes). Returns information about each connected device including its capabilities and available commands.',
+    inputSchema: {
+        type: 'object',
+        properties: {},
+        required: [],
+    },
+    riskLevel: 'read',
+    groups: ['read'],
+    handler: async () => {
+        const nodeManager = (0, control_plane_1.getNodeManager)();
+        if (!nodeManager.isAttached) {
+            return {
+                type: 'text',
+                content: 'No mobile companions are available. The Control Plane server is not running.',
+            };
+        }
+        const nodes = nodeManager.getNodes();
+        if (nodes.length === 0) {
+            return {
+                type: 'text',
+                content: 'No mobile companion devices are currently connected.',
+            };
+        }
+        const nodeList = nodes.map((n) => ({
+            id: n.id,
+            name: n.displayName,
+            platform: n.platform,
+            capabilities: n.capabilities,
+            commands: n.commands,
+            isForeground: n.isForeground,
+        }));
+        return {
+            type: 'json',
+            content: JSON.stringify({ nodes: nodeList }, null, 2),
+        };
+    },
+};
+/**
+ * Get detailed info about a specific node
+ */
+exports.nodeDescribeTool = {
+    name: 'node_describe',
+    description: 'Get detailed information about a connected mobile companion device, including its capabilities, permissions, and connection status.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            nodeId: {
+                type: 'string',
+                description: 'The ID or display name of the node to describe',
+            },
+        },
+        required: ['nodeId'],
+    },
+    riskLevel: 'read',
+    groups: ['read'],
+    handler: async (params) => {
+        const nodeManager = (0, control_plane_1.getNodeManager)();
+        if (!nodeManager.isAttached) {
+            return {
+                type: 'text',
+                content: 'Control Plane server is not running.',
+                isError: true,
+            };
+        }
+        const node = nodeManager.getNode(params.nodeId);
+        if (!node) {
+            return {
+                type: 'text',
+                content: `Node not found: ${params.nodeId}`,
+                isError: true,
+            };
+        }
+        return {
+            type: 'json',
+            content: JSON.stringify(node, null, 2),
+        };
+    },
+};
+/**
+ * Take a photo using a node's camera
+ */
+exports.nodeCameraSnapTool = {
+    name: 'node_camera_snap',
+    description: 'Take a photo using a mobile companion\'s camera. Returns the image as base64-encoded JPEG. The node app must be in the foreground.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            nodeId: {
+                type: 'string',
+                description: 'The ID or display name of the node to use. If omitted, uses the first available node.',
+            },
+            facing: {
+                type: 'string',
+                enum: ['front', 'back'],
+                description: 'Which camera to use: "front" (selfie) or "back" (main camera). Defaults to "back".',
+            },
+            maxWidth: {
+                type: 'number',
+                description: 'Maximum width in pixels for resizing. If omitted, uses original resolution.',
+            },
+            quality: {
+                type: 'number',
+                description: 'JPEG quality from 0 to 1. Defaults to 0.8.',
+            },
+        },
+        required: [],
+    },
+    riskLevel: 'read',
+    groups: ['read'],
+    handler: async (params) => {
+        const nodeManager = (0, control_plane_1.getNodeManager)();
+        if (!nodeManager.isAttached) {
+            return {
+                type: 'text',
+                content: 'Control Plane server is not running.',
+                isError: true,
+            };
+        }
+        // Find the node
+        let nodeId = params.nodeId;
+        if (!nodeId) {
+            const defaultNode = nodeManager.getDefaultNode();
+            if (!defaultNode) {
+                return {
+                    type: 'text',
+                    content: 'No mobile companion devices are connected.',
+                    isError: true,
+                };
+            }
+            nodeId = defaultNode.id;
+        }
+        // Check if node supports camera
+        const node = nodeManager.getNode(nodeId);
+        if (!node) {
+            return {
+                type: 'text',
+                content: `Node not found: ${nodeId}`,
+                isError: true,
+            };
+        }
+        if (!node.capabilities.includes('camera')) {
+            return {
+                type: 'text',
+                content: `Node "${node.displayName}" does not have camera capability.`,
+                isError: true,
+            };
+        }
+        // Invoke the camera snap command
+        const result = await nodeManager.cameraSnap(nodeId, {
+            facing: params.facing,
+            maxWidth: params.maxWidth,
+            quality: params.quality,
+        });
+        if (!result.ok) {
+            return {
+                type: 'text',
+                content: `Failed to take photo: ${result.error?.message || 'Unknown error'}`,
+                isError: true,
+            };
+        }
+        const payload = result.payload;
+        return {
+            type: 'image',
+            content: payload.base64,
+            mimeType: `image/${payload.format}`,
+        };
+    },
+};
+/**
+ * Get location from a node
+ */
+exports.nodeLocationTool = {
+    name: 'node_location',
+    description: 'Get the current GPS location from a mobile companion device. Returns latitude, longitude, and accuracy.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            nodeId: {
+                type: 'string',
+                description: 'The ID or display name of the node to use. If omitted, uses the first available node.',
+            },
+            accuracy: {
+                type: 'string',
+                enum: ['coarse', 'precise'],
+                description: 'Desired accuracy: "coarse" (faster, less battery) or "precise" (GPS). Defaults to "precise".',
+            },
+        },
+        required: [],
+    },
+    riskLevel: 'read',
+    groups: ['read'],
+    handler: async (params) => {
+        const nodeManager = (0, control_plane_1.getNodeManager)();
+        if (!nodeManager.isAttached) {
+            return {
+                type: 'text',
+                content: 'Control Plane server is not running.',
+                isError: true,
+            };
+        }
+        // Find the node
+        let nodeId = params.nodeId;
+        if (!nodeId) {
+            const defaultNode = nodeManager.getDefaultNode();
+            if (!defaultNode) {
+                return {
+                    type: 'text',
+                    content: 'No mobile companion devices are connected.',
+                    isError: true,
+                };
+            }
+            nodeId = defaultNode.id;
+        }
+        // Check if node supports location
+        const node = nodeManager.getNode(nodeId);
+        if (!node) {
+            return {
+                type: 'text',
+                content: `Node not found: ${nodeId}`,
+                isError: true,
+            };
+        }
+        if (!node.capabilities.includes('location')) {
+            return {
+                type: 'text',
+                content: `Node "${node.displayName}" does not have location capability.`,
+                isError: true,
+            };
+        }
+        // Invoke the location get command
+        const result = await nodeManager.locationGet(nodeId, {
+            accuracy: params.accuracy,
+        });
+        if (!result.ok) {
+            return {
+                type: 'text',
+                content: `Failed to get location: ${result.error?.message || 'Unknown error'}`,
+                isError: true,
+            };
+        }
+        const payload = result.payload;
+        return {
+            type: 'json',
+            content: JSON.stringify({
+                latitude: payload.latitude,
+                longitude: payload.longitude,
+                accuracy: `${payload.accuracy.toFixed(1)} meters`,
+                altitude: payload.altitude ? `${payload.altitude.toFixed(1)} meters` : undefined,
+                timestamp: new Date(payload.timestamp).toISOString(),
+                mapsUrl: `https://www.google.com/maps?q=${payload.latitude},${payload.longitude}`,
+            }, null, 2),
+        };
+    },
+};
+/**
+ * Record screen from a node
+ */
+exports.nodeScreenRecordTool = {
+    name: 'node_screen_record',
+    description: 'Record the screen of a mobile companion device. Returns the video as base64-encoded MP4. The node app must be in the foreground.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            nodeId: {
+                type: 'string',
+                description: 'The ID or display name of the node to use. If omitted, uses the first available node.',
+            },
+            durationMs: {
+                type: 'number',
+                description: 'Recording duration in milliseconds. Maximum is 60000 (1 minute). Defaults to 5000.',
+            },
+            fps: {
+                type: 'number',
+                description: 'Frames per second. Defaults to 10.',
+            },
+        },
+        required: [],
+    },
+    riskLevel: 'read',
+    groups: ['read'],
+    handler: async (params) => {
+        const nodeManager = (0, control_plane_1.getNodeManager)();
+        if (!nodeManager.isAttached) {
+            return {
+                type: 'text',
+                content: 'Control Plane server is not running.',
+                isError: true,
+            };
+        }
+        // Find the node
+        let nodeId = params.nodeId;
+        if (!nodeId) {
+            const defaultNode = nodeManager.getDefaultNode();
+            if (!defaultNode) {
+                return {
+                    type: 'text',
+                    content: 'No mobile companion devices are connected.',
+                    isError: true,
+                };
+            }
+            nodeId = defaultNode.id;
+        }
+        // Check if node supports screen recording
+        const node = nodeManager.getNode(nodeId);
+        if (!node) {
+            return {
+                type: 'text',
+                content: `Node not found: ${nodeId}`,
+                isError: true,
+            };
+        }
+        if (!node.capabilities.includes('screen')) {
+            return {
+                type: 'text',
+                content: `Node "${node.displayName}" does not have screen recording capability.`,
+                isError: true,
+            };
+        }
+        const durationMs = Math.min(params.durationMs || 5000, 60000);
+        // Invoke the screen record command
+        const result = await nodeManager.screenRecord(nodeId, {
+            durationMs,
+            fps: params.fps,
+        });
+        if (!result.ok) {
+            return {
+                type: 'text',
+                content: `Failed to record screen: ${result.error?.message || 'Unknown error'}`,
+                isError: true,
+            };
+        }
+        const payload = result.payload;
+        return {
+            type: 'video',
+            content: payload.base64,
+            mimeType: `video/${payload.format}`,
+        };
+    },
+};
+/**
+ * Send SMS from a node (Android only)
+ */
+exports.nodeSmsSendTool = {
+    name: 'node_sms_send',
+    description: 'Send an SMS message using an Android mobile companion device. Not available on iOS.',
+    inputSchema: {
+        type: 'object',
+        properties: {
+            nodeId: {
+                type: 'string',
+                description: 'The ID or display name of the Android node to use. If omitted, uses the first Android node.',
+            },
+            to: {
+                type: 'string',
+                description: 'The phone number to send the SMS to.',
+            },
+            message: {
+                type: 'string',
+                description: 'The message content to send.',
+            },
+        },
+        required: ['to', 'message'],
+    },
+    riskLevel: 'write',
+    groups: ['write', 'network'],
+    handler: async (params) => {
+        const nodeManager = (0, control_plane_1.getNodeManager)();
+        if (!nodeManager.isAttached) {
+            return {
+                type: 'text',
+                content: 'Control Plane server is not running.',
+                isError: true,
+            };
+        }
+        // Find an Android node
+        let nodeId = params.nodeId;
+        if (!nodeId) {
+            const androidNodes = nodeManager.getNodesByPlatform('android');
+            if (androidNodes.length === 0) {
+                return {
+                    type: 'text',
+                    content: 'No Android mobile companion devices are connected. SMS is only available on Android.',
+                    isError: true,
+                };
+            }
+            nodeId = androidNodes[0].id;
+        }
+        // Check if node supports SMS
+        const node = nodeManager.getNode(nodeId);
+        if (!node) {
+            return {
+                type: 'text',
+                content: `Node not found: ${nodeId}`,
+                isError: true,
+            };
+        }
+        if (node.platform !== 'android') {
+            return {
+                type: 'text',
+                content: `SMS is only available on Android devices. Node "${node.displayName}" is ${node.platform}.`,
+                isError: true,
+            };
+        }
+        if (!node.capabilities.includes('sms')) {
+            return {
+                type: 'text',
+                content: `Node "${node.displayName}" does not have SMS capability.`,
+                isError: true,
+            };
+        }
+        // Invoke the SMS send command
+        const result = await nodeManager.smsSend(nodeId, {
+            to: params.to,
+            message: params.message,
+        });
+        if (!result.ok) {
+            return {
+                type: 'text',
+                content: `Failed to send SMS: ${result.error?.message || 'Unknown error'}`,
+                isError: true,
+            };
+        }
+        const payload = result.payload;
+        if (!payload.sent) {
+            return {
+                type: 'text',
+                content: `SMS was not sent: ${payload.error || 'Unknown error'}`,
+                isError: true,
+            };
+        }
+        return {
+            type: 'text',
+            content: `SMS sent successfully to ${params.to}`,
+        };
+    },
+};
+/**
+ * All node tools for registration
+ */
+exports.nodeTools = [
+    exports.nodeListTool,
+    exports.nodeDescribeTool,
+    exports.nodeCameraSnapTool,
+    exports.nodeLocationTool,
+    exports.nodeScreenRecordTool,
+    exports.nodeSmsSendTool,
+];
+/**
+ * Register node tools with the tool registry
+ */
+function registerNodeTools(registry) {
+    for (const tool of exports.nodeTools) {
+        registry.register(tool);
+    }
+}