@xiuchang-midscene/shared 2.0.2

package/dist/es/mcp/tool-generator.mjs

@@ -0,0 +1,297 @@
+import { parseBase64 } from "@xiuchang-midscene/shared/img";
+import { z } from "zod";
+import { getZodDescription, getZodTypeName } from "../zod-schema-utils.mjs";
+function getErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function describeActionForMCP(action) {
+    const actionDesc = action.description || `Execute ${action.name} action`;
+    if (!action.paramSchema) return `${action.name} action, ${actionDesc}`;
+    const schema = action.paramSchema;
+    const isZodObjectType = schema._def?.typeName === 'ZodObject';
+    if (!isZodObjectType || !schema.shape) {
+        const typeName = getZodTypeName(schema);
+        const description = getZodDescription(schema);
+        const paramDesc = description ? `${typeName} - ${description}` : typeName;
+        return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
+    }
+    const paramDescriptions = [];
+    for (const [key, field] of Object.entries(schema.shape))if (field && 'object' == typeof field) {
+        const isFieldOptional = 'function' == typeof field.isOptional && field.isOptional();
+        const typeName = getZodTypeName(field);
+        const description = getZodDescription(field);
+        let paramStr = `${key}${isFieldOptional ? '?' : ''} (${typeName})`;
+        if (description) paramStr += ` - ${description}`;
+        paramDescriptions.push(paramStr);
+    }
+    if (0 === paramDescriptions.length) return `${action.name} action, ${actionDesc}`;
+    return `${action.name} action, ${actionDesc}. Parameters: ${paramDescriptions.join('; ')}`;
+}
+function isZodOptional(value) {
+    return '_def' in value && value._def?.typeName === 'ZodOptional';
+}
+function isZodObject(value) {
+    return '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value;
+}
+function unwrapOptional(value) {
+    if (isZodOptional(value)) return {
+        innerValue: value._def.innerType,
+        isOptional: true
+    };
+    return {
+        innerValue: value,
+        isOptional: false
+    };
+}
+function isLocateField(value) {
+    if (!isZodObject(value)) return false;
+    return 'prompt' in value.shape;
+}
+function makePromptOptional(value, wrapInOptional) {
+    const newShape = {
+        ...value.shape
+    };
+    newShape.prompt = value.shape.prompt.optional();
+    let newSchema = z.object(newShape).passthrough();
+    if (wrapInOptional) newSchema = newSchema.optional();
+    return newSchema;
+}
+function transformSchemaField(key, value) {
+    const { innerValue, isOptional } = unwrapOptional(value);
+    if (isZodObject(innerValue) && isLocateField(innerValue)) return [
+        key,
+        makePromptOptional(innerValue, isOptional)
+    ];
+    return [
+        key,
+        value
+    ];
+}
+function extractActionSchema(paramSchema) {
+    if (!paramSchema) return {};
+    const schema = paramSchema;
+    if (!isZodObject(schema)) return schema;
+    return Object.fromEntries(Object.entries(schema.shape).map(([key, value])=>transformSchemaField(key, value)));
+}
+function serializeArgsToDescription(args) {
+    try {
+        return Object.entries(args).map(([key, value])=>{
+            if ('object' == typeof value && null !== value) try {
+                return `${key}: ${JSON.stringify(value)}`;
+            } catch  {
+                return `${key}: [object]`;
+            }
+            return `${key}: "${value}"`;
+        }).join(', ');
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error serializing args:', errorMessage);
+        return `[args serialization failed: ${errorMessage}]`;
+    }
+}
+function buildActionInstruction(actionName, args) {
+    const locatePrompt = args.locate && 'object' == typeof args.locate ? args.locate.prompt : void 0;
+    switch(actionName){
+        case 'Tap':
+            return locatePrompt ? `Tap on "${locatePrompt}"` : 'Tap';
+        case 'Input':
+            {
+                const value = args.value ?? args.content ?? '';
+                return locatePrompt ? `Input "${value}" into "${locatePrompt}"` : `Input "${value}"`;
+            }
+        case 'Scroll':
+            {
+                const direction = args.direction ?? 'down';
+                return locatePrompt ? `Scroll ${direction} on "${locatePrompt}"` : `Scroll ${direction}`;
+            }
+        case 'Hover':
+            return locatePrompt ? `Hover over "${locatePrompt}"` : 'Hover';
+        case 'KeyboardPress':
+            {
+                const key = args.value ?? args.key ?? '';
+                return `Press key "${key}"`;
+            }
+        default:
+            {
+                const argsDescription = serializeArgsToDescription(args);
+                return argsDescription ? `${actionName}: ${argsDescription}` : actionName;
+            }
+    }
+}
+async function captureScreenshotResult(agent, actionName) {
+    try {
+        const screenshot = await agent.page?.screenshotBase64();
+        if (!screenshot) return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                }
+            ]
+        };
+        const { mimeType, body } = parseBase64(screenshot);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                },
+                {
+                    type: 'image',
+                    data: body,
+                    mimeType
+                }
+            ]
+        };
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error capturing screenshot:', errorMessage);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
+                }
+            ]
+        };
+    }
+}
+function createErrorResult(message) {
+    return {
+        content: [
+            {
+                type: 'text',
+                text: message
+            }
+        ],
+        isError: true
+    };
+}
+async function captureFailureResult(agent, actionName, errorMessage) {
+    const warningText = `Warning: Action "${actionName}" failed: ${errorMessage}. Check the screenshot below for the current page state and decide how to proceed.`;
+    try {
+        const screenshot = await agent.page?.screenshotBase64();
+        if (!screenshot) return {
+            content: [
+                {
+                    type: 'text',
+                    text: warningText
+                }
+            ]
+        };
+        const { mimeType, body } = parseBase64(screenshot);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: warningText
+                },
+                {
+                    type: 'image',
+                    data: body,
+                    mimeType
+                }
+            ]
+        };
+    } catch  {
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: warningText
+                }
+            ]
+        };
+    }
+}
+function generateToolsFromActionSpace(actionSpace, getAgent) {
+    return actionSpace.map((action)=>{
+        const schema = extractActionSchema(action.paramSchema);
+        return {
+            name: action.name,
+            description: describeActionForMCP(action),
+            schema,
+            handler: async (args)=>{
+                try {
+                    const agent = await getAgent();
+                    if (agent.aiAction) {
+                        const instruction = buildActionInstruction(action.name, args);
+                        try {
+                            await agent.aiAction(instruction);
+                        } catch (error) {
+                            const errorMessage = getErrorMessage(error);
+                            console.error(`Error executing action "${action.name}":`, errorMessage);
+                            return await captureFailureResult(agent, action.name, errorMessage);
+                        }
+                    }
+                    return await captureScreenshotResult(agent, action.name);
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error(`Error in handler for "${action.name}":`, errorMessage);
+                    return createErrorResult(`Failed to get agent or execute action "${action.name}": ${errorMessage}`);
+                }
+            }
+        };
+    });
+}
+function generateCommonTools(getAgent) {
+    return [
+        {
+            name: 'take_screenshot',
+            description: 'Capture screenshot of current page/screen',
+            schema: {},
+            handler: async ()=>{
+                try {
+                    const agent = await getAgent();
+                    const screenshot = await agent.page?.screenshotBase64();
+                    if (!screenshot) return createErrorResult('Screenshot not available');
+                    const { mimeType, body } = parseBase64(screenshot);
+                    return {
+                        content: [
+                            {
+                                type: 'image',
+                                data: body,
+                                mimeType
+                            }
+                        ]
+                    };
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error taking screenshot:', errorMessage);
+                    return createErrorResult(`Failed to capture screenshot: ${errorMessage}`);
+                }
+            }
+        },
+        {
+            name: 'act',
+            description: 'Execute a natural language action. The AI will plan and perform multi-step operations in a single invocation, useful for transient UI interactions (e.g., Spotlight, dropdown menus) that disappear between separate commands.',
+            schema: {
+                prompt: z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"')
+            },
+            handler: async (args)=>{
+                const prompt = args.prompt;
+                try {
+                    const agent = await getAgent();
+                    if (!agent.aiAction) return createErrorResult('act is not supported by this agent');
+                    const result = await agent.aiAction(prompt, {
+                        deepThink: false
+                    });
+                    const screenshotResult = await captureScreenshotResult(agent, 'act');
+                    if (result) {
+                        const message = 'string' == typeof result ? result : JSON.stringify(result);
+                        screenshotResult.content.unshift({
+                            type: 'text',
+                            text: `Task finished, message: ${message}`
+                        });
+                    }
+                    return screenshotResult;
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error executing act:', errorMessage);
+                    return createErrorResult(`Failed to execute act: ${errorMessage}`);
+                }
+            }
+        }
+    ];
+}
+export { generateCommonTools, generateToolsFromActionSpace };

package/dist/es/mcp/tool-generator.mjs.bak

@@ -0,0 +1,297 @@
+import { parseBase64 } from "@midscene/shared/img";
+import { z } from "zod";
+import { getZodDescription, getZodTypeName } from "../zod-schema-utils.mjs";
+function getErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function describeActionForMCP(action) {
+    const actionDesc = action.description || `Execute ${action.name} action`;
+    if (!action.paramSchema) return `${action.name} action, ${actionDesc}`;
+    const schema = action.paramSchema;
+    const isZodObjectType = schema._def?.typeName === 'ZodObject';
+    if (!isZodObjectType || !schema.shape) {
+        const typeName = getZodTypeName(schema);
+        const description = getZodDescription(schema);
+        const paramDesc = description ? `${typeName} - ${description}` : typeName;
+        return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
+    }
+    const paramDescriptions = [];
+    for (const [key, field] of Object.entries(schema.shape))if (field && 'object' == typeof field) {
+        const isFieldOptional = 'function' == typeof field.isOptional && field.isOptional();
+        const typeName = getZodTypeName(field);
+        const description = getZodDescription(field);
+        let paramStr = `${key}${isFieldOptional ? '?' : ''} (${typeName})`;
+        if (description) paramStr += ` - ${description}`;
+        paramDescriptions.push(paramStr);
+    }
+    if (0 === paramDescriptions.length) return `${action.name} action, ${actionDesc}`;
+    return `${action.name} action, ${actionDesc}. Parameters: ${paramDescriptions.join('; ')}`;
+}
+function isZodOptional(value) {
+    return '_def' in value && value._def?.typeName === 'ZodOptional';
+}
+function isZodObject(value) {
+    return '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value;
+}
+function unwrapOptional(value) {
+    if (isZodOptional(value)) return {
+        innerValue: value._def.innerType,
+        isOptional: true
+    };
+    return {
+        innerValue: value,
+        isOptional: false
+    };
+}
+function isLocateField(value) {
+    if (!isZodObject(value)) return false;
+    return 'prompt' in value.shape;
+}
+function makePromptOptional(value, wrapInOptional) {
+    const newShape = {
+        ...value.shape
+    };
+    newShape.prompt = value.shape.prompt.optional();
+    let newSchema = z.object(newShape).passthrough();
+    if (wrapInOptional) newSchema = newSchema.optional();
+    return newSchema;
+}
+function transformSchemaField(key, value) {
+    const { innerValue, isOptional } = unwrapOptional(value);
+    if (isZodObject(innerValue) && isLocateField(innerValue)) return [
+        key,
+        makePromptOptional(innerValue, isOptional)
+    ];
+    return [
+        key,
+        value
+    ];
+}
+function extractActionSchema(paramSchema) {
+    if (!paramSchema) return {};
+    const schema = paramSchema;
+    if (!isZodObject(schema)) return schema;
+    return Object.fromEntries(Object.entries(schema.shape).map(([key, value])=>transformSchemaField(key, value)));
+}
+function serializeArgsToDescription(args) {
+    try {
+        return Object.entries(args).map(([key, value])=>{
+            if ('object' == typeof value && null !== value) try {
+                return `${key}: ${JSON.stringify(value)}`;
+            } catch  {
+                return `${key}: [object]`;
+            }
+            return `${key}: "${value}"`;
+        }).join(', ');
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error serializing args:', errorMessage);
+        return `[args serialization failed: ${errorMessage}]`;
+    }
+}
+function buildActionInstruction(actionName, args) {
+    const locatePrompt = args.locate && 'object' == typeof args.locate ? args.locate.prompt : void 0;
+    switch(actionName){
+        case 'Tap':
+            return locatePrompt ? `Tap on "${locatePrompt}"` : 'Tap';
+        case 'Input':
+            {
+                const value = args.value ?? args.content ?? '';
+                return locatePrompt ? `Input "${value}" into "${locatePrompt}"` : `Input "${value}"`;
+            }
+        case 'Scroll':
+            {
+                const direction = args.direction ?? 'down';
+                return locatePrompt ? `Scroll ${direction} on "${locatePrompt}"` : `Scroll ${direction}`;
+            }
+        case 'Hover':
+            return locatePrompt ? `Hover over "${locatePrompt}"` : 'Hover';
+        case 'KeyboardPress':
+            {
+                const key = args.value ?? args.key ?? '';
+                return `Press key "${key}"`;
+            }
+        default:
+            {
+                const argsDescription = serializeArgsToDescription(args);
+                return argsDescription ? `${actionName}: ${argsDescription}` : actionName;
+            }
+    }
+}
+async function captureScreenshotResult(agent, actionName) {
+    try {
+        const screenshot = await agent.page?.screenshotBase64();
+        if (!screenshot) return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                }
+            ]
+        };
+        const { mimeType, body } = parseBase64(screenshot);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                },
+                {
+                    type: 'image',
+                    data: body,
+                    mimeType
+                }
+            ]
+        };
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error capturing screenshot:', errorMessage);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
+                }
+            ]
+        };
+    }
+}
+function createErrorResult(message) {
+    return {
+        content: [
+            {
+                type: 'text',
+                text: message
+            }
+        ],
+        isError: true
+    };
+}
+async function captureFailureResult(agent, actionName, errorMessage) {
+    const warningText = `Warning: Action "${actionName}" failed: ${errorMessage}. Check the screenshot below for the current page state and decide how to proceed.`;
+    try {
+        const screenshot = await agent.page?.screenshotBase64();
+        if (!screenshot) return {
+            content: [
+                {
+                    type: 'text',
+                    text: warningText
+                }
+            ]
+        };
+        const { mimeType, body } = parseBase64(screenshot);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: warningText
+                },
+                {
+                    type: 'image',
+                    data: body,
+                    mimeType
+                }
+            ]
+        };
+    } catch  {
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: warningText
+                }
+            ]
+        };
+    }
+}
+function generateToolsFromActionSpace(actionSpace, getAgent) {
+    return actionSpace.map((action)=>{
+        const schema = extractActionSchema(action.paramSchema);
+        return {
+            name: action.name,
+            description: describeActionForMCP(action),
+            schema,
+            handler: async (args)=>{
+                try {
+                    const agent = await getAgent();
+                    if (agent.aiAction) {
+                        const instruction = buildActionInstruction(action.name, args);
+                        try {
+                            await agent.aiAction(instruction);
+                        } catch (error) {
+                            const errorMessage = getErrorMessage(error);
+                            console.error(`Error executing action "${action.name}":`, errorMessage);
+                            return await captureFailureResult(agent, action.name, errorMessage);
+                        }
+                    }
+                    return await captureScreenshotResult(agent, action.name);
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error(`Error in handler for "${action.name}":`, errorMessage);
+                    return createErrorResult(`Failed to get agent or execute action "${action.name}": ${errorMessage}`);
+                }
+            }
+        };
+    });
+}
+function generateCommonTools(getAgent) {
+    return [
+        {
+            name: 'take_screenshot',
+            description: 'Capture screenshot of current page/screen',
+            schema: {},
+            handler: async ()=>{
+                try {
+                    const agent = await getAgent();
+                    const screenshot = await agent.page?.screenshotBase64();
+                    if (!screenshot) return createErrorResult('Screenshot not available');
+                    const { mimeType, body } = parseBase64(screenshot);
+                    return {
+                        content: [
+                            {
+                                type: 'image',
+                                data: body,
+                                mimeType
+                            }
+                        ]
+                    };
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error taking screenshot:', errorMessage);
+                    return createErrorResult(`Failed to capture screenshot: ${errorMessage}`);
+                }
+            }
+        },
+        {
+            name: 'act',
+            description: 'Execute a natural language action. The AI will plan and perform multi-step operations in a single invocation, useful for transient UI interactions (e.g., Spotlight, dropdown menus) that disappear between separate commands.',
+            schema: {
+                prompt: z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"')
+            },
+            handler: async (args)=>{
+                const prompt = args.prompt;
+                try {
+                    const agent = await getAgent();
+                    if (!agent.aiAction) return createErrorResult('act is not supported by this agent');
+                    const result = await agent.aiAction(prompt, {
+                        deepThink: false
+                    });
+                    const screenshotResult = await captureScreenshotResult(agent, 'act');
+                    if (result) {
+                        const message = 'string' == typeof result ? result : JSON.stringify(result);
+                        screenshotResult.content.unshift({
+                            type: 'text',
+                            text: `Task finished, message: ${message}`
+                        });
+                    }
+                    return screenshotResult;
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error executing act:', errorMessage);
+                    return createErrorResult(`Failed to execute act: ${errorMessage}`);
+                }
+            }
+        }
+    ];
+}
+export { generateCommonTools, generateToolsFromActionSpace };

package/dist/es/mcp/types.mjs

@@ -0,0 +1,3 @@
+const defaultAppLoadingTimeoutMs = 10000;
+const defaultAppLoadingCheckIntervalMs = 2000;
+export { defaultAppLoadingCheckIntervalMs, defaultAppLoadingTimeoutMs };

package/dist/es/mcp/types.mjs.bak

@@ -0,0 +1,3 @@
+const defaultAppLoadingTimeoutMs = 10000;
+const defaultAppLoadingCheckIntervalMs = 2000;
+export { defaultAppLoadingCheckIntervalMs, defaultAppLoadingTimeoutMs };