@midscene/shared 1.6.1-beta-20260331083547.0 → 1.6.1

package/dist/lib/mcp/tool-generator.js

@@ -36,16 +36,15 @@ function getErrorMessage(error) {
 function describeActionForMCP(action) {
     const actionDesc = action.description || `Execute ${action.name} action`;
     if (!action.paramSchema) return `${action.name} action, ${actionDesc}`;
-    const schema = action.paramSchema;
-    const isZodObjectType = schema._def?.typeName === 'ZodObject';
-    if (!isZodObjectType || !schema.shape) {
-        const typeName = (0, external_zod_schema_utils_js_namespaceObject.getZodTypeName)(schema);
-        const description = (0, external_zod_schema_utils_js_namespaceObject.getZodDescription)(schema);
+    const shape = getZodObjectShape(action.paramSchema);
+    if (!shape) {
+        const typeName = (0, external_zod_schema_utils_js_namespaceObject.getZodTypeName)(action.paramSchema);
+        const description = (0, external_zod_schema_utils_js_namespaceObject.getZodDescription)(action.paramSchema);
         const paramDesc = description ? `${typeName} - ${description}` : typeName;
         return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
     }
     const paramDescriptions = [];
-    for (const [key, field] of Object.entries(schema.shape))if (field && 'object' == typeof field) {
+    for (const [key, field] of Object.entries(shape))if (field && 'object' == typeof field) {
         const isFieldOptional = 'function' == typeof field.isOptional && field.isOptional();
         const typeName = (0, external_zod_schema_utils_js_namespaceObject.getZodTypeName)(field);
         const description = (0, external_zod_schema_utils_js_namespaceObject.getZodDescription)(field);
@@ -59,9 +58,6 @@ function describeActionForMCP(action) {
 function isZodOptional(value) {
     return '_def' in value && value._def?.typeName === 'ZodOptional';
 }
-function isZodObject(value) {
-    return '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value;
-}
 function unwrapOptional(value) {
     if (isZodOptional(value)) return {
         innerValue: value._def.innerType,
@@ -72,24 +68,31 @@ function unwrapOptional(value) {
         isOptional: false
     };
 }
-function isLocateField(value) {
-    if (!isZodObject(value)) return false;
-    return 'prompt' in value.shape;
+function getZodObjectShape(value) {
+    if (!value) return;
+    const actualValue = (0, external_zod_schema_utils_js_namespaceObject.unwrapZodField)(value);
+    if (actualValue._def?.typeName !== 'ZodObject') return;
+    if ('function' == typeof actualValue._def.shape) return actualValue._def.shape();
+    return actualValue.shape;
+}
+function isRecord(value) {
+    return 'object' == typeof value && null !== value && !Array.isArray(value);
 }
-function makePromptOptional(value, wrapInOptional) {
+function makePromptOptional(shape, wrapInOptional) {
     const newShape = {
-        ...value.shape
+        ...shape
     };
-    newShape.prompt = value.shape.prompt.optional();
+    newShape.prompt = shape.prompt.optional();
     let newSchema = external_zod_namespaceObject.z.object(newShape).passthrough();
     if (wrapInOptional) newSchema = newSchema.optional();
     return newSchema;
 }
 function transformSchemaField(key, value) {
     const { innerValue, isOptional } = unwrapOptional(value);
-    if (isZodObject(innerValue) && isLocateField(innerValue)) return [
+    const shape = getZodObjectShape(innerValue);
+    if (shape && (0, external_zod_schema_utils_js_namespaceObject.isMidsceneLocatorField)(innerValue)) return [
         key,
-        makePromptOptional(innerValue, isOptional)
+        makePromptOptional(shape, isOptional)
     ];
     return [
         key,
@@ -98,9 +101,66 @@ function transformSchemaField(key, value) {
 }
 function extractActionSchema(paramSchema) {
     if (!paramSchema) return {};
-    const schema = paramSchema;
-    if (!isZodObject(schema)) return schema;
-    return Object.fromEntries(Object.entries(schema.shape).map(([key, value])=>transformSchemaField(key, value)));
+    const shape = getZodObjectShape(paramSchema);
+    if (!shape) return paramSchema;
+    return Object.fromEntries(Object.entries(shape).map(([key, value])=>transformSchemaField(key, value)));
+}
+function getPromptText(prompt) {
+    if ('string' == typeof prompt) return prompt;
+    if (isRecord(prompt) && 'string' == typeof prompt.prompt) return prompt.prompt;
+}
+function moveLocateExtrasIntoPrompt(value, locateFieldKeys) {
+    const promptText = getPromptText(value.prompt);
+    if (!promptText) return value;
+    const normalizedPrompt = isRecord(value.prompt) ? {
+        ...value.prompt
+    } : {
+        prompt: promptText
+    };
+    const normalizedLocate = {};
+    let movedExtraField = false;
+    for (const [key, fieldValue] of Object.entries(value))if ('prompt' !== key) {
+        if (locateFieldKeys.has(key)) {
+            normalizedLocate[key] = fieldValue;
+            continue;
+        }
+        movedExtraField = true;
+        if (!(key in normalizedPrompt)) normalizedPrompt[key] = fieldValue;
+    }
+    if (!movedExtraField) return value;
+    return {
+        ...normalizedLocate,
+        prompt: normalizedPrompt
+    };
+}
+function normalizeLocateLikeArg(value, fieldSchema) {
+    if ('string' == typeof value) return {
+        prompt: value
+    };
+    if (!isRecord(value)) return value;
+    const shape = getZodObjectShape(fieldSchema);
+    if (!shape) return value;
+    return moveLocateExtrasIntoPrompt(value, new Set(Object.keys(shape)));
+}
+function normalizeActionArgs(args, paramSchema) {
+    if (!paramSchema) return args;
+    const shape = getZodObjectShape(paramSchema);
+    if (!shape) return args;
+    return Object.fromEntries(Object.entries(args).map(([key, value])=>{
+        const fieldSchema = shape[key];
+        if (!fieldSchema) return [
+            key,
+            value
+        ];
+        if ((0, external_zod_schema_utils_js_namespaceObject.isMidsceneLocatorField)(fieldSchema)) return [
+            key,
+            normalizeLocateLikeArg(value, fieldSchema)
+        ];
+        return [
+            key,
+            value
+        ];
+    }));
 }
 function serializeArgsToDescription(args) {
     try {
@@ -119,7 +179,7 @@ function serializeArgsToDescription(args) {
     }
 }
 function buildActionInstruction(actionName, args) {
-    const locatePrompt = args.locate && 'object' == typeof args.locate ? args.locate.prompt : void 0;
+    const locatePrompt = isRecord(args.locate) ? getPromptText(args.locate.prompt) : void 0;
     switch(actionName){
         case 'Tap':
             return locatePrompt ? `Tap on "${locatePrompt}"` : 'Tap';
@@ -147,44 +207,59 @@ function buildActionInstruction(actionName, args) {
             }
     }
 }
-async function captureScreenshotResult(agent, actionName) {
+async function executeAction(agent, actionName, args) {
+    if (agent.callActionInActionSpace) return agent.callActionInActionSpace(actionName, args);
+    if (agent.aiAction) {
+        const instruction = buildActionInstruction(actionName, args);
+        return agent.aiAction(instruction);
+    }
+    throw new Error(`Action "${actionName}" is not supported by this agent`);
+}
+async function captureScreenshotResult(agent, actionName, actionResult) {
+    const content = [
+        {
+            type: 'text',
+            text: `Action "${actionName}" completed.`
+        }
+    ];
+    if (void 0 !== actionResult) content.push({
+        type: 'text',
+        text: `Result: ${serializeActionResult(actionResult)}`
+    });
     try {
         const screenshot = await agent.page?.screenshotBase64();
         if (!screenshot) return {
-            content: [
-                {
-                    type: 'text',
-                    text: `Action "${actionName}" completed.`
-                }
-            ]
+            content
         };
         const { mimeType, body } = (0, img_namespaceObject.parseBase64)(screenshot);
+        content.push({
+            type: 'image',
+            data: body,
+            mimeType
+        });
         return {
-            content: [
-                {
-                    type: 'text',
-                    text: `Action "${actionName}" completed.`
-                },
-                {
-                    type: 'image',
-                    data: body,
-                    mimeType
-                }
-            ]
+            content
         };
     } catch (error) {
         const errorMessage = getErrorMessage(error);
         console.error('Error capturing screenshot:', errorMessage);
+        content[0] = {
+            type: 'text',
+            text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
+        };
         return {
-            content: [
-                {
-                    type: 'text',
-                    text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
-                }
-            ]
+            content
         };
     }
 }
+function serializeActionResult(actionResult) {
+    if ('string' == typeof actionResult) return actionResult;
+    try {
+        return JSON.stringify(actionResult);
+    } catch  {
+        return String(actionResult);
+    }
+}
 function createErrorResult(message) {
     return {
         content: [
@@ -243,17 +318,16 @@ function generateToolsFromActionSpace(actionSpace, getAgent) {
             handler: async (args)=>{
                 try {
                     const agent = await getAgent();
-                    if (agent.aiAction) {
-                        const instruction = buildActionInstruction(action.name, args);
-                        try {
-                            await agent.aiAction(instruction);
-                        } catch (error) {
-                            const errorMessage = getErrorMessage(error);
-                            console.error(`Error executing action "${action.name}":`, errorMessage);
-                            return await captureFailureResult(agent, action.name, errorMessage);
-                        }
+                    const normalizedArgs = normalizeActionArgs(args, action.paramSchema);
+                    let actionResult;
+                    try {
+                        actionResult = await executeAction(agent, action.name, normalizedArgs);
+                    } catch (error) {
+                        const errorMessage = getErrorMessage(error);
+                        console.error(`Error executing action "${action.name}":`, errorMessage);
+                        return await captureFailureResult(agent, action.name, errorMessage);
                     }
-                    return await captureScreenshotResult(agent, action.name);
+                    return await captureScreenshotResult(agent, action.name, actionResult);
                 } catch (error) {
                     const errorMessage = getErrorMessage(error);
                     console.error(`Error in handler for "${action.name}":`, errorMessage);