@midscene/shared 1.6.1-beta-20260331083547.0 → 1.6.1

package/dist/es/constants/example-code.mjs

@@ -80,6 +80,23 @@ test('ai shop', async ({
 });
 `;
 const YAML_EXAMPLE_CODE = `
+CRITICAL - YAML Indentation Rules:
+For actions with additional parameters (aiScroll, aiInput, aiKeyboardPress), the parameters must be SIBLING keys at the SAME indentation level as the action key, NOT nested children indented further.
+CORRECT (parameters align with the action key):
+      - aiScroll:
+        direction: 'down'
+        scrollType: 'singleAction'
+        distance: 500
+        locate: "main content area"
+      - aiInput: 'text value'
+        locate: 'input field description'
+WRONG (parameters are indented further than the action key, DO NOT do this):
+      - aiScroll:
+          direction: 'down'
+          scrollType: 'singleAction'
+      - aiInput: 'text value'
+          locate: 'input field description'
+
 1. Format:
 
 web:
@@ -91,13 +108,18 @@ tasks:
   - name: "descriptive task name"
     flow:
       - aiTap: "element description"
+        xpath: '/html/body/div[1]/button[1]'
       - aiInput: 'text value'
         locate: 'input field description'
+        xpath: '/html/body/div[1]/input[1]'
       - aiScroll:
-        direction: down/up
-        scrollType: scrollToBottom/scrollToTop/singleAction
+        direction: 'down'
+        scrollType: 'singleAction'
+        distance: 500
+        locate: "scrollable area description"
+        xpath: '/html/body/div[1]/main[1]'
       - aiAssert: "expected state"
-      - sleep: milliseconds
+      - sleep: 1000
 
 2. Action Types:
 - aiTap: for clicks (natural language targeting)
@@ -106,24 +128,6 @@ tasks:
 - aiAssert: for validations
 - sleep: for delays (milliseconds)
 
-3. Best Practices:
-- Group related actions into logical tasks
-- Use natural language descriptions
-- Add deepLocate: true for complex interactions
-- Keep task names concise but descriptive
-
-4. CRITICAL - YAML Indentation Rules:
-- For actions with additional parameters (aiScroll, aiInput, aiKeyboardPress), the parameters must be SIBLING keys, NOT nested children
-- Parameters like direction, scrollType, locate must align with the action key, not indented further
-- CORRECT indentation example:
-      - aiScroll:
-        direction: down
-        scrollType: singleAction
-- WRONG indentation (DO NOT do this):
-      - aiScroll:
-          direction: down
-          scrollType: singleAction
-
 
 
 YAML type

package/dist/es/extractor/dom-util.mjs

@@ -81,12 +81,16 @@ function generateElementByPoint(center, description, edgeSize = 8) {
         description: description || ''
     };
 }
-function generateElementByRect(sourceRect, description, edgeSize = 8) {
+function generateElementByRect(sourceRect, description, _edgeSize = 8) {
     const centerX = sourceRect.left + Math.floor((sourceRect.width - 1) / 2);
     const centerY = sourceRect.top + Math.floor((sourceRect.height - 1) / 2);
-    return generateElementByPoint([
-        centerX,
-        centerY
-    ], description, edgeSize);
+    return {
+        rect: sourceRect,
+        center: [
+            centerX,
+            centerY
+        ],
+        description: description || ''
+    };
 }
 export { generateElementByPoint, generateElementByRect, isAElement, isButtonElement, isContainerElement, isFormElement, isImgElement, isNotContainerElement, isSvgElement, isTextElement };

package/dist/es/mcp/tool-generator.mjs

@@ -1,22 +1,21 @@
 import { parseBase64 } from "@midscene/shared/img";
 import { z } from "zod";
-import { getZodDescription, getZodTypeName } from "../zod-schema-utils.mjs";
+import { getZodDescription, getZodTypeName, isMidsceneLocatorField, unwrapZodField } from "../zod-schema-utils.mjs";
 function getErrorMessage(error) {
     return error instanceof Error ? error.message : String(error);
 }
 function describeActionForMCP(action) {
     const actionDesc = action.description || `Execute ${action.name} action`;
     if (!action.paramSchema) return `${action.name} action, ${actionDesc}`;
-    const schema = action.paramSchema;
-    const isZodObjectType = schema._def?.typeName === 'ZodObject';
-    if (!isZodObjectType || !schema.shape) {
-        const typeName = getZodTypeName(schema);
-        const description = getZodDescription(schema);
+    const shape = getZodObjectShape(action.paramSchema);
+    if (!shape) {
+        const typeName = getZodTypeName(action.paramSchema);
+        const description = getZodDescription(action.paramSchema);
         const paramDesc = description ? `${typeName} - ${description}` : typeName;
         return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
     }
     const paramDescriptions = [];
-    for (const [key, field] of Object.entries(schema.shape))if (field && 'object' == typeof field) {
+    for (const [key, field] of Object.entries(shape))if (field && 'object' == typeof field) {
         const isFieldOptional = 'function' == typeof field.isOptional && field.isOptional();
         const typeName = getZodTypeName(field);
         const description = getZodDescription(field);
@@ -30,9 +29,6 @@ function describeActionForMCP(action) {
 function isZodOptional(value) {
     return '_def' in value && value._def?.typeName === 'ZodOptional';
 }
-function isZodObject(value) {
-    return '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value;
-}
 function unwrapOptional(value) {
     if (isZodOptional(value)) return {
         innerValue: value._def.innerType,
@@ -43,24 +39,31 @@ function unwrapOptional(value) {
         isOptional: false
     };
 }
-function isLocateField(value) {
-    if (!isZodObject(value)) return false;
-    return 'prompt' in value.shape;
+function getZodObjectShape(value) {
+    if (!value) return;
+    const actualValue = unwrapZodField(value);
+    if (actualValue._def?.typeName !== 'ZodObject') return;
+    if ('function' == typeof actualValue._def.shape) return actualValue._def.shape();
+    return actualValue.shape;
+}
+function isRecord(value) {
+    return 'object' == typeof value && null !== value && !Array.isArray(value);
 }
-function makePromptOptional(value, wrapInOptional) {
+function makePromptOptional(shape, wrapInOptional) {
     const newShape = {
-        ...value.shape
+        ...shape
     };
-    newShape.prompt = value.shape.prompt.optional();
+    newShape.prompt = shape.prompt.optional();
     let newSchema = z.object(newShape).passthrough();
     if (wrapInOptional) newSchema = newSchema.optional();
     return newSchema;
 }
 function transformSchemaField(key, value) {
     const { innerValue, isOptional } = unwrapOptional(value);
-    if (isZodObject(innerValue) && isLocateField(innerValue)) return [
+    const shape = getZodObjectShape(innerValue);
+    if (shape && isMidsceneLocatorField(innerValue)) return [
         key,
-        makePromptOptional(innerValue, isOptional)
+        makePromptOptional(shape, isOptional)
     ];
     return [
         key,
@@ -69,9 +72,66 @@ function transformSchemaField(key, value) {
 }
 function extractActionSchema(paramSchema) {
     if (!paramSchema) return {};
-    const schema = paramSchema;
-    if (!isZodObject(schema)) return schema;
-    return Object.fromEntries(Object.entries(schema.shape).map(([key, value])=>transformSchemaField(key, value)));
+    const shape = getZodObjectShape(paramSchema);
+    if (!shape) return paramSchema;
+    return Object.fromEntries(Object.entries(shape).map(([key, value])=>transformSchemaField(key, value)));
+}
+function getPromptText(prompt) {
+    if ('string' == typeof prompt) return prompt;
+    if (isRecord(prompt) && 'string' == typeof prompt.prompt) return prompt.prompt;
+}
+function moveLocateExtrasIntoPrompt(value, locateFieldKeys) {
+    const promptText = getPromptText(value.prompt);
+    if (!promptText) return value;
+    const normalizedPrompt = isRecord(value.prompt) ? {
+        ...value.prompt
+    } : {
+        prompt: promptText
+    };
+    const normalizedLocate = {};
+    let movedExtraField = false;
+    for (const [key, fieldValue] of Object.entries(value))if ('prompt' !== key) {
+        if (locateFieldKeys.has(key)) {
+            normalizedLocate[key] = fieldValue;
+            continue;
+        }
+        movedExtraField = true;
+        if (!(key in normalizedPrompt)) normalizedPrompt[key] = fieldValue;
+    }
+    if (!movedExtraField) return value;
+    return {
+        ...normalizedLocate,
+        prompt: normalizedPrompt
+    };
+}
+function normalizeLocateLikeArg(value, fieldSchema) {
+    if ('string' == typeof value) return {
+        prompt: value
+    };
+    if (!isRecord(value)) return value;
+    const shape = getZodObjectShape(fieldSchema);
+    if (!shape) return value;
+    return moveLocateExtrasIntoPrompt(value, new Set(Object.keys(shape)));
+}
+function normalizeActionArgs(args, paramSchema) {
+    if (!paramSchema) return args;
+    const shape = getZodObjectShape(paramSchema);
+    if (!shape) return args;
+    return Object.fromEntries(Object.entries(args).map(([key, value])=>{
+        const fieldSchema = shape[key];
+        if (!fieldSchema) return [
+            key,
+            value
+        ];
+        if (isMidsceneLocatorField(fieldSchema)) return [
+            key,
+            normalizeLocateLikeArg(value, fieldSchema)
+        ];
+        return [
+            key,
+            value
+        ];
+    }));
 }
 function serializeArgsToDescription(args) {
     try {
@@ -90,7 +150,7 @@ function serializeArgsToDescription(args) {
     }
 }
 function buildActionInstruction(actionName, args) {
-    const locatePrompt = args.locate && 'object' == typeof args.locate ? args.locate.prompt : void 0;
+    const locatePrompt = isRecord(args.locate) ? getPromptText(args.locate.prompt) : void 0;
     switch(actionName){
         case 'Tap':
             return locatePrompt ? `Tap on "${locatePrompt}"` : 'Tap';
@@ -118,44 +178,59 @@ function buildActionInstruction(actionName, args) {
             }
     }
 }
-async function captureScreenshotResult(agent, actionName) {
+async function executeAction(agent, actionName, args) {
+    if (agent.callActionInActionSpace) return agent.callActionInActionSpace(actionName, args);
+    if (agent.aiAction) {
+        const instruction = buildActionInstruction(actionName, args);
+        return agent.aiAction(instruction);
+    }
+    throw new Error(`Action "${actionName}" is not supported by this agent`);
+}
+async function captureScreenshotResult(agent, actionName, actionResult) {
+    const content = [
+        {
+            type: 'text',
+            text: `Action "${actionName}" completed.`
+        }
+    ];
+    if (void 0 !== actionResult) content.push({
+        type: 'text',
+        text: `Result: ${serializeActionResult(actionResult)}`
+    });
     try {
         const screenshot = await agent.page?.screenshotBase64();
         if (!screenshot) return {
-            content: [
-                {
-                    type: 'text',
-                    text: `Action "${actionName}" completed.`
-                }
-            ]
+            content
         };
         const { mimeType, body } = parseBase64(screenshot);
+        content.push({
+            type: 'image',
+            data: body,
+            mimeType
+        });
         return {
-            content: [
-                {
-                    type: 'text',
-                    text: `Action "${actionName}" completed.`
-                },
-                {
-                    type: 'image',
-                    data: body,
-                    mimeType
-                }
-            ]
+            content
         };
     } catch (error) {
         const errorMessage = getErrorMessage(error);
         console.error('Error capturing screenshot:', errorMessage);
+        content[0] = {
+            type: 'text',
+            text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
+        };
         return {
-            content: [
-                {
-                    type: 'text',
-                    text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
-                }
-            ]
+            content
         };
     }
 }
+function serializeActionResult(actionResult) {
+    if ('string' == typeof actionResult) return actionResult;
+    try {
+        return JSON.stringify(actionResult);
+    } catch  {
+        return String(actionResult);
+    }
+}
 function createErrorResult(message) {
     return {
         content: [
@@ -214,17 +289,16 @@ function generateToolsFromActionSpace(actionSpace, getAgent) {
             handler: async (args)=>{
                 try {
                     const agent = await getAgent();
-                    if (agent.aiAction) {
-                        const instruction = buildActionInstruction(action.name, args);
-                        try {
-                            await agent.aiAction(instruction);
-                        } catch (error) {
-                            const errorMessage = getErrorMessage(error);
-                            console.error(`Error executing action "${action.name}":`, errorMessage);
-                            return await captureFailureResult(agent, action.name, errorMessage);
-                        }
+                    const normalizedArgs = normalizeActionArgs(args, action.paramSchema);
+                    let actionResult;
+                    try {
+                        actionResult = await executeAction(agent, action.name, normalizedArgs);
+                    } catch (error) {
+                        const errorMessage = getErrorMessage(error);
+                        console.error(`Error executing action "${action.name}":`, errorMessage);
+                        return await captureFailureResult(agent, action.name, errorMessage);
                     }
-                    return await captureScreenshotResult(agent, action.name);
+                    return await captureScreenshotResult(agent, action.name, actionResult);
                 } catch (error) {
                     const errorMessage = getErrorMessage(error);
                     console.error(`Error in handler for "${action.name}":`, errorMessage);