@midscene/core 1.12.10-beta-20260920031616.0 → 1.13.0

package/dist/lib/agent/test-runner-nodes.js

@@ -24,24 +24,38 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
+    insightOptionsInputSchema: ()=>insightOptionsInputSchema,
+    javascriptInputSchema: ()=>javascriptInputSchema,
     aiActOptionsInputSchema: ()=>aiActOptionsInputSchema,
     aiTapInputSchema: ()=>aiTapInputSchema,
-    reportScreenshotInputSchema: ()=>reportScreenshotInputSchema,
+    aiScrollInputSchema: ()=>aiScrollInputSchema,
     createAgentTestRunnerNodeDefinition: ()=>createAgentTestRunnerNodeDefinition,
+    aiInputInputSchema: ()=>aiInputInputSchema,
+    reportScreenshotInputSchema: ()=>reportScreenshotInputSchema,
     insightInputSchema: ()=>insightInputSchema,
-    locateOptionsInputSchema: ()=>locateOptionsInputSchema,
-    structuredUserPromptInputSchema: ()=>structuredUserPromptInputSchema,
-    promptImageInputSchema: ()=>promptImageInputSchema,
+    aiKeyboardPressInputSchema: ()=>aiKeyboardPressInputSchema,
+    aiPinchInputSchema: ()=>aiPinchInputSchema,
+    aiQueryInputSchema: ()=>aiQueryInputSchema,
+    aiLongPressInputSchema: ()=>aiLongPressInputSchema,
+    actionInputSchema: ()=>actionInputSchema,
     aiAssertInputSchema: ()=>aiAssertInputSchema,
     commonAgentTestRunnerNodeDefinitions: ()=>commonAgentTestRunnerNodeDefinitions,
+    locateOptionsInputSchema: ()=>locateOptionsInputSchema,
+    promptImageInputSchema: ()=>promptImageInputSchema,
     recordToReportInputSchema: ()=>recordToReportInputSchema,
     recordToReportOptionsInputSchema: ()=>recordToReportOptionsInputSchema,
+    runGherkinScenarioInputSchema: ()=>runGherkinScenarioInputSchema,
+    structuredUserPromptInputSchema: ()=>structuredUserPromptInputSchema,
+    aiWaitForInputSchema: ()=>aiWaitForInputSchema,
     userPromptInputSchema: ()=>userPromptInputSchema,
-    aiAssertOptionsInputSchema: ()=>aiAssertOptionsInputSchema,
+    aiDragAndDropInputSchema: ()=>aiDragAndDropInputSchema,
     aiActInputSchema: ()=>aiActInputSchema,
-    insightOptionsInputSchema: ()=>insightOptionsInputSchema
+    aiAssertOptionsInputSchema: ()=>aiAssertOptionsInputSchema
 });
 const v4_namespaceObject = require("zod/v4");
+const input_strategy_js_namespaceObject = require("../device/input-strategy.js");
+const errors_js_namespaceObject = require("../test-runner/errors.js");
+const utils_js_namespaceObject = require("../yaml/utils.js");
 const createAgentTestRunnerNodeDefinition = (agentLabel = 'an Agent')=>(definition)=>({
             name: definition.name ?? definition.method,
             ...void 0 === definition.title ? {} : {
@@ -78,6 +92,11 @@ const userPromptInputSchema = v4_namespaceObject.z.union([
     structuredUserPromptInputSchema
 ]);
 const aiActOptionsInputSchema = v4_namespaceObject.z.strictObject({
+    effort: v4_namespaceObject.z["enum"]([
+        'fast',
+        'balance',
+        'deepThink'
+    ]).optional().describe('Action planning effort: fast, balanced, or deeper reasoning.'),
     cacheable: v4_namespaceObject.z.boolean().optional().describe('Whether this action may use the Midscene cache.'),
     fileChooserAccept: v4_namespaceObject.z.union([
         v4_namespaceObject.z.string(),
@@ -126,6 +145,100 @@ const aiTapInputSchema = v4_namespaceObject.z.strictObject({
     prompt: userPromptInputSchema,
     options: locateOptionsInputSchema.optional()
 });
+const aiInputInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema,
+    value: v4_namespaceObject.z.union([
+        v4_namespaceObject.z.string(),
+        v4_namespaceObject.z.number()
+    ]),
+    options: locateOptionsInputSchema.extend({
+        mode: v4_namespaceObject.z["enum"]([
+            'replace',
+            'clear',
+            'typeOnly',
+            'append'
+        ]).optional(),
+        autoDismissKeyboard: v4_namespaceObject.z.boolean().optional(),
+        keyboardTypeDelay: v4_namespaceObject.z.number().nonnegative().optional(),
+        inputStrategy: v4_namespaceObject.z["enum"](input_strategy_js_namespaceObject.inputStrategies).optional()
+    }).optional()
+});
+const aiKeyboardPressInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema.optional(),
+    keyName: nonBlankText('The key or key combination to press.'),
+    options: locateOptionsInputSchema.optional()
+});
+const aiScrollInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema.optional(),
+    options: locateOptionsInputSchema.extend({
+        direction: v4_namespaceObject.z["enum"]([
+            'down',
+            'up',
+            'right',
+            'left'
+        ]).optional(),
+        scrollType: v4_namespaceObject.z["enum"]([
+            'singleAction',
+            'scrollToBottom',
+            'scrollToTop',
+            'scrollToRight',
+            'scrollToLeft',
+            'once',
+            'untilBottom',
+            'untilTop',
+            'untilRight',
+            'untilLeft'
+        ]).optional(),
+        distance: v4_namespaceObject.z.number().nullable().optional()
+    }).optional()
+});
+const aiPinchInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema.optional(),
+    direction: v4_namespaceObject.z["enum"]([
+        'in',
+        'out'
+    ]),
+    options: locateOptionsInputSchema.extend({
+        distance: v4_namespaceObject.z.number().optional(),
+        duration: v4_namespaceObject.z.number().nonnegative().optional()
+    }).optional()
+});
+const aiLongPressInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema,
+    options: locateOptionsInputSchema.extend({
+        duration: v4_namespaceObject.z.number().nonnegative().optional()
+    }).optional()
+});
+const aiDragAndDropInputSchema = v4_namespaceObject.z.strictObject({
+    from: userPromptInputSchema,
+    to: userPromptInputSchema,
+    options: locateOptionsInputSchema.optional()
+});
+const aiQueryInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: v4_namespaceObject.z.union([
+        nonBlankText('The extraction request.'),
+        v4_namespaceObject.z.record(v4_namespaceObject.z.string(), v4_namespaceObject.z.string())
+    ]),
+    options: insightOptionsInputSchema.optional()
+});
+const aiWaitForInputSchema = v4_namespaceObject.z.strictObject({
+    prompt: userPromptInputSchema,
+    options: insightOptionsInputSchema.extend({
+        timeoutMs: v4_namespaceObject.z.number().positive().optional(),
+        checkIntervalMs: v4_namespaceObject.z.number().positive().optional()
+    }).optional()
+});
+const javascriptInputSchema = v4_namespaceObject.z.strictObject({
+    script: nonBlankText('JavaScript to evaluate through the current interface.')
+});
+const runGherkinScenarioInputSchema = v4_namespaceObject.z.strictObject({
+    scenario: nonBlankText('The Gherkin scenario to execute.'),
+    options: aiActOptionsInputSchema.optional()
+});
+const actionInputSchema = v4_namespaceObject.z.strictObject({
+    name: nonBlankText('The ActionSpace action name, for example DragAndDrop.'),
+    params: v4_namespaceObject.z.unknown().optional()
+});
 const insightInputSchema = v4_namespaceObject.z.strictObject({
     prompt: userPromptInputSchema,
     options: insightOptionsInputSchema.optional()
@@ -149,7 +262,11 @@ const recordToReportInputSchema = v4_namespaceObject.z.strictObject({
     options: recordToReportOptionsInputSchema.optional()
 });
 const defineCommonAgentNode = createAgentTestRunnerNodeDefinition('an Agent');
+const defineAgentActionNode = createAgentTestRunnerNodeDefinition('an Agent');
 const promptText = (prompt)=>'string' == typeof prompt ? prompt : prompt.prompt;
+const valueResult = (value)=>({
+        data: value
+    });
 const aiActNode = defineCommonAgentNode({
     method: 'aiAct',
     description: 'Perform a natural-language task with a Midscene UI Agent.',
@@ -197,14 +314,15 @@ const aiAssertNode = defineCommonAgentNode({
             input.prompt,
             input.message,
             {
+                keepRawResponse: true,
                 ...input.options,
                 abortSignal: context.signal
             }
         ];
     },
     toResult (output, input) {
-        return {
-            summary: `Assertion passed: ${promptText(input.prompt)}`,
+        const result = {
+            summary: `Assertion ${output?.pass === false ? 'failed' : 'passed'}: ${promptText(input.prompt)}`,
             ...void 0 === output ? {} : {
                 data: {
                     pass: output.pass,
@@ -217,74 +335,201 @@ const aiAssertNode = defineCommonAgentNode({
                 }
             }
         };
+        if (output?.pass === false) throw new errors_js_namespaceObject.NodeExecutionError('aiAssert', new Error(output.message || input.message || output.thought || `Assertion failed: ${promptText(input.prompt)}`), result);
+        return result;
     }
 });
-const insightNode = (method)=>{
-    if ('aiBoolean' === method) return defineCommonAgentNode({
+const locateActionNode = (method)=>defineAgentActionNode({
         method,
-        description: `Run ${method} with a Midscene UI Agent and store its value.`,
+        description: {
+            aiHover: 'Locate and hover over an element with a Midscene UI Agent.',
+            aiDoubleClick: 'Locate and double-click an element with a Midscene UI Agent.',
+            aiRightClick: 'Locate and right-click an element with a Midscene UI Agent.',
+            aiClearInput: 'Locate an input element and clear its value with a Midscene UI Agent.'
+        }[method],
         stringInputKey: 'prompt',
-        inputSchema: insightInputSchema,
+        inputSchema: aiTapInputSchema,
         toArgs: (input)=>[
                 input.prompt,
                 input.options
             ],
-        toResult: (value)=>({
-                summary: `${method} returned ${value}`,
-                data: {
-                    value
-                }
+        toResult: (_output, input)=>({
+                summary: `${method}: ${promptText(input.prompt)}`
             })
     });
-    if ('aiNumber' === method) return defineCommonAgentNode({
-        method,
-        description: `Run ${method} with a Midscene UI Agent and store its value.`,
+const additionalAgentNodes = [
+    locateActionNode('aiHover'),
+    locateActionNode('aiDoubleClick'),
+    locateActionNode('aiRightClick'),
+    locateActionNode('aiClearInput'),
+    defineAgentActionNode({
+        method: 'aiInput',
+        description: 'Locate an input element and enter a value with a Midscene UI Agent.',
+        inputSchema: aiInputInputSchema,
+        toArgs: (input)=>[
+                input.prompt,
+                {
+                    ...input.options,
+                    value: input.value
+                }
+            ]
+    }),
+    defineAgentActionNode({
+        method: 'aiKeyboardPress',
+        description: 'Press a key or key combination, optionally targeting a located element.',
+        stringInputKey: 'keyName',
+        inputSchema: aiKeyboardPressInputSchema,
+        toArgs: (input)=>[
+                input.prompt,
+                {
+                    ...input.options,
+                    keyName: input.keyName
+                }
+            ]
+    }),
+    defineAgentActionNode({
+        method: 'aiScroll',
+        description: 'Scroll the page or a located region with a Midscene UI Agent.',
         stringInputKey: 'prompt',
-        inputSchema: insightInputSchema,
+        inputSchema: aiScrollInputSchema,
         toArgs: (input)=>[
                 input.prompt,
-                input.options
-            ],
-        toResult: (value)=>({
-                summary: `${method} returned ${value}`,
-                data: {
-                    value
+                {
+                    ...input.options
                 }
-            })
-    });
-    if ('aiString' === method) return defineCommonAgentNode({
-        method,
-        description: `Run ${method} with a Midscene UI Agent and store its value.`,
+            ]
+    }),
+    defineAgentActionNode({
+        method: 'aiPinch',
+        description: 'Perform a pinch-in or pinch-out gesture with a Midscene UI Agent.',
+        inputSchema: aiPinchInputSchema,
+        toArgs: (input)=>[
+                input.prompt,
+                {
+                    ...input.options,
+                    direction: input.direction
+                }
+            ]
+    }),
+    defineAgentActionNode({
+        method: 'aiLongPress',
+        description: 'Locate and long-press an element with a Midscene UI Agent.',
         stringInputKey: 'prompt',
-        inputSchema: insightInputSchema,
+        inputSchema: aiLongPressInputSchema,
         toArgs: (input)=>[
                 input.prompt,
                 input.options
+            ]
+    }),
+    defineAgentActionNode({
+        method: 'callActionInActionSpace',
+        name: 'aiDragAndDrop',
+        description: 'Locate source and destination elements, then drag the source to the destination.',
+        inputSchema: aiDragAndDropInputSchema,
+        toArgs: (input)=>[
+                'DragAndDrop',
+                {
+                    from: (0, utils_js_namespaceObject.buildDetailedLocateParam)(input.from, input.options),
+                    to: (0, utils_js_namespaceObject.buildDetailedLocateParam)(input.to, input.options)
+                }
+            ]
+    }),
+    defineAgentActionNode({
+        method: 'aiLocate',
+        description: "Locate an element from a natural-language description and store the result.",
+        stringInputKey: 'prompt',
+        inputSchema: aiTapInputSchema,
+        toArgs: (input, context)=>[
+                input.prompt,
+                {
+                    ...input.options,
+                    abortSignal: context.signal
+                }
             ],
-        toResult: (value)=>({
-                summary: `${method} returned ${JSON.stringify(value)}`,
-                data: {
-                    value
+        toResult: valueResult
+    }),
+    defineAgentActionNode({
+        method: 'aiQuery',
+        description: 'Extract structured data from the current interface and store the result.',
+        stringInputKey: 'prompt',
+        inputSchema: aiQueryInputSchema,
+        toArgs: (input, context)=>[
+                input.prompt,
+                {
+                    ...input.options,
+                    abortSignal: context.signal
                 }
-            })
-    });
-    return defineCommonAgentNode({
+            ],
+        toResult: valueResult
+    }),
+    defineAgentActionNode({
+        method: 'aiWaitFor',
+        description: 'Wait until a natural-language condition is satisfied.',
+        stringInputKey: 'prompt',
+        inputSchema: aiWaitForInputSchema,
+        toArgs: (input, context)=>[
+                input.prompt,
+                {
+                    ...input.options,
+                    abortSignal: context.signal
+                }
+            ]
+    }),
+    defineAgentActionNode({
+        method: 'evaluateJavaScript',
+        name: "javascript",
+        description: 'Evaluate JavaScript in the current interface and store the result.',
+        stringInputKey: "script",
+        inputSchema: javascriptInputSchema,
+        toArgs: (input)=>[
+                input.script
+            ],
+        toResult: valueResult
+    }),
+    defineAgentActionNode({
+        method: 'runGherkinScenario',
+        description: 'Execute a Gherkin scenario with the current Midscene UI Agent.',
+        stringInputKey: 'scenario',
+        inputSchema: runGherkinScenarioInputSchema,
+        toArgs: (input, context)=>[
+                input.scenario,
+                {
+                    ...input.options,
+                    abortSignal: context.signal
+                }
+            ]
+    }),
+    defineAgentActionNode({
+        method: 'callActionInActionSpace',
+        name: 'action',
+        description: 'Call a platform or custom action. Its ActionSpace schema validates params when the action executes.',
+        inputSchema: actionInputSchema,
+        toArgs: (input)=>[
+                input.name,
+                input.params
+            ],
+        toResult: (value)=>void 0 === value ? void 0 : valueResult(value)
+    })
+];
+const insightNode = (method)=>defineCommonAgentNode({
         method,
         description: `Run ${method} with a Midscene UI Agent and store its value.`,
         stringInputKey: 'prompt',
         inputSchema: insightInputSchema,
-        toArgs: (input)=>[
+        toArgs: (input, context)=>[
                 input.prompt,
-                input.options
+                {
+                    ...input.options,
+                    abortSignal: context.signal
+                }
             ],
         toResult: (value)=>({
-                summary: `${method} returned ${JSON.stringify(value)}`,
+                summary: `${method} returned ${'aiString' === method || 'aiAsk' === method ? JSON.stringify(value) : value}`,
                 data: {
                     value
                 }
             })
     });
-};
 const recordToReportNode = defineCommonAgentNode({
     method: 'recordToReport',
     description: 'Add text or screenshots to the current Midscene report.',
@@ -303,6 +548,22 @@ const recordToReportNode = defineCommonAgentNode({
     }
 });
 const commonAgentTestRunnerNodeDefinitions = [
+    defineAgentActionNode({
+        method: 'sleep',
+        description: 'Wait for a fixed number of milliseconds, recording standard UI snapshots and honoring cancellation.',
+        inputSchema: v4_namespaceObject.z.strictObject({
+            ms: v4_namespaceObject.z.number().positive()
+        }),
+        toArgs (input, { signal }) {
+            signal.throwIfAborted();
+            return [
+                input.ms,
+                {
+                    abortSignal: signal
+                }
+            ];
+        }
+    }),
     aiActNode,
     aiTapNode,
     aiAssertNode,
@@ -310,39 +571,62 @@ const commonAgentTestRunnerNodeDefinitions = [
     insightNode('aiNumber'),
     insightNode('aiString'),
     insightNode('aiAsk'),
-    recordToReportNode
+    recordToReportNode,
+    ...additionalAgentNodes
 ];
+exports.actionInputSchema = __webpack_exports__.actionInputSchema;
 exports.aiActInputSchema = __webpack_exports__.aiActInputSchema;
 exports.aiActOptionsInputSchema = __webpack_exports__.aiActOptionsInputSchema;
 exports.aiAssertInputSchema = __webpack_exports__.aiAssertInputSchema;
 exports.aiAssertOptionsInputSchema = __webpack_exports__.aiAssertOptionsInputSchema;
+exports.aiDragAndDropInputSchema = __webpack_exports__.aiDragAndDropInputSchema;
+exports.aiInputInputSchema = __webpack_exports__.aiInputInputSchema;
+exports.aiKeyboardPressInputSchema = __webpack_exports__.aiKeyboardPressInputSchema;
+exports.aiLongPressInputSchema = __webpack_exports__.aiLongPressInputSchema;
+exports.aiPinchInputSchema = __webpack_exports__.aiPinchInputSchema;
+exports.aiQueryInputSchema = __webpack_exports__.aiQueryInputSchema;
+exports.aiScrollInputSchema = __webpack_exports__.aiScrollInputSchema;
 exports.aiTapInputSchema = __webpack_exports__.aiTapInputSchema;
+exports.aiWaitForInputSchema = __webpack_exports__.aiWaitForInputSchema;
 exports.commonAgentTestRunnerNodeDefinitions = __webpack_exports__.commonAgentTestRunnerNodeDefinitions;
 exports.createAgentTestRunnerNodeDefinition = __webpack_exports__.createAgentTestRunnerNodeDefinition;
 exports.insightInputSchema = __webpack_exports__.insightInputSchema;
 exports.insightOptionsInputSchema = __webpack_exports__.insightOptionsInputSchema;
+exports.javascriptInputSchema = __webpack_exports__.javascriptInputSchema;
 exports.locateOptionsInputSchema = __webpack_exports__.locateOptionsInputSchema;
 exports.promptImageInputSchema = __webpack_exports__.promptImageInputSchema;
 exports.recordToReportInputSchema = __webpack_exports__.recordToReportInputSchema;
 exports.recordToReportOptionsInputSchema = __webpack_exports__.recordToReportOptionsInputSchema;
 exports.reportScreenshotInputSchema = __webpack_exports__.reportScreenshotInputSchema;
+exports.runGherkinScenarioInputSchema = __webpack_exports__.runGherkinScenarioInputSchema;
 exports.structuredUserPromptInputSchema = __webpack_exports__.structuredUserPromptInputSchema;
 exports.userPromptInputSchema = __webpack_exports__.userPromptInputSchema;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "actionInputSchema",
     "aiActInputSchema",
     "aiActOptionsInputSchema",
     "aiAssertInputSchema",
     "aiAssertOptionsInputSchema",
+    "aiDragAndDropInputSchema",
+    "aiInputInputSchema",
+    "aiKeyboardPressInputSchema",
+    "aiLongPressInputSchema",
+    "aiPinchInputSchema",
+    "aiQueryInputSchema",
+    "aiScrollInputSchema",
     "aiTapInputSchema",
+    "aiWaitForInputSchema",
     "commonAgentTestRunnerNodeDefinitions",
     "createAgentTestRunnerNodeDefinition",
     "insightInputSchema",
     "insightOptionsInputSchema",
+    "javascriptInputSchema",
     "locateOptionsInputSchema",
     "promptImageInputSchema",
     "recordToReportInputSchema",
     "recordToReportOptionsInputSchema",
     "reportScreenshotInputSchema",
+    "runGherkinScenarioInputSchema",
     "structuredUserPromptInputSchema",
     "userPromptInputSchema"
 ].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];