@midscene/core 1.0.1-beta-20251030070226.0 → 1.0.1-beta-20251104075048.0

package/dist/lib/yaml/player.js

@@ -29,6 +29,7 @@ __webpack_require__.d(__webpack_exports__, {
 const external_node_fs_namespaceObject = require("node:fs");
 const external_node_path_namespaceObject = require("node:path");
 const utils_namespaceObject = require("@midscene/shared/utils");
+const external_zod_namespaceObject = require("zod");
 const common_namespaceObject = require("@midscene/shared/common");
 const logger_namespaceObject = require("@midscene/shared/logger");
 const external_utils_js_namespaceObject = require("./utils.js");
@@ -43,6 +44,34 @@ function _define_property(obj, key, value) {
     return obj;
 }
 const debug = (0, logger_namespaceObject.getDebug)('yaml-player');
+const isStringParamSchema = (schema)=>{
+    if (!schema) return false;
+    const schemaDef = null == schema ? void 0 : schema._def;
+    if (!(null == schemaDef ? void 0 : schemaDef.typeName)) return false;
+    switch(schemaDef.typeName){
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodString:
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodEnum:
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodNativeEnum:
+            return true;
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodLiteral:
+            return 'string' == typeof schemaDef.value;
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodOptional:
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodNullable:
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodDefault:
+            return isStringParamSchema(schemaDef.innerType);
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodEffects:
+            return isStringParamSchema(schemaDef.schema);
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodPipeline:
+            return isStringParamSchema(schemaDef.out);
+        case external_zod_namespaceObject.z.ZodFirstPartyTypeKind.ZodUnion:
+            {
+                const options = schemaDef.options;
+                return Array.isArray(options) ? options.every((option)=>isStringParamSchema(option)) : false;
+            }
+        default:
+            return false;
+    }
+};
 class ScriptPlayer {
     setResult(key, value) {
         const keyToUse = key || this.unnamedResultIndex++;
@@ -250,34 +279,56 @@ class ScriptPlayer {
             } else {
                 const actionSpace = this.actionSpace;
                 let locatePromptShortcut;
+                let actionParamForMatchedAction;
                 const matchedAction = actionSpace.find((action)=>{
                     const actionInterfaceAlias = action.interfaceAlias;
                     if (actionInterfaceAlias && Object.prototype.hasOwnProperty.call(flowItem, actionInterfaceAlias)) {
-                        locatePromptShortcut = flowItem[actionInterfaceAlias];
+                        actionParamForMatchedAction = flowItem[actionInterfaceAlias];
+                        if ('string' == typeof actionParamForMatchedAction) locatePromptShortcut = actionParamForMatchedAction;
                         return true;
                     }
                     const keyOfActionInActionSpace = action.name;
                     if (Object.prototype.hasOwnProperty.call(flowItem, keyOfActionInActionSpace)) {
-                        locatePromptShortcut = flowItem[keyOfActionInActionSpace];
+                        actionParamForMatchedAction = flowItem[keyOfActionInActionSpace];
+                        if ('string' == typeof actionParamForMatchedAction) locatePromptShortcut = actionParamForMatchedAction;
                         return true;
                     }
                     return false;
                 });
                 (0, utils_namespaceObject.assert)(matchedAction, `unknown flowItem in yaml: ${JSON.stringify(flowItem)}`);
-                const flowItemForProcessing = locatePromptShortcut ? {
-                    ...flowItem,
-                    prompt: locatePromptShortcut
-                } : flowItem;
-                const { locateParam, restParams } = (0, external_utils_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePromptShortcut || '', flowItemForProcessing, [
-                    matchedAction.name,
-                    matchedAction.interfaceAlias || '_never_mind_'
-                ]);
-                const flowParams = {
-                    ...restParams,
-                    locate: locateParam
-                };
-                debug(`matchedAction: ${matchedAction.name}`, `flowParams: ${JSON.stringify(flowParams, null, 2)}`);
-                await agent.callActionInActionSpace(matchedAction.name, flowParams);
+                const schemaIsStringParam = isStringParamSchema(matchedAction.paramSchema);
+                let stringParamToCall;
+                if ('string' == typeof actionParamForMatchedAction && schemaIsStringParam) if (matchedAction.paramSchema) {
+                    const parseResult = matchedAction.paramSchema.safeParse(actionParamForMatchedAction);
+                    if (parseResult.success && 'string' == typeof parseResult.data) stringParamToCall = parseResult.data;
+                    else if (!parseResult.success) {
+                        debug(`parse failed for action ${matchedAction.name} with string param`, parseResult.error);
+                        stringParamToCall = actionParamForMatchedAction;
+                    }
+                } else stringParamToCall = actionParamForMatchedAction;
+                if (void 0 !== stringParamToCall) {
+                    debug(`matchedAction: ${matchedAction.name}`, `flowParams: ${JSON.stringify(stringParamToCall)}`);
+                    const result = await agent.callActionInActionSpace(matchedAction.name, stringParamToCall);
+                    const resultName = flowItem.name;
+                    if (void 0 !== result) this.setResult(resultName, result);
+                } else {
+                    const sourceForParams = locatePromptShortcut && 'string' == typeof actionParamForMatchedAction ? {
+                        ...flowItem,
+                        prompt: locatePromptShortcut
+                    } : 'object' == typeof actionParamForMatchedAction && null !== actionParamForMatchedAction ? actionParamForMatchedAction : flowItem;
+                    const { locateParam, restParams } = (0, external_utils_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePromptShortcut || '', sourceForParams, [
+                        matchedAction.name,
+                        matchedAction.interfaceAlias || '_never_mind_'
+                    ]);
+                    const flowParams = {
+                        ...restParams,
+                        locate: locateParam
+                    };
+                    debug(`matchedAction: ${matchedAction.name}`, `flowParams: ${JSON.stringify(flowParams, null, 2)}`);
+                    const result = await agent.callActionInActionSpace(matchedAction.name, flowParams);
+                    const resultName = flowItem.name;
+                    if (void 0 !== result) this.setResult(resultName, result);
+                }
             }
         }
         this.reportFile = agent.reportFile;

package/dist/lib/yaml/player.js.map

@@ -1 +1 @@
-{"version":3,"file":"yaml/player.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/yaml/player.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { basename, dirname, join, resolve } from 'node:path';\nimport { assert, ifInBrowser, ifInWorker } from '@midscene/shared/utils';\n\n// previous defined yaml flow, as a helper\ninterface MidsceneYamlFlowItemAIInput extends LocateOption {\n  // previous version\n  // aiInput: string; // value to input\n  // locate: TUserPrompt; // where to input\n  aiInput: TUserPrompt | undefined; // where to input\n  value: string | number; // value to input\n}\n\ninterface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {\n  // previous version\n  // aiKeyboardPress: string;\n  // locate?: TUserPrompt; // where to press, optional\n  aiKeyboardPress: TUserPrompt | undefined; // where to press\n  keyName: string; // key to press\n}\n\ninterface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {\n  // previous version\n  // aiScroll: null;\n  // locate?: TUserPrompt; // which area to scroll, optional\n  aiScroll: TUserPrompt | undefined; // which area to scroll\n}\n\nimport type { Agent } from '@/agent/agent';\nimport type { TUserPrompt } from '@/ai-model/common';\nimport type {\n  DeviceAction,\n  FreeFn,\n  LocateOption,\n  MidsceneYamlFlowItemAIAction,\n  MidsceneYamlFlowItemAIAsk,\n  MidsceneYamlFlowItemAIAssert,\n  MidsceneYamlFlowItemAIBoolean,\n  MidsceneYamlFlowItemAILocate,\n  MidsceneYamlFlowItemAINumber,\n  MidsceneYamlFlowItemAIQuery,\n  MidsceneYamlFlowItemAIString,\n  MidsceneYamlFlowItemAIWaitFor,\n  MidsceneYamlFlowItemEvaluateJavaScript,\n  MidsceneYamlFlowItemLogScreenshot,\n  MidsceneYamlFlowItemSleep,\n  MidsceneYamlScript,\n  MidsceneYamlScriptEnv,\n  ScriptPlayerStatusValue,\n  ScriptPlayerTaskStatus,\n  ScrollParam,\n} from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { getDebug } from '@midscene/shared/logger';\nimport {\n  buildDetailedLocateParam,\n  buildDetailedLocateParamAndRestParams,\n} from './utils';\n\nconst debug = getDebug('yaml-player');\nexport class ScriptPlayer<T extends MidsceneYamlScriptEnv> {\n  public currentTaskIndex?: number;\n  public taskStatusList: ScriptPlayerTaskStatus[] = [];\n  public status: ScriptPlayerStatusValue = 'init';\n  public reportFile?: string | null;\n  public result: Record<string, any>;\n  private unnamedResultIndex = 0;\n  public output?: string | null;\n  public unstableLogContent?: string | null;\n  public errorInSetup?: Error;\n  private interfaceAgent: Agent | null = null;\n  public agentStatusTip?: string;\n  public target?: MidsceneYamlScriptEnv;\n  private actionSpace: DeviceAction[] = [];\n  private scriptPath?: string;\n  constructor(\n    private script: MidsceneYamlScript,\n    private setupAgent: (platform: T) => Promise<{\n      agent: Agent;\n      freeFn: FreeFn[];\n    }>,\n    public onTaskStatusChange?: (taskStatus: ScriptPlayerTaskStatus) => void,\n    scriptPath?: string,\n  ) {\n    this.scriptPath = scriptPath;\n    this.result = {};\n    this.target =\n      script.target ||\n      script.web ||\n      script.android ||\n      script.ios ||\n      script.config;\n\n    if (ifInBrowser || ifInWorker) {\n      this.output = undefined;\n      debug('output is undefined in browser or worker');\n    } else if (this.target?.output) {\n      this.output = resolve(process.cwd(), this.target.output);\n      debug('setting output by config.output', this.output);\n    } else {\n      const scriptName = this.scriptPath\n        ? basename(this.scriptPath, '.yaml').replace(/\\.(ya?ml)$/i, '')\n        : 'script';\n      this.output = join(\n        getMidsceneRunSubDir('output'),\n        `${scriptName}-${Date.now()}.json`,\n      );\n      debug('setting output by script path', this.output);\n    }\n\n    if (ifInBrowser || ifInWorker) {\n      this.unstableLogContent = undefined;\n    } else if (typeof this.target?.unstableLogContent === 'string') {\n      this.unstableLogContent = resolve(\n        process.cwd(),\n        this.target.unstableLogContent,\n      );\n    } else if (this.target?.unstableLogContent === true) {\n      this.unstableLogContent = join(\n        getMidsceneRunSubDir('output'),\n        'unstableLogContent.json',\n      );\n    }\n\n    this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({\n      ...task,\n      index: taskIndex,\n      status: 'init',\n      totalSteps: task.flow?.length || 0,\n    }));\n  }\n\n  private setResult(key: string | undefined, value: any) {\n    const keyToUse = key || this.unnamedResultIndex++;\n    if (this.result[keyToUse]) {\n      console.warn(`result key ${keyToUse} already exists, will overwrite`);\n    }\n    this.result[keyToUse] = value;\n\n    return this.flushResult();\n  }\n\n  private setPlayerStatus(status: ScriptPlayerStatusValue, error?: Error) {\n    this.status = status;\n    this.errorInSetup = error;\n  }\n\n  private notifyCurrentTaskStatusChange(taskIndex?: number) {\n    const taskIndexToNotify =\n      typeof taskIndex === 'number' ? taskIndex : this.currentTaskIndex;\n\n    if (typeof taskIndexToNotify !== 'number') {\n      return;\n    }\n\n    const taskStatus = this.taskStatusList[taskIndexToNotify];\n    if (this.onTaskStatusChange) {\n      this.onTaskStatusChange(taskStatus);\n    }\n  }\n\n  private async setTaskStatus(\n    index: number,\n    statusValue: ScriptPlayerStatusValue,\n    error?: Error,\n  ) {\n    this.taskStatusList[index].status = statusValue;\n    if (error) {\n      this.taskStatusList[index].error = error;\n    }\n\n    this.notifyCurrentTaskStatusChange(index);\n  }\n\n  private setTaskIndex(taskIndex: number) {\n    this.currentTaskIndex = taskIndex;\n  }\n\n  private flushResult() {\n    if (this.output) {\n      const output = resolve(process.cwd(), this.output);\n      const outputDir = dirname(output);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(output, JSON.stringify(this.result || {}, undefined, 2));\n    }\n  }\n\n  private flushUnstableLogContent() {\n    if (this.unstableLogContent) {\n      const content = this.interfaceAgent?._unstableLogContent();\n      const filePath = resolve(process.cwd(), this.unstableLogContent);\n      const outputDir = dirname(filePath);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(filePath, JSON.stringify(content, null, 2));\n    }\n  }\n\n  async playTask(taskStatus: ScriptPlayerTaskStatus, agent: Agent) {\n    const { flow } = taskStatus;\n    assert(flow, 'missing flow in task');\n\n    for (const flowItemIndex in flow) {\n      const currentStep = Number.parseInt(flowItemIndex, 10);\n      taskStatus.currentStep = currentStep;\n      const flowItem = flow[flowItemIndex];\n      debug(\n        `playing step ${flowItemIndex}, flowItem=${JSON.stringify(flowItem)}`,\n      );\n      if (\n        'aiAct' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n        'aiAction' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n        'ai' in (flowItem as MidsceneYamlFlowItemAIAction)\n      ) {\n        const actionTask = flowItem as MidsceneYamlFlowItemAIAction;\n        const prompt = actionTask.aiAct || actionTask.aiAction || actionTask.ai;\n        assert(prompt, 'missing prompt for ai (aiAct)');\n        await agent.aiAct(prompt, {\n          cacheable: actionTask.cacheable,\n        });\n      } else if ('aiAssert' in (flowItem as MidsceneYamlFlowItemAIAssert)) {\n        const assertTask = flowItem as MidsceneYamlFlowItemAIAssert;\n        const prompt = assertTask.aiAssert;\n        const msg = assertTask.errorMessage;\n        assert(prompt, 'missing prompt for aiAssert');\n        const { pass, thought, message } =\n          (await agent.aiAssert(prompt, msg, {\n            keepRawResponse: true,\n          })) || {};\n\n        this.setResult(assertTask.name, {\n          pass,\n          thought,\n          message,\n        });\n\n        if (!pass) {\n          throw new Error(message);\n        }\n      } else if ('aiQuery' in (flowItem as MidsceneYamlFlowItemAIQuery)) {\n        const queryTask = flowItem as MidsceneYamlFlowItemAIQuery;\n        const prompt = queryTask.aiQuery;\n        const options = {\n          domIncluded: queryTask.domIncluded,\n          screenshotIncluded: queryTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiQuery');\n        const queryResult = await agent.aiQuery(prompt, options);\n        this.setResult(queryTask.name, queryResult);\n      } else if ('aiNumber' in (flowItem as MidsceneYamlFlowItemAINumber)) {\n        const numberTask = flowItem as MidsceneYamlFlowItemAINumber;\n        const prompt = numberTask.aiNumber;\n        const options = {\n          domIncluded: numberTask.domIncluded,\n          screenshotIncluded: numberTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiNumber');\n        const numberResult = await agent.aiNumber(prompt, options);\n        this.setResult(numberTask.name, numberResult);\n      } else if ('aiString' in (flowItem as MidsceneYamlFlowItemAIString)) {\n        const stringTask = flowItem as MidsceneYamlFlowItemAIString;\n        const prompt = stringTask.aiString;\n        const options = {\n          domIncluded: stringTask.domIncluded,\n          screenshotIncluded: stringTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiString');\n        const stringResult = await agent.aiString(prompt, options);\n        this.setResult(stringTask.name, stringResult);\n      } else if ('aiBoolean' in (flowItem as MidsceneYamlFlowItemAIBoolean)) {\n        const booleanTask = flowItem as MidsceneYamlFlowItemAIBoolean;\n        const prompt = booleanTask.aiBoolean;\n        const options = {\n          domIncluded: booleanTask.domIncluded,\n          screenshotIncluded: booleanTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiBoolean');\n        const booleanResult = await agent.aiBoolean(prompt, options);\n        this.setResult(booleanTask.name, booleanResult);\n      } else if ('aiAsk' in (flowItem as MidsceneYamlFlowItemAIAsk)) {\n        const askTask = flowItem as MidsceneYamlFlowItemAIAsk;\n        const prompt = askTask.aiAsk;\n        assert(prompt, 'missing prompt for aiAsk');\n        const askResult = await agent.aiAsk(prompt);\n        this.setResult(askTask.name, askResult);\n      } else if ('aiLocate' in (flowItem as MidsceneYamlFlowItemAILocate)) {\n        const locateTask = flowItem as MidsceneYamlFlowItemAILocate;\n        const prompt = locateTask.aiLocate;\n        assert(prompt, 'missing prompt for aiLocate');\n        const locateResult = await agent.aiLocate(prompt, locateTask);\n        this.setResult(locateTask.name, locateResult);\n      } else if ('aiWaitFor' in (flowItem as MidsceneYamlFlowItemAIWaitFor)) {\n        const waitForTask = flowItem as MidsceneYamlFlowItemAIWaitFor;\n        const prompt = waitForTask.aiWaitFor;\n        assert(prompt, 'missing prompt for aiWaitFor');\n        const timeout = waitForTask.timeout;\n        await agent.aiWaitFor(prompt, { timeoutMs: timeout });\n      } else if ('sleep' in (flowItem as MidsceneYamlFlowItemSleep)) {\n        const sleepTask = flowItem as MidsceneYamlFlowItemSleep;\n        const ms = sleepTask.sleep;\n        let msNumber = ms;\n        if (typeof ms === 'string') {\n          msNumber = Number.parseInt(ms, 10);\n        }\n        assert(\n          msNumber && msNumber > 0,\n          `ms for sleep must be greater than 0, but got ${ms}`,\n        );\n        await new Promise((resolve) => setTimeout(resolve, msNumber));\n      } else if (\n        'javascript' in (flowItem as MidsceneYamlFlowItemEvaluateJavaScript)\n      ) {\n        const evaluateJavaScriptTask =\n          flowItem as MidsceneYamlFlowItemEvaluateJavaScript;\n\n        const result = await agent.evaluateJavaScript(\n          evaluateJavaScriptTask.javascript,\n        );\n        this.setResult(evaluateJavaScriptTask.name, result);\n      } else if (\n        'logScreenshot' in (flowItem as MidsceneYamlFlowItemLogScreenshot) ||\n        'recordToReport' in (flowItem as MidsceneYamlFlowItemLogScreenshot)\n      ) {\n        const recordTask = flowItem as MidsceneYamlFlowItemLogScreenshot;\n        const title =\n          recordTask.recordToReport ?? recordTask.logScreenshot ?? 'untitled';\n        const content = recordTask.content || '';\n        await agent.recordToReport(title, { content });\n      } else if ('aiInput' in (flowItem as MidsceneYamlFlowItemAIInput)) {\n        // may be input empty string ''\n        const {\n          aiInput,\n          value: rawValue,\n          ...inputTask\n        } = flowItem as MidsceneYamlFlowItemAIInput;\n\n        // Compatibility with previous version:\n        // Old format: { aiInput: string (value), locate: TUserPrompt }\n        // New format - 1: { aiInput: TUserPrompt, value: string | number }\n        // New format - 2: { aiInput: undefined, locate: TUserPrompt, value: string | number }\n        let locatePrompt: TUserPrompt | undefined;\n        let value: string | number | undefined;\n        if ((inputTask as any).locate) {\n          // Old format - aiInput is the value, locate is the prompt\n          // Keep backward compatibility: empty string is treated as no value\n          value = (aiInput as string | number) || rawValue;\n          locatePrompt = (inputTask as any).locate;\n        } else {\n          // New format - aiInput is the prompt, value is the value\n          locatePrompt = aiInput || '';\n          value = rawValue;\n        }\n\n        // Convert value to string for Input action\n        await agent.callActionInActionSpace('Input', {\n          ...inputTask,\n          ...(value !== undefined ? { value: String(value) } : {}),\n          ...(locatePrompt\n            ? { locate: buildDetailedLocateParam(locatePrompt, inputTask) }\n            : {}),\n        });\n      } else if (\n        'aiKeyboardPress' in (flowItem as MidsceneYamlFlowItemAIKeyboardPress)\n      ) {\n        const { aiKeyboardPress, ...keyboardPressTask } =\n          flowItem as MidsceneYamlFlowItemAIKeyboardPress;\n\n        // Compatibility with previous version:\n        // Old format: { aiKeyboardPress: string (key), locate?: TUserPrompt }\n        // New format - 1: { aiKeyboardPress: TUserPrompt, keyName: string }\n        // New format - 2: { aiKeyboardPress: , locate?: TUserPrompt, keyName: string }\n        let locatePrompt: TUserPrompt | undefined;\n        let keyName: string | undefined;\n        if ((keyboardPressTask as any).locate) {\n          // Old format - aiKeyboardPress is the key, locate is the prompt\n          keyName = aiKeyboardPress as string;\n          locatePrompt = (keyboardPressTask as any).locate;\n        } else if (keyboardPressTask.keyName) {\n          // New format - aiKeyboardPress is the prompt, key is the key\n          keyName = keyboardPressTask.keyName;\n          locatePrompt = aiKeyboardPress;\n        } else {\n          keyName = aiKeyboardPress as string;\n        }\n\n        await agent.callActionInActionSpace('KeyboardPress', {\n          ...keyboardPressTask,\n          ...(keyName ? { keyName } : {}),\n          ...(locatePrompt\n            ? {\n                locate: buildDetailedLocateParam(\n                  locatePrompt,\n                  keyboardPressTask,\n                ),\n              }\n            : {}),\n        });\n      } else if ('aiScroll' in (flowItem as MidsceneYamlFlowItemAIScroll)) {\n        const { aiScroll, ...scrollTask } =\n          flowItem as MidsceneYamlFlowItemAIScroll;\n\n        // Compatibility with previous version:\n        // Old format: { aiScroll: null, locate?: TUserPrompt, direction, scrollType, distance? }\n        // New format - 1: { aiScroll: TUserPrompt, direction, scrollType, distance? }\n        // New format - 2: { aiScroll: undefined, locate: TUserPrompt, direction, scrollType, distance? }\n        let locatePrompt: TUserPrompt | undefined;\n        if ((scrollTask as any).locate) {\n          // Old format - locate is the prompt, aiScroll is null/ignored\n          locatePrompt = (scrollTask as any).locate;\n        } else {\n          // New format - aiScroll is the prompt, or no prompt for global scroll\n          locatePrompt = aiScroll;\n        }\n\n        await agent.callActionInActionSpace('Scroll', {\n          ...scrollTask,\n          ...(locatePrompt\n            ? { locate: buildDetailedLocateParam(locatePrompt, scrollTask) }\n            : {}),\n        });\n      } else {\n        // generic action, find the action in actionSpace\n\n        /* for aiTap, aiRightClick, the parameters are a flattened data for the 'locate', these are all valid data\n\n        - aiTap: 'search input box'\n        - aiTap: 'search input box'\n          deepThink: true\n          cacheable: false\n        - aiTap:\n          prompt: 'search input box'\n        - aiTap:\n          prompt: 'search input box'\n          deepThink: true\n          cacheable: false\n        */\n\n        const actionSpace = this.actionSpace;\n        let locatePromptShortcut: string | undefined;\n        const matchedAction = actionSpace.find((action) => {\n          const actionInterfaceAlias = action.interfaceAlias;\n          if (\n            actionInterfaceAlias &&\n            Object.prototype.hasOwnProperty.call(flowItem, actionInterfaceAlias)\n          ) {\n            locatePromptShortcut = flowItem[\n              actionInterfaceAlias as keyof typeof flowItem\n            ] as string;\n            return true;\n          }\n\n          const keyOfActionInActionSpace = action.name;\n          if (\n            Object.prototype.hasOwnProperty.call(\n              flowItem,\n              keyOfActionInActionSpace,\n            )\n          ) {\n            locatePromptShortcut = flowItem[\n              keyOfActionInActionSpace as keyof typeof flowItem\n            ] as string;\n            return true;\n          }\n\n          return false;\n        });\n\n        assert(\n          matchedAction,\n          `unknown flowItem in yaml: ${JSON.stringify(flowItem)}`,\n        );\n\n        // Create a new object instead of mutating the original flowItem\n        // This prevents issues when the same YAML script is executed multiple times\n        const flowItemForProcessing = locatePromptShortcut\n          ? { ...flowItem, prompt: locatePromptShortcut }\n          : flowItem;\n\n        const { locateParam, restParams } =\n          buildDetailedLocateParamAndRestParams(\n            locatePromptShortcut || '',\n            flowItemForProcessing as LocateOption,\n            [\n              matchedAction.name,\n              matchedAction.interfaceAlias || '_never_mind_',\n            ],\n          );\n\n        const flowParams = {\n          ...restParams,\n          locate: locateParam,\n        };\n\n        debug(\n          `matchedAction: ${matchedAction.name}`,\n          `flowParams: ${JSON.stringify(flowParams, null, 2)}`,\n        );\n        await agent.callActionInActionSpace(matchedAction.name, flowParams);\n      }\n    }\n    this.reportFile = agent.reportFile;\n    await this.flushUnstableLogContent();\n  }\n\n  async run() {\n    const { target, web, android, ios, tasks } = this.script;\n    const webEnv = web || target;\n    const androidEnv = android;\n    const iosEnv = ios;\n    const platform = webEnv || androidEnv || iosEnv;\n\n    this.setPlayerStatus('running');\n\n    let agent: Agent | null = null;\n    let freeFn: FreeFn[] = [];\n    try {\n      const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(\n        platform as T,\n      );\n      this.actionSpace = await newAgent.getActionSpace();\n      agent = newAgent;\n      const originalOnTaskStartTip = agent.onTaskStartTip;\n      agent.onTaskStartTip = (tip) => {\n        if (this.status === 'running') {\n          this.agentStatusTip = tip;\n        }\n        originalOnTaskStartTip?.(tip);\n      };\n      freeFn = [\n        ...(newFreeFn || []),\n        {\n          name: 'restore-agent-onTaskStartTip',\n          fn: () => {\n            if (agent) {\n              agent.onTaskStartTip = originalOnTaskStartTip;\n            }\n          },\n        },\n      ];\n    } catch (e) {\n      this.setPlayerStatus('error', e as Error);\n      return;\n    }\n    this.interfaceAgent = agent;\n\n    let taskIndex = 0;\n    this.setPlayerStatus('running');\n    let errorFlag = false;\n    while (taskIndex < tasks.length) {\n      const taskStatus = this.taskStatusList[taskIndex];\n      this.setTaskStatus(taskIndex, 'running' as any);\n      this.setTaskIndex(taskIndex);\n\n      try {\n        await this.playTask(taskStatus, this.interfaceAgent);\n        this.setTaskStatus(taskIndex, 'done' as any);\n      } catch (e) {\n        this.setTaskStatus(taskIndex, 'error' as any, e as Error);\n\n        if (taskStatus.continueOnError) {\n          // nothing more to do\n        } else {\n          this.reportFile = agent.reportFile;\n          errorFlag = true;\n          break;\n        }\n      }\n      this.reportFile = agent?.reportFile;\n      taskIndex++;\n    }\n\n    if (errorFlag) {\n      this.setPlayerStatus('error');\n    } else {\n      this.setPlayerStatus('done');\n    }\n    this.agentStatusTip = '';\n\n    // free the resources\n    for (const fn of freeFn) {\n      try {\n        // console.log('freeing', fn.name);\n        await fn.fn();\n        // console.log('freed', fn.name);\n      } catch (e) {\n        // console.error('error freeing', fn.name, e);\n      }\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","ScriptPlayer","value","keyToUse","console","status","error","taskIndex","taskIndexToNotify","taskStatus","index","statusValue","output","resolve","process","outputDir","dirname","existsSync","mkdirSync","writeFileSync","JSON","undefined","_this_interfaceAgent","content","filePath","agent","flow","assert","flowItemIndex","currentStep","Number","flowItem","actionTask","prompt","assertTask","msg","pass","thought","message","Error","queryTask","options","queryResult","numberTask","numberResult","stringTask","stringResult","booleanTask","booleanResult","askTask","askResult","locateTask","locateResult","waitForTask","timeout","sleepTask","ms","msNumber","Promise","setTimeout","evaluateJavaScriptTask","result","recordTask","title","aiInput","rawValue","inputTask","locatePrompt","String","buildDetailedLocateParam","aiKeyboardPress","keyboardPressTask","keyName","aiScroll","scrollTask","actionSpace","locatePromptShortcut","matchedAction","action","actionInterfaceAlias","keyOfActionInActionSpace","flowItemForProcessing","locateParam","restParams","buildDetailedLocateParamAndRestParams","flowParams","target","web","android","ios","tasks","webEnv","androidEnv","iosEnv","platform","freeFn","newAgent","newFreeFn","originalOnTaskStartTip","tip","e","errorFlag","fn","script","setupAgent","onTaskStatusChange","scriptPath","_this_target","_this_target1","_this_target2","ifInBrowser","ifInWorker","scriptName","basename","join","getMidsceneRunSubDir","Date","task","_task_flow"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;ACqDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAChB,MAAMC;IAwEH,UAAUP,GAAuB,EAAEQ,KAAU,EAAE;QACrD,MAAMC,WAAWT,OAAO,IAAI,CAAC,kBAAkB;QAC/C,IAAI,IAAI,CAAC,MAAM,CAACS,SAAS,EACvBC,QAAQ,IAAI,CAAC,CAAC,WAAW,EAAED,SAAS,+BAA+B,CAAC;QAEtE,IAAI,CAAC,MAAM,CAACA,SAAS,GAAGD;QAExB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEQ,gBAAgBG,MAA+B,EAAEC,KAAa,EAAE;QACtE,IAAI,CAAC,MAAM,GAAGD;QACd,IAAI,CAAC,YAAY,GAAGC;IACtB;IAEQ,8BAA8BC,SAAkB,EAAE;QACxD,MAAMC,oBACJ,AAAqB,YAArB,OAAOD,YAAyBA,YAAY,IAAI,CAAC,gBAAgB;QAEnE,IAAI,AAA6B,YAA7B,OAAOC,mBACT;QAGF,MAAMC,aAAa,IAAI,CAAC,cAAc,CAACD,kBAAkB;QACzD,IAAI,IAAI,CAAC,kBAAkB,EACzB,IAAI,CAAC,kBAAkB,CAACC;IAE5B;IAEA,MAAc,cACZC,KAAa,EACbC,WAAoC,EACpCL,KAAa,EACb;QACA,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,MAAM,GAAGC;QACpC,IAAIL,OACF,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,KAAK,GAAGJ;QAGrC,IAAI,CAAC,6BAA6B,CAACI;IACrC;IAEQ,aAAaH,SAAiB,EAAE;QACtC,IAAI,CAAC,gBAAgB,GAAGA;IAC1B;IAEQ,cAAc;QACpB,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,MAAMK,SAASC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM;YACjD,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQJ;YAC1B,IAAI,CAACK,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcP,QAAQQ,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,GAAGC,QAAW;QACrE;IACF;IAEQ,0BAA0B;QAChC,IAAI,IAAI,CAAC,kBAAkB,EAAE;gBACXC;YAAhB,MAAMC,UAAU,QAAAD,CAAAA,uBAAAA,IAAI,CAAC,cAAc,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,mBAAmB;YACxD,MAAME,WAAWX,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,kBAAkB;YAC/D,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQQ;YAC1B,IAAI,CAACP,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcK,UAAUJ,KAAK,SAAS,CAACG,SAAS,MAAM;QACxD;IACF;IAEA,MAAM,SAASd,UAAkC,EAAEgB,KAAY,EAAE;QAC/D,MAAM,EAAEC,IAAI,EAAE,GAAGjB;QACjBkB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,MAAM;QAEb,IAAK,MAAME,iBAAiBF,KAAM;YAChC,MAAMG,cAAcC,OAAO,QAAQ,CAACF,eAAe;YACnDnB,WAAW,WAAW,GAAGoB;YACzB,MAAME,WAAWL,IAAI,CAACE,cAAc;YACpC7B,MACE,CAAC,aAAa,EAAE6B,cAAc,WAAW,EAAER,KAAK,SAAS,CAACW,WAAW;YAEvE,IACE,WAAYA,YACZ,cAAeA,YACf,QAASA,UACT;gBACA,MAAMC,aAAaD;gBACnB,MAAME,SAASD,WAAW,KAAK,IAAIA,WAAW,QAAQ,IAAIA,WAAW,EAAE;gBACvEL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMR,MAAM,KAAK,CAACQ,QAAQ;oBACxB,WAAWD,WAAW,SAAS;gBACjC;YACF,OAAO,IAAI,cAAeD,UAA2C;gBACnE,MAAMG,aAAaH;gBACnB,MAAME,SAASC,WAAW,QAAQ;gBAClC,MAAMC,MAAMD,WAAW,YAAY;gBACnCP,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAM,EAAEG,IAAI,EAAEC,OAAO,EAAEC,OAAO,EAAE,GAC7B,MAAMb,MAAM,QAAQ,CAACQ,QAAQE,KAAK;oBACjC,iBAAiB;gBACnB,MAAO,CAAC;gBAEV,IAAI,CAAC,SAAS,CAACD,WAAW,IAAI,EAAE;oBAC9BE;oBACAC;oBACAC;gBACF;gBAEA,IAAI,CAACF,MACH,MAAM,IAAIG,MAAMD;YAEpB,OAAO,IAAI,aAAcP,UAA0C;gBACjE,MAAMS,YAAYT;gBAClB,MAAME,SAASO,UAAU,OAAO;gBAChC,MAAMC,UAAU;oBACd,aAAaD,UAAU,WAAW;oBAClC,oBAAoBA,UAAU,kBAAkB;gBAClD;gBACAb,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMS,cAAc,MAAMjB,MAAM,OAAO,CAACQ,QAAQQ;gBAChD,IAAI,CAAC,SAAS,CAACD,UAAU,IAAI,EAAEE;YACjC,OAAO,IAAI,cAAeX,UAA2C;gBACnE,MAAMY,aAAaZ;gBACnB,MAAME,SAASU,WAAW,QAAQ;gBAClC,MAAMF,UAAU;oBACd,aAAaE,WAAW,WAAW;oBACnC,oBAAoBA,WAAW,kBAAkB;gBACnD;gBACAhB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMW,eAAe,MAAMnB,MAAM,QAAQ,CAACQ,QAAQQ;gBAClD,IAAI,CAAC,SAAS,CAACE,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,cAAeb,UAA2C;gBACnE,MAAMc,aAAad;gBACnB,MAAME,SAASY,WAAW,QAAQ;gBAClC,MAAMJ,UAAU;oBACd,aAAaI,WAAW,WAAW;oBACnC,oBAAoBA,WAAW,kBAAkB;gBACnD;gBACAlB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMa,eAAe,MAAMrB,MAAM,QAAQ,CAACQ,QAAQQ;gBAClD,IAAI,CAAC,SAAS,CAACI,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,eAAgBf,UAA4C;gBACrE,MAAMgB,cAAchB;gBACpB,MAAME,SAASc,YAAY,SAAS;gBACpC,MAAMN,UAAU;oBACd,aAAaM,YAAY,WAAW;oBACpC,oBAAoBA,YAAY,kBAAkB;gBACpD;gBACApB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMe,gBAAgB,MAAMvB,MAAM,SAAS,CAACQ,QAAQQ;gBACpD,IAAI,CAAC,SAAS,CAACM,YAAY,IAAI,EAAEC;YACnC,OAAO,IAAI,WAAYjB,UAAwC;gBAC7D,MAAMkB,UAAUlB;gBAChB,MAAME,SAASgB,QAAQ,KAAK;gBAC5BtB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMiB,YAAY,MAAMzB,MAAM,KAAK,CAACQ;gBACpC,IAAI,CAAC,SAAS,CAACgB,QAAQ,IAAI,EAAEC;YAC/B,OAAO,IAAI,cAAenB,UAA2C;gBACnE,MAAMoB,aAAapB;gBACnB,MAAME,SAASkB,WAAW,QAAQ;gBAClCxB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMmB,eAAe,MAAM3B,MAAM,QAAQ,CAACQ,QAAQkB;gBAClD,IAAI,CAAC,SAAS,CAACA,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,eAAgBrB,UAA4C;gBACrE,MAAMsB,cAActB;gBACpB,MAAME,SAASoB,YAAY,SAAS;gBACpC1B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMqB,UAAUD,YAAY,OAAO;gBACnC,MAAM5B,MAAM,SAAS,CAACQ,QAAQ;oBAAE,WAAWqB;gBAAQ;YACrD,OAAO,IAAI,WAAYvB,UAAwC;gBAC7D,MAAMwB,YAAYxB;gBAClB,MAAMyB,KAAKD,UAAU,KAAK;gBAC1B,IAAIE,WAAWD;gBACf,IAAI,AAAc,YAAd,OAAOA,IACTC,WAAW3B,OAAO,QAAQ,CAAC0B,IAAI;gBAEjC7B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8B,YAAYA,WAAW,GACvB,CAAC,6CAA6C,EAAED,IAAI;gBAEtD,MAAM,IAAIE,QAAQ,CAAC7C,UAAY8C,WAAW9C,SAAS4C;YACrD,OAAO,IACL,gBAAiB1B,UACjB;gBACA,MAAM6B,yBACJ7B;gBAEF,MAAM8B,SAAS,MAAMpC,MAAM,kBAAkB,CAC3CmC,uBAAuB,UAAU;gBAEnC,IAAI,CAAC,SAAS,CAACA,uBAAuB,IAAI,EAAEC;YAC9C,OAAO,IACL,mBAAoB9B,YACpB,oBAAqBA,UACrB;gBACA,MAAM+B,aAAa/B;gBACnB,MAAMgC,QACJD,WAAW,cAAc,IAAIA,WAAW,aAAa,IAAI;gBAC3D,MAAMvC,UAAUuC,WAAW,OAAO,IAAI;gBACtC,MAAMrC,MAAM,cAAc,CAACsC,OAAO;oBAAExC;gBAAQ;YAC9C,OAAO,IAAI,aAAcQ,UAA0C;gBAEjE,MAAM,EACJiC,OAAO,EACP,OAAOC,QAAQ,EACf,GAAGC,WACJ,GAAGnC;gBAMJ,IAAIoC;gBACJ,IAAIjE;gBACJ,IAAKgE,UAAkB,MAAM,EAAE;oBAG7BhE,QAAS8D,WAA+BC;oBACxCE,eAAgBD,UAAkB,MAAM;gBAC1C,OAAO;oBAELC,eAAeH,WAAW;oBAC1B9D,QAAQ+D;gBACV;gBAGA,MAAMxC,MAAM,uBAAuB,CAAC,SAAS;oBAC3C,GAAGyC,SAAS;oBACZ,GAAIhE,AAAUmB,WAAVnB,QAAsB;wBAAE,OAAOkE,OAAOlE;oBAAO,IAAI,CAAC,CAAC;oBACvD,GAAIiE,eACA;wBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcD;oBAAW,IAC5D,CAAC,CAAC;gBACR;YACF,OAAO,IACL,qBAAsBnC,UACtB;gBACA,MAAM,EAAEuC,eAAe,EAAE,GAAGC,mBAAmB,GAC7CxC;gBAMF,IAAIoC;gBACJ,IAAIK;gBACJ,IAAKD,kBAA0B,MAAM,EAAE;oBAErCC,UAAUF;oBACVH,eAAgBI,kBAA0B,MAAM;gBAClD,OAAO,IAAIA,kBAAkB,OAAO,EAAE;oBAEpCC,UAAUD,kBAAkB,OAAO;oBACnCJ,eAAeG;gBACjB,OACEE,UAAUF;gBAGZ,MAAM7C,MAAM,uBAAuB,CAAC,iBAAiB;oBACnD,GAAG8C,iBAAiB;oBACpB,GAAIC,UAAU;wBAAEA;oBAAQ,IAAI,CAAC,CAAC;oBAC9B,GAAIL,eACA;wBACE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACNF,cACAI;oBAEJ,IACA,CAAC,CAAC;gBACR;YACF,OAAO,IAAI,cAAexC,UAA2C;gBACnE,MAAM,EAAE0C,QAAQ,EAAE,GAAGC,YAAY,GAC/B3C;gBAMF,IAAIoC;gBAGFA,eAFGO,WAAmB,MAAM,GAEZA,WAAmB,MAAM,GAG1BD;gBAGjB,MAAMhD,MAAM,uBAAuB,CAAC,UAAU;oBAC5C,GAAGiD,UAAU;oBACb,GAAIP,eACA;wBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcO;oBAAY,IAC7D,CAAC,CAAC;gBACR;YACF,OAAO;gBAiBL,MAAMC,cAAc,IAAI,CAAC,WAAW;gBACpC,IAAIC;gBACJ,MAAMC,gBAAgBF,YAAY,IAAI,CAAC,CAACG;oBACtC,MAAMC,uBAAuBD,OAAO,cAAc;oBAClD,IACEC,wBACApF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACoC,UAAUgD,uBAC/C;wBACAH,uBAAuB7C,QAAQ,CAC7BgD,qBACD;wBACD,OAAO;oBACT;oBAEA,MAAMC,2BAA2BF,OAAO,IAAI;oBAC5C,IACEnF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAClCoC,UACAiD,2BAEF;wBACAJ,uBAAuB7C,QAAQ,CAC7BiD,yBACD;wBACD,OAAO;oBACT;oBAEA,OAAO;gBACT;gBAEArD,IAAAA,sBAAAA,MAAAA,AAAAA,EACEkD,eACA,CAAC,0BAA0B,EAAEzD,KAAK,SAAS,CAACW,WAAW;gBAKzD,MAAMkD,wBAAwBL,uBAC1B;oBAAE,GAAG7C,QAAQ;oBAAE,QAAQ6C;gBAAqB,IAC5C7C;gBAEJ,MAAM,EAAEmD,WAAW,EAAEC,UAAU,EAAE,GAC/BC,AAAAA,IAAAA,kCAAAA,qCAAAA,AAAAA,EACER,wBAAwB,IACxBK,uBACA;oBACEJ,cAAc,IAAI;oBAClBA,cAAc,cAAc,IAAI;iBACjC;gBAGL,MAAMQ,aAAa;oBACjB,GAAGF,UAAU;oBACb,QAAQD;gBACV;gBAEAnF,MACE,CAAC,eAAe,EAAE8E,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAEzD,KAAK,SAAS,CAACiE,YAAY,MAAM,IAAI;gBAEtD,MAAM5D,MAAM,uBAAuB,CAACoD,cAAc,IAAI,EAAEQ;YAC1D;QACF;QACA,IAAI,CAAC,UAAU,GAAG5D,MAAM,UAAU;QAClC,MAAM,IAAI,CAAC,uBAAuB;IACpC;IAEA,MAAM,MAAM;QACV,MAAM,EAAE6D,MAAM,EAAEC,GAAG,EAAEC,OAAO,EAAEC,GAAG,EAAEC,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM;QACxD,MAAMC,SAASJ,OAAOD;QACtB,MAAMM,aAAaJ;QACnB,MAAMK,SAASJ;QACf,MAAMK,WAAWH,UAAUC,cAAcC;QAEzC,IAAI,CAAC,eAAe,CAAC;QAErB,IAAIpE,QAAsB;QAC1B,IAAIsE,SAAmB,EAAE;QACzB,IAAI;YACF,MAAM,EAAE,OAAOC,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAClEH;YAEF,IAAI,CAAC,WAAW,GAAG,MAAME,SAAS,cAAc;YAChDvE,QAAQuE;YACR,MAAME,yBAAyBzE,MAAM,cAAc;YACnDA,MAAM,cAAc,GAAG,CAAC0E;gBACtB,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,cAAc,GAAGA;gBAExBD,QAAAA,0BAAAA,uBAAyBC;YAC3B;YACAJ,SAAS;mBACHE,aAAa,EAAE;gBACnB;oBACE,MAAM;oBACN,IAAI;wBACF,IAAIxE,OACFA,MAAM,cAAc,GAAGyE;oBAE3B;gBACF;aACD;QACH,EAAE,OAAOE,GAAG;YACV,IAAI,CAAC,eAAe,CAAC,SAASA;YAC9B;QACF;QACA,IAAI,CAAC,cAAc,GAAG3E;QAEtB,IAAIlB,YAAY;QAChB,IAAI,CAAC,eAAe,CAAC;QACrB,IAAI8F,YAAY;QAChB,MAAO9F,YAAYmF,MAAM,MAAM,CAAE;YAC/B,MAAMjF,aAAa,IAAI,CAAC,cAAc,CAACF,UAAU;YACjD,IAAI,CAAC,aAAa,CAACA,WAAW;YAC9B,IAAI,CAAC,YAAY,CAACA;YAElB,IAAI;gBACF,MAAM,IAAI,CAAC,QAAQ,CAACE,YAAY,IAAI,CAAC,cAAc;gBACnD,IAAI,CAAC,aAAa,CAACF,WAAW;YAChC,EAAE,OAAO6F,GAAG;gBACV,IAAI,CAAC,aAAa,CAAC7F,WAAW,SAAgB6F;gBAE9C,IAAI3F,WAAW,eAAe;qBAEvB;oBACL,IAAI,CAAC,UAAU,GAAGgB,MAAM,UAAU;oBAClC4E,YAAY;oBACZ;gBACF;YACF;YACA,IAAI,CAAC,UAAU,GAAG5E,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,UAAU;YACnClB;QACF;QAEA,IAAI8F,WACF,IAAI,CAAC,eAAe,CAAC;aAErB,IAAI,CAAC,eAAe,CAAC;QAEvB,IAAI,CAAC,cAAc,GAAG;QAGtB,KAAK,MAAMC,MAAMP,OACf,IAAI;YAEF,MAAMO,GAAG,EAAE;QAEb,EAAE,OAAOF,GAAG,CAEZ;IAEJ;IApgBA,YACUG,MAA0B,EAC1BC,UAGN,EACKC,kBAAiE,EACxEC,UAAmB,CACnB;YAaWC,cAgBOC,eAKPC;;;;QAxDb,uBAAO,oBAAP;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAO,cAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,sBAAR;QACA,uBAAO,UAAP;QACA,uBAAO,sBAAP;QACA,uBAAO,gBAAP;QACA,uBAAQ,kBAAR;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,eAAR;QACA,uBAAQ,cAAR;aAEUN,MAAM,GAANA;aACAC,UAAU,GAAVA;aAIDC,kBAAkB,GAAlBA;aAnBF,cAAc,GAA6B,EAAE;aAC7C,MAAM,GAA4B;aAGjC,kBAAkB,GAAG;aAIrB,cAAc,GAAiB;aAG/B,WAAW,GAAmB,EAAE;QAWtC,IAAI,CAAC,UAAU,GAAGC;QAClB,IAAI,CAAC,MAAM,GAAG,CAAC;QACf,IAAI,CAAC,MAAM,GACTH,OAAO,MAAM,IACbA,OAAO,GAAG,IACVA,OAAO,OAAO,IACdA,OAAO,GAAG,IACVA,OAAO,MAAM;QAEf,IAAIO,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAAE;YAC7B,IAAI,CAAC,MAAM,GAAG1F;YACdtB,MAAM;QACR,OAAO,IAAI,QAAA4G,CAAAA,eAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,MAAM,EAAE;YAC9B,IAAI,CAAC,MAAM,GAAG9F,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM;YACvDf,MAAM,mCAAmC,IAAI,CAAC,MAAM;QACtD,OAAO;YACL,MAAMiH,aAAa,IAAI,CAAC,UAAU,GAC9BC,AAAAA,IAAAA,mCAAAA,QAAAA,AAAAA,EAAS,IAAI,CAAC,UAAU,EAAE,SAAS,OAAO,CAAC,eAAe,MAC1D;YACJ,IAAI,CAAC,MAAM,GAAGC,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACZC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB,GAAGH,WAAW,CAAC,EAAEI,KAAK,GAAG,GAAG,KAAK,CAAC;YAEpCrH,MAAM,iCAAiC,IAAI,CAAC,MAAM;QACpD;QAEA,IAAI+G,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAC3B,IAAI,CAAC,kBAAkB,GAAG1F;aACrB,IAAI,AAA2C,YAA3C,gBAAOuF,CAAAA,gBAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,kBAAkB,AAAD,GAC9C,IAAI,CAAC,kBAAkB,GAAG/F,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EACxBC,QAAQ,GAAG,IACX,IAAI,CAAC,MAAM,CAAC,kBAAkB;aAE3B,IAAI+F,AAAAA,SAAAA,CAAAA,gBAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,kBAAkB,AAAD,MAAM,MAC7C,IAAI,CAAC,kBAAkB,GAAGK,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACxBC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB;QAIJ,IAAI,CAAC,cAAc,GAAIZ,AAAAA,CAAAA,OAAO,KAAK,IAAI,EAAC,EAAG,GAAG,CAAC,CAACc,MAAM9G;gBAIxC+G;mBAJuD;gBACnE,GAAGD,IAAI;gBACP,OAAO9G;gBACP,QAAQ;gBACR,YAAY+G,AAAAA,SAAAA,CAAAA,aAAAA,KAAK,IAAI,AAAD,IAARA,KAAAA,IAAAA,WAAW,MAAM,AAAD,KAAK;YACnC;;IACF;AA8cF"}
+{"version":3,"file":"yaml/player.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/yaml/player.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { basename, dirname, join, resolve } from 'node:path';\nimport { assert, ifInBrowser, ifInWorker } from '@midscene/shared/utils';\nimport { type ZodTypeAny, z } from 'zod';\n\n// previous defined yaml flow, as a helper\ninterface MidsceneYamlFlowItemAIInput extends LocateOption {\n  // previous version\n  // aiInput: string; // value to input\n  // locate: TUserPrompt; // where to input\n  aiInput: TUserPrompt | undefined; // where to input\n  value: string | number; // value to input\n}\n\ninterface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {\n  // previous version\n  // aiKeyboardPress: string;\n  // locate?: TUserPrompt; // where to press, optional\n  aiKeyboardPress: TUserPrompt | undefined; // where to press\n  keyName: string; // key to press\n}\n\ninterface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {\n  // previous version\n  // aiScroll: null;\n  // locate?: TUserPrompt; // which area to scroll, optional\n  aiScroll: TUserPrompt | undefined; // which area to scroll\n}\n\nimport type { Agent } from '@/agent/agent';\nimport type { TUserPrompt } from '@/ai-model/common';\nimport type {\n  DeviceAction,\n  FreeFn,\n  LocateOption,\n  MidsceneYamlFlowItemAIAction,\n  MidsceneYamlFlowItemAIAsk,\n  MidsceneYamlFlowItemAIAssert,\n  MidsceneYamlFlowItemAIBoolean,\n  MidsceneYamlFlowItemAILocate,\n  MidsceneYamlFlowItemAINumber,\n  MidsceneYamlFlowItemAIQuery,\n  MidsceneYamlFlowItemAIString,\n  MidsceneYamlFlowItemAIWaitFor,\n  MidsceneYamlFlowItemEvaluateJavaScript,\n  MidsceneYamlFlowItemLogScreenshot,\n  MidsceneYamlFlowItemSleep,\n  MidsceneYamlScript,\n  MidsceneYamlScriptEnv,\n  ScriptPlayerStatusValue,\n  ScriptPlayerTaskStatus,\n  ScrollParam,\n} from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { getDebug } from '@midscene/shared/logger';\nimport {\n  buildDetailedLocateParam,\n  buildDetailedLocateParamAndRestParams,\n} from './utils';\n\nconst debug = getDebug('yaml-player');\n\nconst isStringParamSchema = (schema?: ZodTypeAny): boolean => {\n  if (!schema) {\n    return false;\n  }\n\n  const schemaDef = (schema as any)?._def;\n  if (!schemaDef?.typeName) {\n    return false;\n  }\n\n  switch (schemaDef.typeName) {\n    case z.ZodFirstPartyTypeKind.ZodString:\n    case z.ZodFirstPartyTypeKind.ZodEnum:\n    case z.ZodFirstPartyTypeKind.ZodNativeEnum:\n      return true;\n    case z.ZodFirstPartyTypeKind.ZodLiteral:\n      return typeof schemaDef.value === 'string';\n    case z.ZodFirstPartyTypeKind.ZodOptional:\n    case z.ZodFirstPartyTypeKind.ZodNullable:\n    case z.ZodFirstPartyTypeKind.ZodDefault:\n      return isStringParamSchema(schemaDef.innerType);\n    case z.ZodFirstPartyTypeKind.ZodEffects:\n      return isStringParamSchema(schemaDef.schema);\n    case z.ZodFirstPartyTypeKind.ZodPipeline:\n      return isStringParamSchema(schemaDef.out);\n    case z.ZodFirstPartyTypeKind.ZodUnion: {\n      const options = schemaDef.options as ZodTypeAny[] | undefined;\n      return Array.isArray(options)\n        ? options.every((option) => isStringParamSchema(option))\n        : false;\n    }\n    default:\n      return false;\n  }\n};\nexport class ScriptPlayer<T extends MidsceneYamlScriptEnv> {\n  public currentTaskIndex?: number;\n  public taskStatusList: ScriptPlayerTaskStatus[] = [];\n  public status: ScriptPlayerStatusValue = 'init';\n  public reportFile?: string | null;\n  public result: Record<string, any>;\n  private unnamedResultIndex = 0;\n  public output?: string | null;\n  public unstableLogContent?: string | null;\n  public errorInSetup?: Error;\n  private interfaceAgent: Agent | null = null;\n  public agentStatusTip?: string;\n  public target?: MidsceneYamlScriptEnv;\n  private actionSpace: DeviceAction[] = [];\n  private scriptPath?: string;\n  constructor(\n    private script: MidsceneYamlScript,\n    private setupAgent: (platform: T) => Promise<{\n      agent: Agent;\n      freeFn: FreeFn[];\n    }>,\n    public onTaskStatusChange?: (taskStatus: ScriptPlayerTaskStatus) => void,\n    scriptPath?: string,\n  ) {\n    this.scriptPath = scriptPath;\n    this.result = {};\n    this.target =\n      script.target ||\n      script.web ||\n      script.android ||\n      script.ios ||\n      script.config;\n\n    if (ifInBrowser || ifInWorker) {\n      this.output = undefined;\n      debug('output is undefined in browser or worker');\n    } else if (this.target?.output) {\n      this.output = resolve(process.cwd(), this.target.output);\n      debug('setting output by config.output', this.output);\n    } else {\n      const scriptName = this.scriptPath\n        ? basename(this.scriptPath, '.yaml').replace(/\\.(ya?ml)$/i, '')\n        : 'script';\n      this.output = join(\n        getMidsceneRunSubDir('output'),\n        `${scriptName}-${Date.now()}.json`,\n      );\n      debug('setting output by script path', this.output);\n    }\n\n    if (ifInBrowser || ifInWorker) {\n      this.unstableLogContent = undefined;\n    } else if (typeof this.target?.unstableLogContent === 'string') {\n      this.unstableLogContent = resolve(\n        process.cwd(),\n        this.target.unstableLogContent,\n      );\n    } else if (this.target?.unstableLogContent === true) {\n      this.unstableLogContent = join(\n        getMidsceneRunSubDir('output'),\n        'unstableLogContent.json',\n      );\n    }\n\n    this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({\n      ...task,\n      index: taskIndex,\n      status: 'init',\n      totalSteps: task.flow?.length || 0,\n    }));\n  }\n\n  private setResult(key: string | undefined, value: any) {\n    const keyToUse = key || this.unnamedResultIndex++;\n    if (this.result[keyToUse]) {\n      console.warn(`result key ${keyToUse} already exists, will overwrite`);\n    }\n    this.result[keyToUse] = value;\n\n    return this.flushResult();\n  }\n\n  private setPlayerStatus(status: ScriptPlayerStatusValue, error?: Error) {\n    this.status = status;\n    this.errorInSetup = error;\n  }\n\n  private notifyCurrentTaskStatusChange(taskIndex?: number) {\n    const taskIndexToNotify =\n      typeof taskIndex === 'number' ? taskIndex : this.currentTaskIndex;\n\n    if (typeof taskIndexToNotify !== 'number') {\n      return;\n    }\n\n    const taskStatus = this.taskStatusList[taskIndexToNotify];\n    if (this.onTaskStatusChange) {\n      this.onTaskStatusChange(taskStatus);\n    }\n  }\n\n  private async setTaskStatus(\n    index: number,\n    statusValue: ScriptPlayerStatusValue,\n    error?: Error,\n  ) {\n    this.taskStatusList[index].status = statusValue;\n    if (error) {\n      this.taskStatusList[index].error = error;\n    }\n\n    this.notifyCurrentTaskStatusChange(index);\n  }\n\n  private setTaskIndex(taskIndex: number) {\n    this.currentTaskIndex = taskIndex;\n  }\n\n  private flushResult() {\n    if (this.output) {\n      const output = resolve(process.cwd(), this.output);\n      const outputDir = dirname(output);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(output, JSON.stringify(this.result || {}, undefined, 2));\n    }\n  }\n\n  private flushUnstableLogContent() {\n    if (this.unstableLogContent) {\n      const content = this.interfaceAgent?._unstableLogContent();\n      const filePath = resolve(process.cwd(), this.unstableLogContent);\n      const outputDir = dirname(filePath);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(filePath, JSON.stringify(content, null, 2));\n    }\n  }\n\n  async playTask(taskStatus: ScriptPlayerTaskStatus, agent: Agent) {\n    const { flow } = taskStatus;\n    assert(flow, 'missing flow in task');\n\n    for (const flowItemIndex in flow) {\n      const currentStep = Number.parseInt(flowItemIndex, 10);\n      taskStatus.currentStep = currentStep;\n      const flowItem = flow[flowItemIndex];\n      debug(\n        `playing step ${flowItemIndex}, flowItem=${JSON.stringify(flowItem)}`,\n      );\n      if (\n        'aiAct' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n        'aiAction' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n        'ai' in (flowItem as MidsceneYamlFlowItemAIAction)\n      ) {\n        const actionTask = flowItem as MidsceneYamlFlowItemAIAction;\n        const prompt = actionTask.aiAct || actionTask.aiAction || actionTask.ai;\n        assert(prompt, 'missing prompt for ai (aiAct)');\n        await agent.aiAct(prompt, {\n          cacheable: actionTask.cacheable,\n        });\n      } else if ('aiAssert' in (flowItem as MidsceneYamlFlowItemAIAssert)) {\n        const assertTask = flowItem as MidsceneYamlFlowItemAIAssert;\n        const prompt = assertTask.aiAssert;\n        const msg = assertTask.errorMessage;\n        assert(prompt, 'missing prompt for aiAssert');\n        const { pass, thought, message } =\n          (await agent.aiAssert(prompt, msg, {\n            keepRawResponse: true,\n          })) || {};\n\n        this.setResult(assertTask.name, {\n          pass,\n          thought,\n          message,\n        });\n\n        if (!pass) {\n          throw new Error(message);\n        }\n      } else if ('aiQuery' in (flowItem as MidsceneYamlFlowItemAIQuery)) {\n        const queryTask = flowItem as MidsceneYamlFlowItemAIQuery;\n        const prompt = queryTask.aiQuery;\n        const options = {\n          domIncluded: queryTask.domIncluded,\n          screenshotIncluded: queryTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiQuery');\n        const queryResult = await agent.aiQuery(prompt, options);\n        this.setResult(queryTask.name, queryResult);\n      } else if ('aiNumber' in (flowItem as MidsceneYamlFlowItemAINumber)) {\n        const numberTask = flowItem as MidsceneYamlFlowItemAINumber;\n        const prompt = numberTask.aiNumber;\n        const options = {\n          domIncluded: numberTask.domIncluded,\n          screenshotIncluded: numberTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiNumber');\n        const numberResult = await agent.aiNumber(prompt, options);\n        this.setResult(numberTask.name, numberResult);\n      } else if ('aiString' in (flowItem as MidsceneYamlFlowItemAIString)) {\n        const stringTask = flowItem as MidsceneYamlFlowItemAIString;\n        const prompt = stringTask.aiString;\n        const options = {\n          domIncluded: stringTask.domIncluded,\n          screenshotIncluded: stringTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiString');\n        const stringResult = await agent.aiString(prompt, options);\n        this.setResult(stringTask.name, stringResult);\n      } else if ('aiBoolean' in (flowItem as MidsceneYamlFlowItemAIBoolean)) {\n        const booleanTask = flowItem as MidsceneYamlFlowItemAIBoolean;\n        const prompt = booleanTask.aiBoolean;\n        const options = {\n          domIncluded: booleanTask.domIncluded,\n          screenshotIncluded: booleanTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiBoolean');\n        const booleanResult = await agent.aiBoolean(prompt, options);\n        this.setResult(booleanTask.name, booleanResult);\n      } else if ('aiAsk' in (flowItem as MidsceneYamlFlowItemAIAsk)) {\n        const askTask = flowItem as MidsceneYamlFlowItemAIAsk;\n        const prompt = askTask.aiAsk;\n        assert(prompt, 'missing prompt for aiAsk');\n        const askResult = await agent.aiAsk(prompt);\n        this.setResult(askTask.name, askResult);\n      } else if ('aiLocate' in (flowItem as MidsceneYamlFlowItemAILocate)) {\n        const locateTask = flowItem as MidsceneYamlFlowItemAILocate;\n        const prompt = locateTask.aiLocate;\n        assert(prompt, 'missing prompt for aiLocate');\n        const locateResult = await agent.aiLocate(prompt, locateTask);\n        this.setResult(locateTask.name, locateResult);\n      } else if ('aiWaitFor' in (flowItem as MidsceneYamlFlowItemAIWaitFor)) {\n        const waitForTask = flowItem as MidsceneYamlFlowItemAIWaitFor;\n        const prompt = waitForTask.aiWaitFor;\n        assert(prompt, 'missing prompt for aiWaitFor');\n        const timeout = waitForTask.timeout;\n        await agent.aiWaitFor(prompt, { timeoutMs: timeout });\n      } else if ('sleep' in (flowItem as MidsceneYamlFlowItemSleep)) {\n        const sleepTask = flowItem as MidsceneYamlFlowItemSleep;\n        const ms = sleepTask.sleep;\n        let msNumber = ms;\n        if (typeof ms === 'string') {\n          msNumber = Number.parseInt(ms, 10);\n        }\n        assert(\n          msNumber && msNumber > 0,\n          `ms for sleep must be greater than 0, but got ${ms}`,\n        );\n        await new Promise((resolve) => setTimeout(resolve, msNumber));\n      } else if (\n        'javascript' in (flowItem as MidsceneYamlFlowItemEvaluateJavaScript)\n      ) {\n        const evaluateJavaScriptTask =\n          flowItem as MidsceneYamlFlowItemEvaluateJavaScript;\n\n        const result = await agent.evaluateJavaScript(\n          evaluateJavaScriptTask.javascript,\n        );\n        this.setResult(evaluateJavaScriptTask.name, result);\n      } else if (\n        'logScreenshot' in (flowItem as MidsceneYamlFlowItemLogScreenshot) ||\n        'recordToReport' in (flowItem as MidsceneYamlFlowItemLogScreenshot)\n      ) {\n        const recordTask = flowItem as MidsceneYamlFlowItemLogScreenshot;\n        const title =\n          recordTask.recordToReport ?? recordTask.logScreenshot ?? 'untitled';\n        const content = recordTask.content || '';\n        await agent.recordToReport(title, { content });\n      } else if ('aiInput' in (flowItem as MidsceneYamlFlowItemAIInput)) {\n        // may be input empty string ''\n        const {\n          aiInput,\n          value: rawValue,\n          ...inputTask\n        } = flowItem as MidsceneYamlFlowItemAIInput;\n\n        // Compatibility with previous version:\n        // Old format: { aiInput: string (value), locate: TUserPrompt }\n        // New format - 1: { aiInput: TUserPrompt, value: string | number }\n        // New format - 2: { aiInput: undefined, locate: TUserPrompt, value: string | number }\n        let locatePrompt: TUserPrompt | undefined;\n        let value: string | number | undefined;\n        if ((inputTask as any).locate) {\n          // Old format - aiInput is the value, locate is the prompt\n          // Keep backward compatibility: empty string is treated as no value\n          value = (aiInput as string | number) || rawValue;\n          locatePrompt = (inputTask as any).locate;\n        } else {\n          // New format - aiInput is the prompt, value is the value\n          locatePrompt = aiInput || '';\n          value = rawValue;\n        }\n\n        // Convert value to string for Input action\n        await agent.callActionInActionSpace('Input', {\n          ...inputTask,\n          ...(value !== undefined ? { value: String(value) } : {}),\n          ...(locatePrompt\n            ? { locate: buildDetailedLocateParam(locatePrompt, inputTask) }\n            : {}),\n        });\n      } else if (\n        'aiKeyboardPress' in (flowItem as MidsceneYamlFlowItemAIKeyboardPress)\n      ) {\n        const { aiKeyboardPress, ...keyboardPressTask } =\n          flowItem as MidsceneYamlFlowItemAIKeyboardPress;\n\n        // Compatibility with previous version:\n        // Old format: { aiKeyboardPress: string (key), locate?: TUserPrompt }\n        // New format - 1: { aiKeyboardPress: TUserPrompt, keyName: string }\n        // New format - 2: { aiKeyboardPress: , locate?: TUserPrompt, keyName: string }\n        let locatePrompt: TUserPrompt | undefined;\n        let keyName: string | undefined;\n        if ((keyboardPressTask as any).locate) {\n          // Old format - aiKeyboardPress is the key, locate is the prompt\n          keyName = aiKeyboardPress as string;\n          locatePrompt = (keyboardPressTask as any).locate;\n        } else if (keyboardPressTask.keyName) {\n          // New format - aiKeyboardPress is the prompt, key is the key\n          keyName = keyboardPressTask.keyName;\n          locatePrompt = aiKeyboardPress;\n        } else {\n          keyName = aiKeyboardPress as string;\n        }\n\n        await agent.callActionInActionSpace('KeyboardPress', {\n          ...keyboardPressTask,\n          ...(keyName ? { keyName } : {}),\n          ...(locatePrompt\n            ? {\n                locate: buildDetailedLocateParam(\n                  locatePrompt,\n                  keyboardPressTask,\n                ),\n              }\n            : {}),\n        });\n      } else if ('aiScroll' in (flowItem as MidsceneYamlFlowItemAIScroll)) {\n        const { aiScroll, ...scrollTask } =\n          flowItem as MidsceneYamlFlowItemAIScroll;\n\n        // Compatibility with previous version:\n        // Old format: { aiScroll: null, locate?: TUserPrompt, direction, scrollType, distance? }\n        // New format - 1: { aiScroll: TUserPrompt, direction, scrollType, distance? }\n        // New format - 2: { aiScroll: undefined, locate: TUserPrompt, direction, scrollType, distance? }\n        let locatePrompt: TUserPrompt | undefined;\n        if ((scrollTask as any).locate) {\n          // Old format - locate is the prompt, aiScroll is null/ignored\n          locatePrompt = (scrollTask as any).locate;\n        } else {\n          // New format - aiScroll is the prompt, or no prompt for global scroll\n          locatePrompt = aiScroll;\n        }\n\n        await agent.callActionInActionSpace('Scroll', {\n          ...scrollTask,\n          ...(locatePrompt\n            ? { locate: buildDetailedLocateParam(locatePrompt, scrollTask) }\n            : {}),\n        });\n      } else {\n        // generic action, find the action in actionSpace\n\n        /* for aiTap, aiRightClick, the parameters are a flattened data for the 'locate', these are all valid data\n\n        - aiTap: 'search input box'\n        - aiTap: 'search input box'\n          deepThink: true\n          cacheable: false\n        - aiTap:\n          prompt: 'search input box'\n        - aiTap:\n          prompt: 'search input box'\n          deepThink: true\n          cacheable: false\n        */\n\n        const actionSpace = this.actionSpace;\n        let locatePromptShortcut: string | undefined;\n        let actionParamForMatchedAction: unknown;\n        const matchedAction = actionSpace.find((action) => {\n          const actionInterfaceAlias = action.interfaceAlias;\n          if (\n            actionInterfaceAlias &&\n            Object.prototype.hasOwnProperty.call(flowItem, actionInterfaceAlias)\n          ) {\n            actionParamForMatchedAction =\n              flowItem[actionInterfaceAlias as keyof typeof flowItem];\n            if (typeof actionParamForMatchedAction === 'string') {\n              locatePromptShortcut = actionParamForMatchedAction;\n            }\n            return true;\n          }\n\n          const keyOfActionInActionSpace = action.name;\n          if (\n            Object.prototype.hasOwnProperty.call(\n              flowItem,\n              keyOfActionInActionSpace,\n            )\n          ) {\n            actionParamForMatchedAction =\n              flowItem[keyOfActionInActionSpace as keyof typeof flowItem];\n            if (typeof actionParamForMatchedAction === 'string') {\n              locatePromptShortcut = actionParamForMatchedAction;\n            }\n            return true;\n          }\n\n          return false;\n        });\n\n        assert(\n          matchedAction,\n          `unknown flowItem in yaml: ${JSON.stringify(flowItem)}`,\n        );\n\n        const schemaIsStringParam = isStringParamSchema(\n          matchedAction.paramSchema,\n        );\n        let stringParamToCall: string | undefined;\n        if (\n          typeof actionParamForMatchedAction === 'string' &&\n          schemaIsStringParam\n        ) {\n          if (matchedAction.paramSchema) {\n            const parseResult = matchedAction.paramSchema.safeParse(\n              actionParamForMatchedAction,\n            );\n            if (parseResult.success && typeof parseResult.data === 'string') {\n              stringParamToCall = parseResult.data;\n            } else if (!parseResult.success) {\n              debug(\n                `parse failed for action ${matchedAction.name} with string param`,\n                parseResult.error,\n              );\n              stringParamToCall = actionParamForMatchedAction;\n            }\n          } else {\n            stringParamToCall = actionParamForMatchedAction;\n          }\n        }\n\n        if (stringParamToCall !== undefined) {\n          debug(\n            `matchedAction: ${matchedAction.name}`,\n            `flowParams: ${JSON.stringify(stringParamToCall)}`,\n          );\n          const result = await agent.callActionInActionSpace(\n            matchedAction.name,\n            stringParamToCall,\n          );\n\n          // Store result if there's a name property in flowItem\n          const resultName = (flowItem as any).name;\n          if (result !== undefined) {\n            this.setResult(resultName, result);\n          }\n        } else {\n          // Determine the source for parameter extraction:\n          // - If we have a locatePromptShortcut, use the flowItem (for actions like aiTap with prompt)\n          // - Otherwise, use actionParamForMatchedAction (for actions like runWdaRequest with structured params)\n          const sourceForParams =\n            locatePromptShortcut &&\n            typeof actionParamForMatchedAction === 'string'\n              ? { ...flowItem, prompt: locatePromptShortcut }\n              : typeof actionParamForMatchedAction === 'object' &&\n                  actionParamForMatchedAction !== null\n                ? actionParamForMatchedAction\n                : flowItem;\n\n          const { locateParam, restParams } =\n            buildDetailedLocateParamAndRestParams(\n              locatePromptShortcut || '',\n              sourceForParams as LocateOption,\n              [\n                matchedAction.name,\n                matchedAction.interfaceAlias || '_never_mind_',\n              ],\n            );\n\n          const flowParams = {\n            ...restParams,\n            locate: locateParam,\n          };\n\n          debug(\n            `matchedAction: ${matchedAction.name}`,\n            `flowParams: ${JSON.stringify(flowParams, null, 2)}`,\n          );\n          const result = await agent.callActionInActionSpace(\n            matchedAction.name,\n            flowParams,\n          );\n\n          // Store result if there's a name property in flowItem\n          const resultName = (flowItem as any).name;\n          if (result !== undefined) {\n            this.setResult(resultName, result);\n          }\n        }\n      }\n    }\n    this.reportFile = agent.reportFile;\n    await this.flushUnstableLogContent();\n  }\n\n  async run() {\n    const { target, web, android, ios, tasks } = this.script;\n    const webEnv = web || target;\n    const androidEnv = android;\n    const iosEnv = ios;\n    const platform = webEnv || androidEnv || iosEnv;\n\n    this.setPlayerStatus('running');\n\n    let agent: Agent | null = null;\n    let freeFn: FreeFn[] = [];\n    try {\n      const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(\n        platform as T,\n      );\n      this.actionSpace = await newAgent.getActionSpace();\n      agent = newAgent;\n      const originalOnTaskStartTip = agent.onTaskStartTip;\n      agent.onTaskStartTip = (tip) => {\n        if (this.status === 'running') {\n          this.agentStatusTip = tip;\n        }\n        originalOnTaskStartTip?.(tip);\n      };\n      freeFn = [\n        ...(newFreeFn || []),\n        {\n          name: 'restore-agent-onTaskStartTip',\n          fn: () => {\n            if (agent) {\n              agent.onTaskStartTip = originalOnTaskStartTip;\n            }\n          },\n        },\n      ];\n    } catch (e) {\n      this.setPlayerStatus('error', e as Error);\n      return;\n    }\n    this.interfaceAgent = agent;\n\n    let taskIndex = 0;\n    this.setPlayerStatus('running');\n    let errorFlag = false;\n    while (taskIndex < tasks.length) {\n      const taskStatus = this.taskStatusList[taskIndex];\n      this.setTaskStatus(taskIndex, 'running' as any);\n      this.setTaskIndex(taskIndex);\n\n      try {\n        await this.playTask(taskStatus, this.interfaceAgent);\n        this.setTaskStatus(taskIndex, 'done' as any);\n      } catch (e) {\n        this.setTaskStatus(taskIndex, 'error' as any, e as Error);\n\n        if (taskStatus.continueOnError) {\n          // nothing more to do\n        } else {\n          this.reportFile = agent.reportFile;\n          errorFlag = true;\n          break;\n        }\n      }\n      this.reportFile = agent?.reportFile;\n      taskIndex++;\n    }\n\n    if (errorFlag) {\n      this.setPlayerStatus('error');\n    } else {\n      this.setPlayerStatus('done');\n    }\n    this.agentStatusTip = '';\n\n    // free the resources\n    for (const fn of freeFn) {\n      try {\n        // console.log('freeing', fn.name);\n        await fn.fn();\n        // console.log('freed', fn.name);\n      } catch (e) {\n        // console.error('error freeing', fn.name, e);\n      }\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","isStringParamSchema","schema","schemaDef","z","options","Array","option","ScriptPlayer","value","keyToUse","console","status","error","taskIndex","taskIndexToNotify","taskStatus","index","statusValue","output","resolve","process","outputDir","dirname","existsSync","mkdirSync","writeFileSync","JSON","undefined","_this_interfaceAgent","content","filePath","agent","flow","assert","flowItemIndex","currentStep","Number","flowItem","actionTask","prompt","assertTask","msg","pass","thought","message","Error","queryTask","queryResult","numberTask","numberResult","stringTask","stringResult","booleanTask","booleanResult","askTask","askResult","locateTask","locateResult","waitForTask","timeout","sleepTask","ms","msNumber","Promise","setTimeout","evaluateJavaScriptTask","result","recordTask","title","aiInput","rawValue","inputTask","locatePrompt","String","buildDetailedLocateParam","aiKeyboardPress","keyboardPressTask","keyName","aiScroll","scrollTask","actionSpace","locatePromptShortcut","actionParamForMatchedAction","matchedAction","action","actionInterfaceAlias","keyOfActionInActionSpace","schemaIsStringParam","stringParamToCall","parseResult","resultName","sourceForParams","locateParam","restParams","buildDetailedLocateParamAndRestParams","flowParams","target","web","android","ios","tasks","webEnv","androidEnv","iosEnv","platform","freeFn","newAgent","newFreeFn","originalOnTaskStartTip","tip","e","errorFlag","fn","script","setupAgent","onTaskStatusChange","scriptPath","_this_target","_this_target1","_this_target2","ifInBrowser","ifInWorker","scriptName","basename","join","getMidsceneRunSubDir","Date","task","_task_flow"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACsDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEvB,MAAMC,sBAAsB,CAACC;IAC3B,IAAI,CAACA,QACH,OAAO;IAGT,MAAMC,YAAaD,QAAAA,SAAAA,KAAAA,IAAAA,OAAgB,IAAI;IACvC,IAAI,CAACC,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,QAAQ,AAAD,GACrB,OAAO;IAGT,OAAQA,UAAU,QAAQ;QACxB,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,SAAiC;QACtC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,OAA+B;QACpC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,aAAqC;YACxC,OAAO;QACT,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,UAAkC;YACrC,OAAO,AAA2B,YAA3B,OAAOD,UAAU,KAAK;QAC/B,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,WAAmC;QACxC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,WAAmC;QACxC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,UAAkC;YACrC,OAAOH,oBAAoBE,UAAU,SAAS;QAChD,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,UAAkC;YACrC,OAAOH,oBAAoBE,UAAU,MAAM;QAC7C,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,WAAmC;YACtC,OAAOH,oBAAoBE,UAAU,GAAG;QAC1C,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,QAAgC;YAAE;gBACrC,MAAMC,UAAUF,UAAU,OAAO;gBACjC,OAAOG,MAAM,OAAO,CAACD,WACjBA,QAAQ,KAAK,CAAC,CAACE,SAAWN,oBAAoBM,WAC9C;YACN;QACA;YACE,OAAO;IACX;AACF;AACO,MAAMC;IAwEH,UAAUd,GAAuB,EAAEe,KAAU,EAAE;QACrD,MAAMC,WAAWhB,OAAO,IAAI,CAAC,kBAAkB;QAC/C,IAAI,IAAI,CAAC,MAAM,CAACgB,SAAS,EACvBC,QAAQ,IAAI,CAAC,CAAC,WAAW,EAAED,SAAS,+BAA+B,CAAC;QAEtE,IAAI,CAAC,MAAM,CAACA,SAAS,GAAGD;QAExB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEQ,gBAAgBG,MAA+B,EAAEC,KAAa,EAAE;QACtE,IAAI,CAAC,MAAM,GAAGD;QACd,IAAI,CAAC,YAAY,GAAGC;IACtB;IAEQ,8BAA8BC,SAAkB,EAAE;QACxD,MAAMC,oBACJ,AAAqB,YAArB,OAAOD,YAAyBA,YAAY,IAAI,CAAC,gBAAgB;QAEnE,IAAI,AAA6B,YAA7B,OAAOC,mBACT;QAGF,MAAMC,aAAa,IAAI,CAAC,cAAc,CAACD,kBAAkB;QACzD,IAAI,IAAI,CAAC,kBAAkB,EACzB,IAAI,CAAC,kBAAkB,CAACC;IAE5B;IAEA,MAAc,cACZC,KAAa,EACbC,WAAoC,EACpCL,KAAa,EACb;QACA,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,MAAM,GAAGC;QACpC,IAAIL,OACF,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,KAAK,GAAGJ;QAGrC,IAAI,CAAC,6BAA6B,CAACI;IACrC;IAEQ,aAAaH,SAAiB,EAAE;QACtC,IAAI,CAAC,gBAAgB,GAAGA;IAC1B;IAEQ,cAAc;QACpB,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,MAAMK,SAASC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM;YACjD,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQJ;YAC1B,IAAI,CAACK,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcP,QAAQQ,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,GAAGC,QAAW;QACrE;IACF;IAEQ,0BAA0B;QAChC,IAAI,IAAI,CAAC,kBAAkB,EAAE;gBACXC;YAAhB,MAAMC,UAAU,QAAAD,CAAAA,uBAAAA,IAAI,CAAC,cAAc,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,mBAAmB;YACxD,MAAME,WAAWX,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,kBAAkB;YAC/D,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQQ;YAC1B,IAAI,CAACP,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcK,UAAUJ,KAAK,SAAS,CAACG,SAAS,MAAM;QACxD;IACF;IAEA,MAAM,SAASd,UAAkC,EAAEgB,KAAY,EAAE;QAC/D,MAAM,EAAEC,IAAI,EAAE,GAAGjB;QACjBkB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,MAAM;QAEb,IAAK,MAAME,iBAAiBF,KAAM;YAChC,MAAMG,cAAcC,OAAO,QAAQ,CAACF,eAAe;YACnDnB,WAAW,WAAW,GAAGoB;YACzB,MAAME,WAAWL,IAAI,CAACE,cAAc;YACpCpC,MACE,CAAC,aAAa,EAAEoC,cAAc,WAAW,EAAER,KAAK,SAAS,CAACW,WAAW;YAEvE,IACE,WAAYA,YACZ,cAAeA,YACf,QAASA,UACT;gBACA,MAAMC,aAAaD;gBACnB,MAAME,SAASD,WAAW,KAAK,IAAIA,WAAW,QAAQ,IAAIA,WAAW,EAAE;gBACvEL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMR,MAAM,KAAK,CAACQ,QAAQ;oBACxB,WAAWD,WAAW,SAAS;gBACjC;YACF,OAAO,IAAI,cAAeD,UAA2C;gBACnE,MAAMG,aAAaH;gBACnB,MAAME,SAASC,WAAW,QAAQ;gBAClC,MAAMC,MAAMD,WAAW,YAAY;gBACnCP,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAM,EAAEG,IAAI,EAAEC,OAAO,EAAEC,OAAO,EAAE,GAC7B,MAAMb,MAAM,QAAQ,CAACQ,QAAQE,KAAK;oBACjC,iBAAiB;gBACnB,MAAO,CAAC;gBAEV,IAAI,CAAC,SAAS,CAACD,WAAW,IAAI,EAAE;oBAC9BE;oBACAC;oBACAC;gBACF;gBAEA,IAAI,CAACF,MACH,MAAM,IAAIG,MAAMD;YAEpB,OAAO,IAAI,aAAcP,UAA0C;gBACjE,MAAMS,YAAYT;gBAClB,MAAME,SAASO,UAAU,OAAO;gBAChC,MAAM1C,UAAU;oBACd,aAAa0C,UAAU,WAAW;oBAClC,oBAAoBA,UAAU,kBAAkB;gBAClD;gBACAb,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMQ,cAAc,MAAMhB,MAAM,OAAO,CAACQ,QAAQnC;gBAChD,IAAI,CAAC,SAAS,CAAC0C,UAAU,IAAI,EAAEC;YACjC,OAAO,IAAI,cAAeV,UAA2C;gBACnE,MAAMW,aAAaX;gBACnB,MAAME,SAASS,WAAW,QAAQ;gBAClC,MAAM5C,UAAU;oBACd,aAAa4C,WAAW,WAAW;oBACnC,oBAAoBA,WAAW,kBAAkB;gBACnD;gBACAf,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMU,eAAe,MAAMlB,MAAM,QAAQ,CAACQ,QAAQnC;gBAClD,IAAI,CAAC,SAAS,CAAC4C,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,cAAeZ,UAA2C;gBACnE,MAAMa,aAAab;gBACnB,MAAME,SAASW,WAAW,QAAQ;gBAClC,MAAM9C,UAAU;oBACd,aAAa8C,WAAW,WAAW;oBACnC,oBAAoBA,WAAW,kBAAkB;gBACnD;gBACAjB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMY,eAAe,MAAMpB,MAAM,QAAQ,CAACQ,QAAQnC;gBAClD,IAAI,CAAC,SAAS,CAAC8C,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,eAAgBd,UAA4C;gBACrE,MAAMe,cAAcf;gBACpB,MAAME,SAASa,YAAY,SAAS;gBACpC,MAAMhD,UAAU;oBACd,aAAagD,YAAY,WAAW;oBACpC,oBAAoBA,YAAY,kBAAkB;gBACpD;gBACAnB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMc,gBAAgB,MAAMtB,MAAM,SAAS,CAACQ,QAAQnC;gBACpD,IAAI,CAAC,SAAS,CAACgD,YAAY,IAAI,EAAEC;YACnC,OAAO,IAAI,WAAYhB,UAAwC;gBAC7D,MAAMiB,UAAUjB;gBAChB,MAAME,SAASe,QAAQ,KAAK;gBAC5BrB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMgB,YAAY,MAAMxB,MAAM,KAAK,CAACQ;gBACpC,IAAI,CAAC,SAAS,CAACe,QAAQ,IAAI,EAAEC;YAC/B,OAAO,IAAI,cAAelB,UAA2C;gBACnE,MAAMmB,aAAanB;gBACnB,MAAME,SAASiB,WAAW,QAAQ;gBAClCvB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMkB,eAAe,MAAM1B,MAAM,QAAQ,CAACQ,QAAQiB;gBAClD,IAAI,CAAC,SAAS,CAACA,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,eAAgBpB,UAA4C;gBACrE,MAAMqB,cAAcrB;gBACpB,MAAME,SAASmB,YAAY,SAAS;gBACpCzB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMoB,UAAUD,YAAY,OAAO;gBACnC,MAAM3B,MAAM,SAAS,CAACQ,QAAQ;oBAAE,WAAWoB;gBAAQ;YACrD,OAAO,IAAI,WAAYtB,UAAwC;gBAC7D,MAAMuB,YAAYvB;gBAClB,MAAMwB,KAAKD,UAAU,KAAK;gBAC1B,IAAIE,WAAWD;gBACf,IAAI,AAAc,YAAd,OAAOA,IACTC,WAAW1B,OAAO,QAAQ,CAACyB,IAAI;gBAEjC5B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE6B,YAAYA,WAAW,GACvB,CAAC,6CAA6C,EAAED,IAAI;gBAEtD,MAAM,IAAIE,QAAQ,CAAC5C,UAAY6C,WAAW7C,SAAS2C;YACrD,OAAO,IACL,gBAAiBzB,UACjB;gBACA,MAAM4B,yBACJ5B;gBAEF,MAAM6B,SAAS,MAAMnC,MAAM,kBAAkB,CAC3CkC,uBAAuB,UAAU;gBAEnC,IAAI,CAAC,SAAS,CAACA,uBAAuB,IAAI,EAAEC;YAC9C,OAAO,IACL,mBAAoB7B,YACpB,oBAAqBA,UACrB;gBACA,MAAM8B,aAAa9B;gBACnB,MAAM+B,QACJD,WAAW,cAAc,IAAIA,WAAW,aAAa,IAAI;gBAC3D,MAAMtC,UAAUsC,WAAW,OAAO,IAAI;gBACtC,MAAMpC,MAAM,cAAc,CAACqC,OAAO;oBAAEvC;gBAAQ;YAC9C,OAAO,IAAI,aAAcQ,UAA0C;gBAEjE,MAAM,EACJgC,OAAO,EACP,OAAOC,QAAQ,EACf,GAAGC,WACJ,GAAGlC;gBAMJ,IAAImC;gBACJ,IAAIhE;gBACJ,IAAK+D,UAAkB,MAAM,EAAE;oBAG7B/D,QAAS6D,WAA+BC;oBACxCE,eAAgBD,UAAkB,MAAM;gBAC1C,OAAO;oBAELC,eAAeH,WAAW;oBAC1B7D,QAAQ8D;gBACV;gBAGA,MAAMvC,MAAM,uBAAuB,CAAC,SAAS;oBAC3C,GAAGwC,SAAS;oBACZ,GAAI/D,AAAUmB,WAAVnB,QAAsB;wBAAE,OAAOiE,OAAOjE;oBAAO,IAAI,CAAC,CAAC;oBACvD,GAAIgE,eACA;wBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcD;oBAAW,IAC5D,CAAC,CAAC;gBACR;YACF,OAAO,IACL,qBAAsBlC,UACtB;gBACA,MAAM,EAAEsC,eAAe,EAAE,GAAGC,mBAAmB,GAC7CvC;gBAMF,IAAImC;gBACJ,IAAIK;gBACJ,IAAKD,kBAA0B,MAAM,EAAE;oBAErCC,UAAUF;oBACVH,eAAgBI,kBAA0B,MAAM;gBAClD,OAAO,IAAIA,kBAAkB,OAAO,EAAE;oBAEpCC,UAAUD,kBAAkB,OAAO;oBACnCJ,eAAeG;gBACjB,OACEE,UAAUF;gBAGZ,MAAM5C,MAAM,uBAAuB,CAAC,iBAAiB;oBACnD,GAAG6C,iBAAiB;oBACpB,GAAIC,UAAU;wBAAEA;oBAAQ,IAAI,CAAC,CAAC;oBAC9B,GAAIL,eACA;wBACE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACNF,cACAI;oBAEJ,IACA,CAAC,CAAC;gBACR;YACF,OAAO,IAAI,cAAevC,UAA2C;gBACnE,MAAM,EAAEyC,QAAQ,EAAE,GAAGC,YAAY,GAC/B1C;gBAMF,IAAImC;gBAGFA,eAFGO,WAAmB,MAAM,GAEZA,WAAmB,MAAM,GAG1BD;gBAGjB,MAAM/C,MAAM,uBAAuB,CAAC,UAAU;oBAC5C,GAAGgD,UAAU;oBACb,GAAIP,eACA;wBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcO;oBAAY,IAC7D,CAAC,CAAC;gBACR;YACF,OAAO;gBAiBL,MAAMC,cAAc,IAAI,CAAC,WAAW;gBACpC,IAAIC;gBACJ,IAAIC;gBACJ,MAAMC,gBAAgBH,YAAY,IAAI,CAAC,CAACI;oBACtC,MAAMC,uBAAuBD,OAAO,cAAc;oBAClD,IACEC,wBACA3F,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC2C,UAAUgD,uBAC/C;wBACAH,8BACE7C,QAAQ,CAACgD,qBAA8C;wBACzD,IAAI,AAAuC,YAAvC,OAAOH,6BACTD,uBAAuBC;wBAEzB,OAAO;oBACT;oBAEA,MAAMI,2BAA2BF,OAAO,IAAI;oBAC5C,IACE1F,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAClC2C,UACAiD,2BAEF;wBACAJ,8BACE7C,QAAQ,CAACiD,yBAAkD;wBAC7D,IAAI,AAAuC,YAAvC,OAAOJ,6BACTD,uBAAuBC;wBAEzB,OAAO;oBACT;oBAEA,OAAO;gBACT;gBAEAjD,IAAAA,sBAAAA,MAAAA,AAAAA,EACEkD,eACA,CAAC,0BAA0B,EAAEzD,KAAK,SAAS,CAACW,WAAW;gBAGzD,MAAMkD,sBAAsBvF,oBAC1BmF,cAAc,WAAW;gBAE3B,IAAIK;gBACJ,IACE,AAAuC,YAAvC,OAAON,+BACPK,qBAEA,IAAIJ,cAAc,WAAW,EAAE;oBAC7B,MAAMM,cAAcN,cAAc,WAAW,CAAC,SAAS,CACrDD;oBAEF,IAAIO,YAAY,OAAO,IAAI,AAA4B,YAA5B,OAAOA,YAAY,IAAI,EAChDD,oBAAoBC,YAAY,IAAI;yBAC/B,IAAI,CAACA,YAAY,OAAO,EAAE;wBAC/B3F,MACE,CAAC,wBAAwB,EAAEqF,cAAc,IAAI,CAAC,kBAAkB,CAAC,EACjEM,YAAY,KAAK;wBAEnBD,oBAAoBN;oBACtB;gBACF,OACEM,oBAAoBN;gBAIxB,IAAIM,AAAsB7D,WAAtB6D,mBAAiC;oBACnC1F,MACE,CAAC,eAAe,EAAEqF,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAEzD,KAAK,SAAS,CAAC8D,oBAAoB;oBAEpD,MAAMtB,SAAS,MAAMnC,MAAM,uBAAuB,CAChDoD,cAAc,IAAI,EAClBK;oBAIF,MAAME,aAAcrD,SAAiB,IAAI;oBACzC,IAAI6B,AAAWvC,WAAXuC,QACF,IAAI,CAAC,SAAS,CAACwB,YAAYxB;gBAE/B,OAAO;oBAIL,MAAMyB,kBACJV,wBACA,AAAuC,YAAvC,OAAOC,8BACH;wBAAE,GAAG7C,QAAQ;wBAAE,QAAQ4C;oBAAqB,IAC5C,AAAuC,YAAvC,OAAOC,+BACLA,AAAgC,SAAhCA,8BACAA,8BACA7C;oBAER,MAAM,EAAEuD,WAAW,EAAEC,UAAU,EAAE,GAC/BC,AAAAA,IAAAA,kCAAAA,qCAAAA,AAAAA,EACEb,wBAAwB,IACxBU,iBACA;wBACER,cAAc,IAAI;wBAClBA,cAAc,cAAc,IAAI;qBACjC;oBAGL,MAAMY,aAAa;wBACjB,GAAGF,UAAU;wBACb,QAAQD;oBACV;oBAEA9F,MACE,CAAC,eAAe,EAAEqF,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAEzD,KAAK,SAAS,CAACqE,YAAY,MAAM,IAAI;oBAEtD,MAAM7B,SAAS,MAAMnC,MAAM,uBAAuB,CAChDoD,cAAc,IAAI,EAClBY;oBAIF,MAAML,aAAcrD,SAAiB,IAAI;oBACzC,IAAI6B,AAAWvC,WAAXuC,QACF,IAAI,CAAC,SAAS,CAACwB,YAAYxB;gBAE/B;YACF;QACF;QACA,IAAI,CAAC,UAAU,GAAGnC,MAAM,UAAU;QAClC,MAAM,IAAI,CAAC,uBAAuB;IACpC;IAEA,MAAM,MAAM;QACV,MAAM,EAAEiE,MAAM,EAAEC,GAAG,EAAEC,OAAO,EAAEC,GAAG,EAAEC,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM;QACxD,MAAMC,SAASJ,OAAOD;QACtB,MAAMM,aAAaJ;QACnB,MAAMK,SAASJ;QACf,MAAMK,WAAWH,UAAUC,cAAcC;QAEzC,IAAI,CAAC,eAAe,CAAC;QAErB,IAAIxE,QAAsB;QAC1B,IAAI0E,SAAmB,EAAE;QACzB,IAAI;YACF,MAAM,EAAE,OAAOC,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAClEH;YAEF,IAAI,CAAC,WAAW,GAAG,MAAME,SAAS,cAAc;YAChD3E,QAAQ2E;YACR,MAAME,yBAAyB7E,MAAM,cAAc;YACnDA,MAAM,cAAc,GAAG,CAAC8E;gBACtB,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,cAAc,GAAGA;gBAExBD,QAAAA,0BAAAA,uBAAyBC;YAC3B;YACAJ,SAAS;mBACHE,aAAa,EAAE;gBACnB;oBACE,MAAM;oBACN,IAAI;wBACF,IAAI5E,OACFA,MAAM,cAAc,GAAG6E;oBAE3B;gBACF;aACD;QACH,EAAE,OAAOE,GAAG;YACV,IAAI,CAAC,eAAe,CAAC,SAASA;YAC9B;QACF;QACA,IAAI,CAAC,cAAc,GAAG/E;QAEtB,IAAIlB,YAAY;QAChB,IAAI,CAAC,eAAe,CAAC;QACrB,IAAIkG,YAAY;QAChB,MAAOlG,YAAYuF,MAAM,MAAM,CAAE;YAC/B,MAAMrF,aAAa,IAAI,CAAC,cAAc,CAACF,UAAU;YACjD,IAAI,CAAC,aAAa,CAACA,WAAW;YAC9B,IAAI,CAAC,YAAY,CAACA;YAElB,IAAI;gBACF,MAAM,IAAI,CAAC,QAAQ,CAACE,YAAY,IAAI,CAAC,cAAc;gBACnD,IAAI,CAAC,aAAa,CAACF,WAAW;YAChC,EAAE,OAAOiG,GAAG;gBACV,IAAI,CAAC,aAAa,CAACjG,WAAW,SAAgBiG;gBAE9C,IAAI/F,WAAW,eAAe;qBAEvB;oBACL,IAAI,CAAC,UAAU,GAAGgB,MAAM,UAAU;oBAClCgF,YAAY;oBACZ;gBACF;YACF;YACA,IAAI,CAAC,UAAU,GAAGhF,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,UAAU;YACnClB;QACF;QAEA,IAAIkG,WACF,IAAI,CAAC,eAAe,CAAC;aAErB,IAAI,CAAC,eAAe,CAAC;QAEvB,IAAI,CAAC,cAAc,GAAG;QAGtB,KAAK,MAAMC,MAAMP,OACf,IAAI;YAEF,MAAMO,GAAG,EAAE;QAEb,EAAE,OAAOF,GAAG,CAEZ;IAEJ;IAnkBA,YACUG,MAA0B,EAC1BC,UAGN,EACKC,kBAAiE,EACxEC,UAAmB,CACnB;YAaWC,cAgBOC,eAKPC;;;;QAxDb,uBAAO,oBAAP;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAO,cAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,sBAAR;QACA,uBAAO,UAAP;QACA,uBAAO,sBAAP;QACA,uBAAO,gBAAP;QACA,uBAAQ,kBAAR;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,eAAR;QACA,uBAAQ,cAAR;aAEUN,MAAM,GAANA;aACAC,UAAU,GAAVA;aAIDC,kBAAkB,GAAlBA;aAnBF,cAAc,GAA6B,EAAE;aAC7C,MAAM,GAA4B;aAGjC,kBAAkB,GAAG;aAIrB,cAAc,GAAiB;aAG/B,WAAW,GAAmB,EAAE;QAWtC,IAAI,CAAC,UAAU,GAAGC;QAClB,IAAI,CAAC,MAAM,GAAG,CAAC;QACf,IAAI,CAAC,MAAM,GACTH,OAAO,MAAM,IACbA,OAAO,GAAG,IACVA,OAAO,OAAO,IACdA,OAAO,GAAG,IACVA,OAAO,MAAM;QAEf,IAAIO,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAAE;YAC7B,IAAI,CAAC,MAAM,GAAG9F;YACd7B,MAAM;QACR,OAAO,IAAI,QAAAuH,CAAAA,eAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,MAAM,EAAE;YAC9B,IAAI,CAAC,MAAM,GAAGlG,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM;YACvDtB,MAAM,mCAAmC,IAAI,CAAC,MAAM;QACtD,OAAO;YACL,MAAM4H,aAAa,IAAI,CAAC,UAAU,GAC9BC,AAAAA,IAAAA,mCAAAA,QAAAA,AAAAA,EAAS,IAAI,CAAC,UAAU,EAAE,SAAS,OAAO,CAAC,eAAe,MAC1D;YACJ,IAAI,CAAC,MAAM,GAAGC,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACZC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB,GAAGH,WAAW,CAAC,EAAEI,KAAK,GAAG,GAAG,KAAK,CAAC;YAEpChI,MAAM,iCAAiC,IAAI,CAAC,MAAM;QACpD;QAEA,IAAI0H,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAC3B,IAAI,CAAC,kBAAkB,GAAG9F;aACrB,IAAI,AAA2C,YAA3C,gBAAO2F,CAAAA,gBAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,kBAAkB,AAAD,GAC9C,IAAI,CAAC,kBAAkB,GAAGnG,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EACxBC,QAAQ,GAAG,IACX,IAAI,CAAC,MAAM,CAAC,kBAAkB;aAE3B,IAAImG,AAAAA,SAAAA,CAAAA,gBAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,kBAAkB,AAAD,MAAM,MAC7C,IAAI,CAAC,kBAAkB,GAAGK,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACxBC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB;QAIJ,IAAI,CAAC,cAAc,GAAIZ,AAAAA,CAAAA,OAAO,KAAK,IAAI,EAAC,EAAG,GAAG,CAAC,CAACc,MAAMlH;gBAIxCmH;mBAJuD;gBACnE,GAAGD,IAAI;gBACP,OAAOlH;gBACP,QAAQ;gBACR,YAAYmH,AAAAA,SAAAA,CAAAA,aAAAA,KAAK,IAAI,AAAD,IAARA,KAAAA,IAAAA,WAAW,MAAM,AAAD,KAAK;YACnC;;IACF;AA6gBF"}

package/dist/types/agent/agent.d.ts

@@ -1,6 +1,7 @@
-import { type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeviceAction, type ExecutionDump, type GroupedActionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, type ScrollParam, Service, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type TUserPrompt, type UIContext } from '../index';
+import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeviceAction, type ExecutionDump, type GroupedActionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, type ScrollParam, Service, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type TUserPrompt, type UIContext } from '../index';
 export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
 import type { AbstractInterface } from '../device';
+import type { TaskRunner } from '../task-runner';
 import { ModelConfigManager } from '@midscene/shared/env';
 import { TaskCache } from './task-cache';
 import { TaskExecutor } from './tasks';
@@ -37,6 +38,7 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      * Internal promise to deduplicate screenshot scale computation
      */
     private screenshotScalePromise?;
+    private executionDumpIndexByRunner;
     get page(): InterfaceType;
     /**
      * Ensures VL model warning is shown once when needed
@@ -52,12 +54,13 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     _snapshotContext(): Promise<UIContext>;
     setAIActionContext(prompt: string): Promise<void>;
     resetDump(): GroupedActionDump;
-    appendExecutionDump(execution: ExecutionDump): void;
+    appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner): void;
     dumpDataString(): string;
     reportHTMLString(): string;
     writeOutActionDumps(): void;
     private callbackOnTaskStartTip;
-    private afterTaskRunning;
+    private handleRunnerAfterFlush;
+    wrapActionInActionSpace<T extends DeviceAction>(name: string): (param: ActionParam<T>) => Promise<ActionReturn<T>>;
     callActionInActionSpace<T = any>(type: string, opt?: T): Promise<any>;
     aiTap(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
     aiRightClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;

package/dist/types/agent/execution-session.d.ts

@@ -1,13 +1,15 @@
-import { TaskRunner } from '../task-runner';
+import { type TaskExecutionError, TaskRunner } from '../task-runner';
 import type { ExecutionTaskApply, ExecutionTaskProgressOptions, UIContext } from '../types';
+type ExecutionSessionOptions = ExecutionTaskProgressOptions & {
+    tasks?: ExecutionTaskApply[];
+    onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;
+};
 /**
  * Thin wrapper around {@link TaskRunner} that represents a single linear execution run.
  */
 export declare class ExecutionSession {
     private readonly runner;
-    constructor(name: string, contextProvider: () => Promise<UIContext>, options?: ExecutionTaskProgressOptions & {
-        tasks?: ExecutionTaskApply[];
-    });
+    constructor(name: string, contextProvider: () => Promise<UIContext>, options?: ExecutionSessionOptions);
     append(tasks: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void>;
     appendAndRun(tasks: ExecutionTaskApply[] | ExecutionTaskApply): Promise<{
         output: any;
@@ -25,3 +27,4 @@ export declare class ExecutionSession {
     }>;
     getRunner(): TaskRunner;
 }
+export {};

package/dist/types/agent/tasks.d.ts

@@ -2,6 +2,7 @@ import type { TMultimodalPrompt, TUserPrompt } from '../ai-model/common';
 import type { AbstractInterface } from '../device';
 import type Service from '../service';
 import type { TaskRunner } from '../task-runner';
+import { TaskExecutionError } from '../task-runner';
 import type { ExecutionTaskApply, ExecutionTaskProgressOptions, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamSleep, PlanningActionParamWaitFor, ServiceExtractOption, ServiceExtractParam } from '../types';
 import { type IModelConfig } from '@midscene/shared/env';
 import type { TaskCache } from './task-cache';
@@ -11,6 +12,10 @@ interface ExecutionResult<OutputType = any> {
     thought?: string;
     runner: TaskRunner;
 }
+interface TaskExecutorHooks {
+    onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;
+}
+export { TaskExecutionError };
 export declare class TaskExecutor {
     interface: AbstractInterface;
     service: Service;
@@ -18,12 +23,14 @@ export declare class TaskExecutor {
     private readonly taskBuilder;
     private conversationHistory;
     onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];
+    private readonly hooks?;
     replanningCycleLimit?: number;
     get page(): AbstractInterface;
     constructor(interfaceInstance: AbstractInterface, service: Service, opts: {
         taskCache?: TaskCache;
         onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
         replanningCycleLimit?: number;
+        hooks?: TaskExecutorHooks;
     });
     private createExecutionSession;
     convertPlanToExecutable(plans: PlanningAction[], modelConfig: IModelConfig, options?: {

package/dist/types/device/index.d.ts

@@ -22,13 +22,13 @@ export declare abstract class AbstractInterface {
     abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;
     abstract getContext?(): Promise<UIContext>;
 }
-export declare const defineAction: <TSchema extends z.ZodType, TRuntime = z.infer<TSchema>>(config: {
+export declare const defineAction: <TSchema extends z.ZodType, TRuntime = z.infer<TSchema>, TReturn = any>(config: {
     name: string;
     description: string;
     interfaceAlias?: string;
     paramSchema: TSchema;
-    call: (param: TRuntime) => Promise<void>;
-} & Partial<Omit<DeviceAction<TRuntime>, "name" | "description" | "interfaceAlias" | "paramSchema" | "call">>) => DeviceAction<TRuntime>;
+    call: (param: TRuntime) => Promise<TReturn> | TReturn;
+} & Partial<Omit<DeviceAction<TRuntime, TReturn>, "name" | "description" | "interfaceAlias" | "paramSchema" | "call">>) => DeviceAction<TRuntime, TReturn>;
 export declare const actionTapParamSchema: z.ZodObject<{
     locate: z.ZodObject<{
         prompt: z.ZodUnion<[z.ZodString, z.ZodIntersection<z.ZodObject<{

package/dist/types/task-runner.d.ts

@@ -1,13 +1,17 @@
 import type { ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, UIContext } from './types';
+type TaskRunnerInitOptions = ExecutionTaskProgressOptions & {
+    tasks?: ExecutionTaskApply[];
+    onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;
+};
 export declare class TaskRunner {
     name: string;
     tasks: ExecutionTask[];
     status: 'init' | 'pending' | 'running' | 'completed' | 'error';
     onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
     private readonly uiContextBuilder;
-    constructor(name: string, uiContextBuilder: () => Promise<UIContext>, options?: ExecutionTaskProgressOptions & {
-        tasks?: ExecutionTaskApply[];
-    });
+    private readonly onTaskUpdate?;
+    constructor(name: string, uiContextBuilder: () => Promise<UIContext>, options?: TaskRunnerInitOptions);
+    private emitOnTaskUpdate;
     private lastUiContext?;
     private getUiContext;
     private captureScreenshot;
@@ -31,3 +35,11 @@ export declare class TaskRunner {
         runner: TaskRunner;
     }>;
 }
+export declare class TaskExecutionError extends Error {
+    runner: TaskRunner;
+    errorTask: ExecutionTask | null;
+    constructor(message: string, runner: TaskRunner, errorTask: ExecutionTask | null, options?: {
+        cause?: unknown;
+    });
+}
+export {};

package/dist/types/types.d.ts

@@ -229,7 +229,7 @@ export interface ExecutionRecorderItem {
     screenshot?: string;
     timing?: string;
 }
-export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action' | 'Assertion' | 'Log';
+export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';
 export interface ExecutorContext {
     task: ExecutionTask;
     element?: LocateResultElement | null;
@@ -294,7 +294,7 @@ export interface ExecutionTaskInsightAssertionParam {
 }
 export type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightAssertionParam, ServiceAssertionResponse, ExecutionTaskInsightDump>;
 export type ExecutionTaskInsightAssertion = ExecutionTask<ExecutionTaskInsightAssertionApply>;
-export type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<'Action', ActionParam, void, void>;
+export type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<'Action Space', ActionParam, void, void>;
 export type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;
 export type ExecutionTaskLogApply<LogParam = {
     content: string;
@@ -302,8 +302,16 @@ export type ExecutionTaskLogApply<LogParam = {
 export type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;
 export type ExecutionTaskPlanningApply = ExecutionTaskApply<'Planning', {
     userInstruction: string;
+    aiActionContext?: string;
 }, PlanningAIResponse>;
 export type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;
+export type ExecutionTaskPlanningLocateParam = PlanningLocateParam;
+export interface ExecutionTaskPlanningLocateOutput {
+    element: LocateResultElement | null;
+}
+export type ExecutionTaskPlanningDump = ServiceDump;
+export type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<'Planning', ExecutionTaskPlanningLocateParam, ExecutionTaskPlanningLocateOutput, ExecutionTaskPlanningDump>;
+export type ExecutionTaskPlanningLocate = ExecutionTask<ExecutionTaskPlanningLocateApply>;
 export interface GroupedActionDump {
     sdkVersion: string;
     groupName: string;
@@ -343,15 +351,25 @@ export interface StreamingAIResponse {
     /** Whether the response was streamed */
     isStreamed: boolean;
 }
-export interface DeviceAction<T = any> {
+export interface DeviceAction<TParam = any, TReturn = any> {
     name: string;
     description?: string;
     interfaceAlias?: string;
-    paramSchema?: z.ZodType<T>;
-    /** Optional delay (ms) after runner execution. This is useful to wait for the UI to update. Defaults to 300ms. Set to 0 to skip. */
+    paramSchema?: z.ZodType<TParam>;
+    call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;
     delayAfterRunner?: number;
-    call: (param: T, context: ExecutorContext) => Promise<void> | void;
 }
+/**
+ * Type utilities for extracting types from DeviceAction definitions
+ */
+/**
+ * Extract parameter type from a DeviceAction
+ */
+export type ActionParam<Action extends DeviceAction<any, any>> = Action extends DeviceAction<infer P, any> ? P : never;
+/**
+ * Extract return type from a DeviceAction
+ */
+export type ActionReturn<Action extends DeviceAction<any, any>> = Action extends DeviceAction<any, infer R> ? R : never;
 /**
  * Web-specific types
  */

package/dist/types/yaml.d.ts

@@ -12,7 +12,6 @@ export interface LocateOption {
 export interface ServiceExtractOption {
     domIncluded?: boolean | 'visible-only';
     screenshotIncluded?: boolean;
-    doNotThrowError?: boolean;
 }
 export interface ReferenceImage {
     base64: string;
@@ -129,7 +128,7 @@ export interface MidsceneYamlFlowItemLogScreenshot {
     recordToReport?: string;
     content?: string;
 }
-export type MidsceneYamlFlowItem = MidsceneYamlFlowItemAIAction | MidsceneYamlFlowItemAIAssert | MidsceneYamlFlowItemAIQuery | MidsceneYamlFlowItemAIWaitFor | MidsceneYamlFlowItemSleep | MidsceneYamlFlowItemLogScreenshot;
+export type MidsceneYamlFlowItem = MidsceneYamlFlowItemAIAction | MidsceneYamlFlowItemAIAssert | MidsceneYamlFlowItemAIQuery | MidsceneYamlFlowItemAINumber | MidsceneYamlFlowItemAIString | MidsceneYamlFlowItemAIAsk | MidsceneYamlFlowItemAIBoolean | MidsceneYamlFlowItemAILocate | MidsceneYamlFlowItemAIWaitFor | MidsceneYamlFlowItemEvaluateJavaScript | MidsceneYamlFlowItemSleep | MidsceneYamlFlowItemLogScreenshot;
 export interface FreeFn {
     name: string;
     fn: () => void;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "1.0.1-beta-20251030070226.0",
+  "version": "1.0.1-beta-20251104075048.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -89,7 +89,7 @@
     "zod": "3.24.3",
     "semver": "7.5.2",
     "js-yaml": "4.1.0",
-    "@midscene/shared": "1.0.1-beta-20251030070226.0"
+    "@midscene/shared": "1.0.1-beta-20251104075048.0"
   },
   "devDependencies": {
     "@rslib/core": "^0.11.2",