@midscene/core 0.30.5 → 1.0.1-beta-20251021060907.0

package/dist/lib/yaml/player.js

@@ -98,11 +98,11 @@ class ScriptPlayer {
             taskStatus.currentStep = currentStep;
             const flowItem = flow[flowItemIndex];
             debug(`playing step ${flowItemIndex}, flowItem=${JSON.stringify(flowItem)}`);
-            if ('aiAction' in flowItem || 'ai' in flowItem) {
+            if ('aiAct' in flowItem || 'aiAction' in flowItem || 'ai' in flowItem) {
                 const actionTask = flowItem;
-                const prompt = actionTask.aiAction || actionTask.ai;
-                (0, utils_namespaceObject.assert)(prompt, 'missing prompt for ai (aiAction)');
-                await agent.aiAction(prompt, {
+                const prompt = actionTask.aiAct || actionTask.aiAction || actionTask.ai;
+                (0, utils_namespaceObject.assert)(prompt, 'missing prompt for ai (aiAct)');
+                await agent.aiAct(prompt, {
                     cacheable: actionTask.cacheable
                 });
             } else if ('aiAssert' in flowItem) {
@@ -190,21 +190,23 @@ class ScriptPlayer {
                 const evaluateJavaScriptTask = flowItem;
                 const result = await agent.evaluateJavaScript(evaluateJavaScriptTask.javascript);
                 this.setResult(evaluateJavaScriptTask.name, result);
-            } else if ('logScreenshot' in flowItem) {
-                const logScreenshotTask = flowItem;
-                await agent.logScreenshot(logScreenshotTask.logScreenshot, {
-                    content: logScreenshotTask.content || ''
+            } else if ('logScreenshot' in flowItem || 'recordToReport' in flowItem) {
+                const recordTask = flowItem;
+                const title = recordTask.recordToReport ?? recordTask.logScreenshot ?? 'untitled';
+                const content = recordTask.content || '';
+                await agent.recordToReport(title, {
+                    content
                 });
             } else if ('aiInput' in flowItem) {
-                const { aiInput, ...inputTask } = flowItem;
+                const { aiInput, value: rawValue, ...inputTask } = flowItem;
                 let locatePrompt;
                 let value;
                 if (inputTask.locate) {
-                    value = aiInput || inputTask.value;
+                    value = aiInput || rawValue;
                     locatePrompt = inputTask.locate;
                 } else {
                     locatePrompt = aiInput || '';
-                    value = inputTask.value;
+                    value = rawValue;
                 }
                 await agent.callActionInActionSpace('Input', {
                     ...inputTask,
@@ -262,9 +264,11 @@ class ScriptPlayer {
                     return false;
                 });
                 (0, utils_namespaceObject.assert)(matchedAction, `unknown flowItem in yaml: ${JSON.stringify(flowItem)}`);
-                (0, utils_namespaceObject.assert)(!(flowItem.prompt && locatePromptShortcut), `conflict locate prompt for item: ${JSON.stringify(flowItem)}`);
-                if (locatePromptShortcut) flowItem.prompt = locatePromptShortcut;
-                const { locateParam, restParams } = (0, external_utils_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePromptShortcut || '', flowItem, [
+                const flowItemForProcessing = locatePromptShortcut ? {
+                    ...flowItem,
+                    prompt: locatePromptShortcut
+                } : flowItem;
+                const { locateParam, restParams } = (0, external_utils_js_namespaceObject.buildDetailedLocateParamAndRestParams)(locatePromptShortcut || '', flowItemForProcessing, [
                     matchedAction.name,
                     matchedAction.interfaceAlias || '_never_mind_'
                 ]);

package/dist/lib/yaml/player.js.map

@@ -1 +1 @@
-{"version":3,"file":"yaml/player.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/yaml/player.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { basename, dirname, join, resolve } from 'node:path';\nimport { assert, ifInBrowser, ifInWorker } from '@midscene/shared/utils';\n\n// previous defined yaml flow, as a helper\ninterface MidsceneYamlFlowItemAIInput extends LocateOption {\n  // previous version\n  // aiInput: string; // value to input\n  // locate: TUserPrompt; // where to input\n  aiInput: TUserPrompt | undefined; // where to input\n  value: string | number; // value to input\n}\n\ninterface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {\n  // previous version\n  // aiKeyboardPress: string;\n  // locate?: TUserPrompt; // where to press, optional\n  aiKeyboardPress: TUserPrompt | undefined; // where to press\n  keyName: string; // key to press\n}\n\ninterface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {\n  // previous version\n  // aiScroll: null;\n  // locate?: TUserPrompt; // which area to scroll, optional\n  aiScroll: TUserPrompt | undefined; // which area to scroll\n}\n\nimport type { Agent } from '@/agent/agent';\nimport type { TUserPrompt } from '@/ai-model/common';\nimport type {\n  DeviceAction,\n  FreeFn,\n  LocateOption,\n  MidsceneYamlFlowItemAIAction,\n  MidsceneYamlFlowItemAIAsk,\n  MidsceneYamlFlowItemAIAssert,\n  MidsceneYamlFlowItemAIBoolean,\n  MidsceneYamlFlowItemAILocate,\n  MidsceneYamlFlowItemAINumber,\n  MidsceneYamlFlowItemAIQuery,\n  MidsceneYamlFlowItemAIString,\n  MidsceneYamlFlowItemAIWaitFor,\n  MidsceneYamlFlowItemEvaluateJavaScript,\n  MidsceneYamlFlowItemLogScreenshot,\n  MidsceneYamlFlowItemSleep,\n  MidsceneYamlScript,\n  MidsceneYamlScriptEnv,\n  ScriptPlayerStatusValue,\n  ScriptPlayerTaskStatus,\n  ScrollParam,\n} from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { getDebug } from '@midscene/shared/logger';\nimport {\n  buildDetailedLocateParam,\n  buildDetailedLocateParamAndRestParams,\n} from './utils';\n\nconst debug = getDebug('yaml-player');\nexport class ScriptPlayer<T extends MidsceneYamlScriptEnv> {\n  public currentTaskIndex?: number;\n  public taskStatusList: ScriptPlayerTaskStatus[] = [];\n  public status: ScriptPlayerStatusValue = 'init';\n  public reportFile?: string | null;\n  public result: Record<string, any>;\n  private unnamedResultIndex = 0;\n  public output?: string | null;\n  public unstableLogContent?: string | null;\n  public errorInSetup?: Error;\n  private interfaceAgent: Agent | null = null;\n  public agentStatusTip?: string;\n  public target?: MidsceneYamlScriptEnv;\n  private actionSpace: DeviceAction[] = [];\n  private scriptPath?: string;\n  constructor(\n    private script: MidsceneYamlScript,\n    private setupAgent: (platform: T) => Promise<{\n      agent: Agent;\n      freeFn: FreeFn[];\n    }>,\n    public onTaskStatusChange?: (taskStatus: ScriptPlayerTaskStatus) => void,\n    scriptPath?: string,\n  ) {\n    this.scriptPath = scriptPath;\n    this.result = {};\n    this.target =\n      script.target ||\n      script.web ||\n      script.android ||\n      script.ios ||\n      script.config;\n\n    if (ifInBrowser || ifInWorker) {\n      this.output = undefined;\n      debug('output is undefined in browser or worker');\n    } else if (this.target?.output) {\n      this.output = resolve(process.cwd(), this.target.output);\n      debug('setting output by config.output', this.output);\n    } else {\n      const scriptName = this.scriptPath\n        ? basename(this.scriptPath, '.yaml').replace(/\\.(ya?ml)$/i, '')\n        : 'script';\n      this.output = join(\n        getMidsceneRunSubDir('output'),\n        `${scriptName}-${Date.now()}.json`,\n      );\n      debug('setting output by script path', this.output);\n    }\n\n    if (ifInBrowser || ifInWorker) {\n      this.unstableLogContent = undefined;\n    } else if (typeof this.target?.unstableLogContent === 'string') {\n      this.unstableLogContent = resolve(\n        process.cwd(),\n        this.target.unstableLogContent,\n      );\n    } else if (this.target?.unstableLogContent === true) {\n      this.unstableLogContent = join(\n        getMidsceneRunSubDir('output'),\n        'unstableLogContent.json',\n      );\n    }\n\n    this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({\n      ...task,\n      index: taskIndex,\n      status: 'init',\n      totalSteps: task.flow?.length || 0,\n    }));\n  }\n\n  private setResult(key: string | undefined, value: any) {\n    const keyToUse = key || this.unnamedResultIndex++;\n    if (this.result[keyToUse]) {\n      console.warn(`result key ${keyToUse} already exists, will overwrite`);\n    }\n    this.result[keyToUse] = value;\n\n    return this.flushResult();\n  }\n\n  private setPlayerStatus(status: ScriptPlayerStatusValue, error?: Error) {\n    this.status = status;\n    this.errorInSetup = error;\n  }\n\n  private notifyCurrentTaskStatusChange(taskIndex?: number) {\n    const taskIndexToNotify =\n      typeof taskIndex === 'number' ? taskIndex : this.currentTaskIndex;\n\n    if (typeof taskIndexToNotify !== 'number') {\n      return;\n    }\n\n    const taskStatus = this.taskStatusList[taskIndexToNotify];\n    if (this.onTaskStatusChange) {\n      this.onTaskStatusChange(taskStatus);\n    }\n  }\n\n  private async setTaskStatus(\n    index: number,\n    statusValue: ScriptPlayerStatusValue,\n    error?: Error,\n  ) {\n    this.taskStatusList[index].status = statusValue;\n    if (error) {\n      this.taskStatusList[index].error = error;\n    }\n\n    this.notifyCurrentTaskStatusChange(index);\n  }\n\n  private setTaskIndex(taskIndex: number) {\n    this.currentTaskIndex = taskIndex;\n  }\n\n  private flushResult() {\n    if (this.output) {\n      const output = resolve(process.cwd(), this.output);\n      const outputDir = dirname(output);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(output, JSON.stringify(this.result || {}, undefined, 2));\n    }\n  }\n\n  private flushUnstableLogContent() {\n    if (this.unstableLogContent) {\n      const content = this.interfaceAgent?._unstableLogContent();\n      const filePath = resolve(process.cwd(), this.unstableLogContent);\n      const outputDir = dirname(filePath);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(filePath, JSON.stringify(content, null, 2));\n    }\n  }\n\n  async playTask(taskStatus: ScriptPlayerTaskStatus, agent: Agent) {\n    const { flow } = taskStatus;\n    assert(flow, 'missing flow in task');\n\n    for (const flowItemIndex in flow) {\n      const currentStep = Number.parseInt(flowItemIndex, 10);\n      taskStatus.currentStep = currentStep;\n      const flowItem = flow[flowItemIndex];\n      debug(\n        `playing step ${flowItemIndex}, flowItem=${JSON.stringify(flowItem)}`,\n      );\n      if (\n        'aiAction' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n        'ai' in (flowItem as MidsceneYamlFlowItemAIAction)\n      ) {\n        const actionTask = flowItem as MidsceneYamlFlowItemAIAction;\n        const prompt = actionTask.aiAction || actionTask.ai;\n        assert(prompt, 'missing prompt for ai (aiAction)');\n        await agent.aiAction(prompt, {\n          cacheable: actionTask.cacheable,\n        });\n      } else if ('aiAssert' in (flowItem as MidsceneYamlFlowItemAIAssert)) {\n        const assertTask = flowItem as MidsceneYamlFlowItemAIAssert;\n        const prompt = assertTask.aiAssert;\n        const msg = assertTask.errorMessage;\n        assert(prompt, 'missing prompt for aiAssert');\n        const { pass, thought, message } =\n          (await agent.aiAssert(prompt, msg, {\n            keepRawResponse: true,\n          })) || {};\n\n        this.setResult(assertTask.name, {\n          pass,\n          thought,\n          message,\n        });\n\n        if (!pass) {\n          throw new Error(message);\n        }\n      } else if ('aiQuery' in (flowItem as MidsceneYamlFlowItemAIQuery)) {\n        const queryTask = flowItem as MidsceneYamlFlowItemAIQuery;\n        const prompt = queryTask.aiQuery;\n        const options = {\n          domIncluded: queryTask.domIncluded,\n          screenshotIncluded: queryTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiQuery');\n        const queryResult = await agent.aiQuery(prompt, options);\n        this.setResult(queryTask.name, queryResult);\n      } else if ('aiNumber' in (flowItem as MidsceneYamlFlowItemAINumber)) {\n        const numberTask = flowItem as MidsceneYamlFlowItemAINumber;\n        const prompt = numberTask.aiNumber;\n        const options = {\n          domIncluded: numberTask.domIncluded,\n          screenshotIncluded: numberTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiNumber');\n        const numberResult = await agent.aiNumber(prompt, options);\n        this.setResult(numberTask.name, numberResult);\n      } else if ('aiString' in (flowItem as MidsceneYamlFlowItemAIString)) {\n        const stringTask = flowItem as MidsceneYamlFlowItemAIString;\n        const prompt = stringTask.aiString;\n        const options = {\n          domIncluded: stringTask.domIncluded,\n          screenshotIncluded: stringTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiString');\n        const stringResult = await agent.aiString(prompt, options);\n        this.setResult(stringTask.name, stringResult);\n      } else if ('aiBoolean' in (flowItem as MidsceneYamlFlowItemAIBoolean)) {\n        const booleanTask = flowItem as MidsceneYamlFlowItemAIBoolean;\n        const prompt = booleanTask.aiBoolean;\n        const options = {\n          domIncluded: booleanTask.domIncluded,\n          screenshotIncluded: booleanTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiBoolean');\n        const booleanResult = await agent.aiBoolean(prompt, options);\n        this.setResult(booleanTask.name, booleanResult);\n      } else if ('aiAsk' in (flowItem as MidsceneYamlFlowItemAIAsk)) {\n        const askTask = flowItem as MidsceneYamlFlowItemAIAsk;\n        const prompt = askTask.aiAsk;\n        assert(prompt, 'missing prompt for aiAsk');\n        const askResult = await agent.aiAsk(prompt);\n        this.setResult(askTask.name, askResult);\n      } else if ('aiLocate' in (flowItem as MidsceneYamlFlowItemAILocate)) {\n        const locateTask = flowItem as MidsceneYamlFlowItemAILocate;\n        const prompt = locateTask.aiLocate;\n        assert(prompt, 'missing prompt for aiLocate');\n        const locateResult = await agent.aiLocate(prompt, locateTask);\n        this.setResult(locateTask.name, locateResult);\n      } else if ('aiWaitFor' in (flowItem as MidsceneYamlFlowItemAIWaitFor)) {\n        const waitForTask = flowItem as MidsceneYamlFlowItemAIWaitFor;\n        const prompt = waitForTask.aiWaitFor;\n        assert(prompt, 'missing prompt for aiWaitFor');\n        const timeout = waitForTask.timeout;\n        await agent.aiWaitFor(prompt, { timeoutMs: timeout });\n      } else if ('sleep' in (flowItem as MidsceneYamlFlowItemSleep)) {\n        const sleepTask = flowItem as MidsceneYamlFlowItemSleep;\n        const ms = sleepTask.sleep;\n        let msNumber = ms;\n        if (typeof ms === 'string') {\n          msNumber = Number.parseInt(ms, 10);\n        }\n        assert(\n          msNumber && msNumber > 0,\n          `ms for sleep must be greater than 0, but got ${ms}`,\n        );\n        await new Promise((resolve) => setTimeout(resolve, msNumber));\n      } else if (\n        'javascript' in (flowItem as MidsceneYamlFlowItemEvaluateJavaScript)\n      ) {\n        const evaluateJavaScriptTask =\n          flowItem as MidsceneYamlFlowItemEvaluateJavaScript;\n\n        const result = await agent.evaluateJavaScript(\n          evaluateJavaScriptTask.javascript,\n        );\n        this.setResult(evaluateJavaScriptTask.name, result);\n      } else if (\n        'logScreenshot' in (flowItem as MidsceneYamlFlowItemLogScreenshot)\n      ) {\n        const logScreenshotTask = flowItem as MidsceneYamlFlowItemLogScreenshot;\n        await agent.logScreenshot(logScreenshotTask.logScreenshot, {\n          content: logScreenshotTask.content || '',\n        });\n      } else if ('aiInput' in (flowItem as MidsceneYamlFlowItemAIInput)) {\n        // may be input empty string ''\n        const { aiInput, ...inputTask } =\n          flowItem as MidsceneYamlFlowItemAIInput;\n\n        // Compatibility with previous version:\n        // Old format: { aiInput: string (value), locate: TUserPrompt }\n        // New format - 1: { aiInput: TUserPrompt, value: string | number }\n        // New format - 2: { aiInput: undefined, locate: TUserPrompt, value: string | number }\n        let locatePrompt: TUserPrompt | undefined;\n        let value: string | number | undefined;\n        if ((inputTask as any).locate) {\n          // Old format - aiInput is the value, locate is the prompt\n          value = (aiInput as string) || inputTask.value;\n          locatePrompt = (inputTask as any).locate;\n        } else {\n          // New format - aiInput is the prompt, value is the value\n          locatePrompt = aiInput || '';\n          value = inputTask.value;\n        }\n\n        await agent.callActionInActionSpace('Input', {\n          ...inputTask,\n          ...(value !== undefined ? { value: String(value) } : {}),\n          ...(locatePrompt\n            ? { locate: buildDetailedLocateParam(locatePrompt, inputTask) }\n            : {}),\n        });\n      } else if (\n        'aiKeyboardPress' in (flowItem as MidsceneYamlFlowItemAIKeyboardPress)\n      ) {\n        const { aiKeyboardPress, ...keyboardPressTask } =\n          flowItem as MidsceneYamlFlowItemAIKeyboardPress;\n\n        // Compatibility with previous version:\n        // Old format: { aiKeyboardPress: string (key), locate?: TUserPrompt }\n        // New format - 1: { aiKeyboardPress: TUserPrompt, keyName: string }\n        // New format - 2: { aiKeyboardPress: , locate?: TUserPrompt, keyName: string }\n        let locatePrompt: TUserPrompt | undefined;\n        let keyName: string | undefined;\n        if ((keyboardPressTask as any).locate) {\n          // Old format - aiKeyboardPress is the key, locate is the prompt\n          keyName = aiKeyboardPress as string;\n          locatePrompt = (keyboardPressTask as any).locate;\n        } else if (keyboardPressTask.keyName) {\n          // New format - aiKeyboardPress is the prompt, key is the key\n          keyName = keyboardPressTask.keyName;\n          locatePrompt = aiKeyboardPress;\n        } else {\n          keyName = aiKeyboardPress as string;\n        }\n\n        await agent.callActionInActionSpace('KeyboardPress', {\n          ...keyboardPressTask,\n          ...(keyName ? { keyName } : {}),\n          ...(locatePrompt\n            ? {\n                locate: buildDetailedLocateParam(\n                  locatePrompt,\n                  keyboardPressTask,\n                ),\n              }\n            : {}),\n        });\n      } else if ('aiScroll' in (flowItem as MidsceneYamlFlowItemAIScroll)) {\n        const { aiScroll, ...scrollTask } =\n          flowItem as MidsceneYamlFlowItemAIScroll;\n\n        // Compatibility with previous version:\n        // Old format: { aiScroll: null, locate?: TUserPrompt, direction, scrollType, distance? }\n        // New format - 1: { aiScroll: TUserPrompt, direction, scrollType, distance? }\n        // New format - 2: { aiScroll: undefined, locate: TUserPrompt, direction, scrollType, distance? }\n        let locatePrompt: TUserPrompt | undefined;\n        if ((scrollTask as any).locate) {\n          // Old format - locate is the prompt, aiScroll is null/ignored\n          locatePrompt = (scrollTask as any).locate;\n        } else {\n          // New format - aiScroll is the prompt, or no prompt for global scroll\n          locatePrompt = aiScroll;\n        }\n\n        await agent.callActionInActionSpace('Scroll', {\n          ...scrollTask,\n          ...(locatePrompt\n            ? { locate: buildDetailedLocateParam(locatePrompt, scrollTask) }\n            : {}),\n        });\n      } else {\n        // generic action, find the action in actionSpace\n\n        /* for aiTap, aiRightClick, the parameters are a flattened data for the 'locate', these are all valid data\n\n        - aiTap: 'search input box'\n        - aiTap: 'search input box'\n          deepThink: true\n          cacheable: false\n        - aiTap:\n          prompt: 'search input box'\n        - aiTap:\n          prompt: 'search input box'\n          deepThink: true\n          cacheable: false\n        */\n\n        const actionSpace = this.actionSpace;\n        let locatePromptShortcut: string | undefined;\n        const matchedAction = actionSpace.find((action) => {\n          const actionInterfaceAlias = action.interfaceAlias;\n          if (\n            actionInterfaceAlias &&\n            Object.prototype.hasOwnProperty.call(flowItem, actionInterfaceAlias)\n          ) {\n            locatePromptShortcut = flowItem[\n              actionInterfaceAlias as keyof typeof flowItem\n            ] as string;\n            return true;\n          }\n\n          const keyOfActionInActionSpace = action.name;\n          if (\n            Object.prototype.hasOwnProperty.call(\n              flowItem,\n              keyOfActionInActionSpace,\n            )\n          ) {\n            locatePromptShortcut = flowItem[\n              keyOfActionInActionSpace as keyof typeof flowItem\n            ] as string;\n            return true;\n          }\n\n          return false;\n        });\n\n        assert(\n          matchedAction,\n          `unknown flowItem in yaml: ${JSON.stringify(flowItem)}`,\n        );\n\n        assert(\n          !((flowItem as any).prompt && locatePromptShortcut),\n          `conflict locate prompt for item: ${JSON.stringify(flowItem)}`,\n        );\n\n        if (locatePromptShortcut) {\n          (flowItem as any).prompt = locatePromptShortcut;\n        }\n\n        const { locateParam, restParams } =\n          buildDetailedLocateParamAndRestParams(\n            locatePromptShortcut || '',\n            flowItem as LocateOption,\n            [\n              matchedAction.name,\n              matchedAction.interfaceAlias || '_never_mind_',\n            ],\n          );\n\n        const flowParams = {\n          ...restParams,\n          locate: locateParam,\n        };\n\n        debug(\n          `matchedAction: ${matchedAction.name}`,\n          `flowParams: ${JSON.stringify(flowParams, null, 2)}`,\n        );\n        await agent.callActionInActionSpace(matchedAction.name, flowParams);\n      }\n    }\n    this.reportFile = agent.reportFile;\n    await this.flushUnstableLogContent();\n  }\n\n  async run() {\n    const { target, web, android, ios, tasks } = this.script;\n    const webEnv = web || target;\n    const androidEnv = android;\n    const iosEnv = ios;\n    const platform = webEnv || androidEnv || iosEnv;\n\n    this.setPlayerStatus('running');\n\n    let agent: Agent | null = null;\n    let freeFn: FreeFn[] = [];\n    try {\n      const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(\n        platform as T,\n      );\n      this.actionSpace = await newAgent.getActionSpace();\n      agent = newAgent;\n      const originalOnTaskStartTip = agent.onTaskStartTip;\n      agent.onTaskStartTip = (tip) => {\n        if (this.status === 'running') {\n          this.agentStatusTip = tip;\n        }\n        originalOnTaskStartTip?.(tip);\n      };\n      freeFn = [\n        ...(newFreeFn || []),\n        {\n          name: 'restore-agent-onTaskStartTip',\n          fn: () => {\n            if (agent) {\n              agent.onTaskStartTip = originalOnTaskStartTip;\n            }\n          },\n        },\n      ];\n    } catch (e) {\n      this.setPlayerStatus('error', e as Error);\n      return;\n    }\n    this.interfaceAgent = agent;\n\n    let taskIndex = 0;\n    this.setPlayerStatus('running');\n    let errorFlag = false;\n    while (taskIndex < tasks.length) {\n      const taskStatus = this.taskStatusList[taskIndex];\n      this.setTaskStatus(taskIndex, 'running' as any);\n      this.setTaskIndex(taskIndex);\n\n      try {\n        await this.playTask(taskStatus, this.interfaceAgent);\n        this.setTaskStatus(taskIndex, 'done' as any);\n      } catch (e) {\n        this.setTaskStatus(taskIndex, 'error' as any, e as Error);\n\n        if (taskStatus.continueOnError) {\n          // nothing more to do\n        } else {\n          this.reportFile = agent.reportFile;\n          errorFlag = true;\n          break;\n        }\n      }\n      this.reportFile = agent?.reportFile;\n      taskIndex++;\n    }\n\n    if (errorFlag) {\n      this.setPlayerStatus('error');\n    } else {\n      this.setPlayerStatus('done');\n    }\n    this.agentStatusTip = '';\n\n    // free the resources\n    for (const fn of freeFn) {\n      try {\n        // console.log('freeing', fn.name);\n        await fn.fn();\n        // console.log('freed', fn.name);\n      } catch (e) {\n        // console.error('error freeing', fn.name, e);\n      }\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","ScriptPlayer","value","keyToUse","console","status","error","taskIndex","taskIndexToNotify","taskStatus","index","statusValue","output","resolve","process","outputDir","dirname","existsSync","mkdirSync","writeFileSync","JSON","undefined","_this_interfaceAgent","content","filePath","agent","flow","assert","flowItemIndex","currentStep","Number","flowItem","actionTask","prompt","assertTask","msg","pass","thought","message","Error","queryTask","options","queryResult","numberTask","numberResult","stringTask","stringResult","booleanTask","booleanResult","askTask","askResult","locateTask","locateResult","waitForTask","timeout","sleepTask","ms","msNumber","Promise","setTimeout","evaluateJavaScriptTask","result","logScreenshotTask","aiInput","inputTask","locatePrompt","String","buildDetailedLocateParam","aiKeyboardPress","keyboardPressTask","keyName","aiScroll","scrollTask","actionSpace","locatePromptShortcut","matchedAction","action","actionInterfaceAlias","keyOfActionInActionSpace","locateParam","restParams","buildDetailedLocateParamAndRestParams","flowParams","target","web","android","ios","tasks","webEnv","androidEnv","iosEnv","platform","freeFn","newAgent","newFreeFn","originalOnTaskStartTip","tip","e","errorFlag","fn","script","setupAgent","onTaskStatusChange","scriptPath","_this_target","_this_target1","_this_target2","ifInBrowser","ifInWorker","scriptName","basename","join","getMidsceneRunSubDir","Date","task","_task_flow"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;ACqDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAChB,MAAMC;IAwEH,UAAUP,GAAuB,EAAEQ,KAAU,EAAE;QACrD,MAAMC,WAAWT,OAAO,IAAI,CAAC,kBAAkB;QAC/C,IAAI,IAAI,CAAC,MAAM,CAACS,SAAS,EACvBC,QAAQ,IAAI,CAAC,CAAC,WAAW,EAAED,SAAS,+BAA+B,CAAC;QAEtE,IAAI,CAAC,MAAM,CAACA,SAAS,GAAGD;QAExB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEQ,gBAAgBG,MAA+B,EAAEC,KAAa,EAAE;QACtE,IAAI,CAAC,MAAM,GAAGD;QACd,IAAI,CAAC,YAAY,GAAGC;IACtB;IAEQ,8BAA8BC,SAAkB,EAAE;QACxD,MAAMC,oBACJ,AAAqB,YAArB,OAAOD,YAAyBA,YAAY,IAAI,CAAC,gBAAgB;QAEnE,IAAI,AAA6B,YAA7B,OAAOC,mBACT;QAGF,MAAMC,aAAa,IAAI,CAAC,cAAc,CAACD,kBAAkB;QACzD,IAAI,IAAI,CAAC,kBAAkB,EACzB,IAAI,CAAC,kBAAkB,CAACC;IAE5B;IAEA,MAAc,cACZC,KAAa,EACbC,WAAoC,EACpCL,KAAa,EACb;QACA,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,MAAM,GAAGC;QACpC,IAAIL,OACF,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,KAAK,GAAGJ;QAGrC,IAAI,CAAC,6BAA6B,CAACI;IACrC;IAEQ,aAAaH,SAAiB,EAAE;QACtC,IAAI,CAAC,gBAAgB,GAAGA;IAC1B;IAEQ,cAAc;QACpB,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,MAAMK,SAASC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM;YACjD,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQJ;YAC1B,IAAI,CAACK,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcP,QAAQQ,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,GAAGC,QAAW;QACrE;IACF;IAEQ,0BAA0B;QAChC,IAAI,IAAI,CAAC,kBAAkB,EAAE;gBACXC;YAAhB,MAAMC,UAAU,QAAAD,CAAAA,uBAAAA,IAAI,CAAC,cAAc,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,mBAAmB;YACxD,MAAME,WAAWX,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,kBAAkB;YAC/D,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQQ;YAC1B,IAAI,CAACP,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcK,UAAUJ,KAAK,SAAS,CAACG,SAAS,MAAM;QACxD;IACF;IAEA,MAAM,SAASd,UAAkC,EAAEgB,KAAY,EAAE;QAC/D,MAAM,EAAEC,IAAI,EAAE,GAAGjB;QACjBkB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,MAAM;QAEb,IAAK,MAAME,iBAAiBF,KAAM;YAChC,MAAMG,cAAcC,OAAO,QAAQ,CAACF,eAAe;YACnDnB,WAAW,WAAW,GAAGoB;YACzB,MAAME,WAAWL,IAAI,CAACE,cAAc;YACpC7B,MACE,CAAC,aAAa,EAAE6B,cAAc,WAAW,EAAER,KAAK,SAAS,CAACW,WAAW;YAEvE,IACE,cAAeA,YACf,QAASA,UACT;gBACA,MAAMC,aAAaD;gBACnB,MAAME,SAASD,WAAW,QAAQ,IAAIA,WAAW,EAAE;gBACnDL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMR,MAAM,QAAQ,CAACQ,QAAQ;oBAC3B,WAAWD,WAAW,SAAS;gBACjC;YACF,OAAO,IAAI,cAAeD,UAA2C;gBACnE,MAAMG,aAAaH;gBACnB,MAAME,SAASC,WAAW,QAAQ;gBAClC,MAAMC,MAAMD,WAAW,YAAY;gBACnCP,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAM,EAAEG,IAAI,EAAEC,OAAO,EAAEC,OAAO,EAAE,GAC7B,MAAMb,MAAM,QAAQ,CAACQ,QAAQE,KAAK;oBACjC,iBAAiB;gBACnB,MAAO,CAAC;gBAEV,IAAI,CAAC,SAAS,CAACD,WAAW,IAAI,EAAE;oBAC9BE;oBACAC;oBACAC;gBACF;gBAEA,IAAI,CAACF,MACH,MAAM,IAAIG,MAAMD;YAEpB,OAAO,IAAI,aAAcP,UAA0C;gBACjE,MAAMS,YAAYT;gBAClB,MAAME,SAASO,UAAU,OAAO;gBAChC,MAAMC,UAAU;oBACd,aAAaD,UAAU,WAAW;oBAClC,oBAAoBA,UAAU,kBAAkB;gBAClD;gBACAb,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMS,cAAc,MAAMjB,MAAM,OAAO,CAACQ,QAAQQ;gBAChD,IAAI,CAAC,SAAS,CAACD,UAAU,IAAI,EAAEE;YACjC,OAAO,IAAI,cAAeX,UAA2C;gBACnE,MAAMY,aAAaZ;gBACnB,MAAME,SAASU,WAAW,QAAQ;gBAClC,MAAMF,UAAU;oBACd,aAAaE,WAAW,WAAW;oBACnC,oBAAoBA,WAAW,kBAAkB;gBACnD;gBACAhB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMW,eAAe,MAAMnB,MAAM,QAAQ,CAACQ,QAAQQ;gBAClD,IAAI,CAAC,SAAS,CAACE,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,cAAeb,UAA2C;gBACnE,MAAMc,aAAad;gBACnB,MAAME,SAASY,WAAW,QAAQ;gBAClC,MAAMJ,UAAU;oBACd,aAAaI,WAAW,WAAW;oBACnC,oBAAoBA,WAAW,kBAAkB;gBACnD;gBACAlB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMa,eAAe,MAAMrB,MAAM,QAAQ,CAACQ,QAAQQ;gBAClD,IAAI,CAAC,SAAS,CAACI,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,eAAgBf,UAA4C;gBACrE,MAAMgB,cAAchB;gBACpB,MAAME,SAASc,YAAY,SAAS;gBACpC,MAAMN,UAAU;oBACd,aAAaM,YAAY,WAAW;oBACpC,oBAAoBA,YAAY,kBAAkB;gBACpD;gBACApB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMe,gBAAgB,MAAMvB,MAAM,SAAS,CAACQ,QAAQQ;gBACpD,IAAI,CAAC,SAAS,CAACM,YAAY,IAAI,EAAEC;YACnC,OAAO,IAAI,WAAYjB,UAAwC;gBAC7D,MAAMkB,UAAUlB;gBAChB,MAAME,SAASgB,QAAQ,KAAK;gBAC5BtB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMiB,YAAY,MAAMzB,MAAM,KAAK,CAACQ;gBACpC,IAAI,CAAC,SAAS,CAACgB,QAAQ,IAAI,EAAEC;YAC/B,OAAO,IAAI,cAAenB,UAA2C;gBACnE,MAAMoB,aAAapB;gBACnB,MAAME,SAASkB,WAAW,QAAQ;gBAClCxB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMmB,eAAe,MAAM3B,MAAM,QAAQ,CAACQ,QAAQkB;gBAClD,IAAI,CAAC,SAAS,CAACA,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,eAAgBrB,UAA4C;gBACrE,MAAMsB,cAActB;gBACpB,MAAME,SAASoB,YAAY,SAAS;gBACpC1B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMqB,UAAUD,YAAY,OAAO;gBACnC,MAAM5B,MAAM,SAAS,CAACQ,QAAQ;oBAAE,WAAWqB;gBAAQ;YACrD,OAAO,IAAI,WAAYvB,UAAwC;gBAC7D,MAAMwB,YAAYxB;gBAClB,MAAMyB,KAAKD,UAAU,KAAK;gBAC1B,IAAIE,WAAWD;gBACf,IAAI,AAAc,YAAd,OAAOA,IACTC,WAAW3B,OAAO,QAAQ,CAAC0B,IAAI;gBAEjC7B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8B,YAAYA,WAAW,GACvB,CAAC,6CAA6C,EAAED,IAAI;gBAEtD,MAAM,IAAIE,QAAQ,CAAC7C,UAAY8C,WAAW9C,SAAS4C;YACrD,OAAO,IACL,gBAAiB1B,UACjB;gBACA,MAAM6B,yBACJ7B;gBAEF,MAAM8B,SAAS,MAAMpC,MAAM,kBAAkB,CAC3CmC,uBAAuB,UAAU;gBAEnC,IAAI,CAAC,SAAS,CAACA,uBAAuB,IAAI,EAAEC;YAC9C,OAAO,IACL,mBAAoB9B,UACpB;gBACA,MAAM+B,oBAAoB/B;gBAC1B,MAAMN,MAAM,aAAa,CAACqC,kBAAkB,aAAa,EAAE;oBACzD,SAASA,kBAAkB,OAAO,IAAI;gBACxC;YACF,OAAO,IAAI,aAAc/B,UAA0C;gBAEjE,MAAM,EAAEgC,OAAO,EAAE,GAAGC,WAAW,GAC7BjC;gBAMF,IAAIkC;gBACJ,IAAI/D;gBACJ,IAAK8D,UAAkB,MAAM,EAAE;oBAE7B9D,QAAS6D,WAAsBC,UAAU,KAAK;oBAC9CC,eAAgBD,UAAkB,MAAM;gBAC1C,OAAO;oBAELC,eAAeF,WAAW;oBAC1B7D,QAAQ8D,UAAU,KAAK;gBACzB;gBAEA,MAAMvC,MAAM,uBAAuB,CAAC,SAAS;oBAC3C,GAAGuC,SAAS;oBACZ,GAAI9D,AAAUmB,WAAVnB,QAAsB;wBAAE,OAAOgE,OAAOhE;oBAAO,IAAI,CAAC,CAAC;oBACvD,GAAI+D,eACA;wBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcD;oBAAW,IAC5D,CAAC,CAAC;gBACR;YACF,OAAO,IACL,qBAAsBjC,UACtB;gBACA,MAAM,EAAEqC,eAAe,EAAE,GAAGC,mBAAmB,GAC7CtC;gBAMF,IAAIkC;gBACJ,IAAIK;gBACJ,IAAKD,kBAA0B,MAAM,EAAE;oBAErCC,UAAUF;oBACVH,eAAgBI,kBAA0B,MAAM;gBAClD,OAAO,IAAIA,kBAAkB,OAAO,EAAE;oBAEpCC,UAAUD,kBAAkB,OAAO;oBACnCJ,eAAeG;gBACjB,OACEE,UAAUF;gBAGZ,MAAM3C,MAAM,uBAAuB,CAAC,iBAAiB;oBACnD,GAAG4C,iBAAiB;oBACpB,GAAIC,UAAU;wBAAEA;oBAAQ,IAAI,CAAC,CAAC;oBAC9B,GAAIL,eACA;wBACE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACNF,cACAI;oBAEJ,IACA,CAAC,CAAC;gBACR;YACF,OAAO,IAAI,cAAetC,UAA2C;gBACnE,MAAM,EAAEwC,QAAQ,EAAE,GAAGC,YAAY,GAC/BzC;gBAMF,IAAIkC;gBAGFA,eAFGO,WAAmB,MAAM,GAEZA,WAAmB,MAAM,GAG1BD;gBAGjB,MAAM9C,MAAM,uBAAuB,CAAC,UAAU;oBAC5C,GAAG+C,UAAU;oBACb,GAAIP,eACA;wBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcO;oBAAY,IAC7D,CAAC,CAAC;gBACR;YACF,OAAO;gBAiBL,MAAMC,cAAc,IAAI,CAAC,WAAW;gBACpC,IAAIC;gBACJ,MAAMC,gBAAgBF,YAAY,IAAI,CAAC,CAACG;oBACtC,MAAMC,uBAAuBD,OAAO,cAAc;oBAClD,IACEC,wBACAlF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACoC,UAAU8C,uBAC/C;wBACAH,uBAAuB3C,QAAQ,CAC7B8C,qBACD;wBACD,OAAO;oBACT;oBAEA,MAAMC,2BAA2BF,OAAO,IAAI;oBAC5C,IACEjF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAClCoC,UACA+C,2BAEF;wBACAJ,uBAAuB3C,QAAQ,CAC7B+C,yBACD;wBACD,OAAO;oBACT;oBAEA,OAAO;gBACT;gBAEAnD,IAAAA,sBAAAA,MAAAA,AAAAA,EACEgD,eACA,CAAC,0BAA0B,EAAEvD,KAAK,SAAS,CAACW,WAAW;gBAGzDJ,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAAGI,CAAAA,SAAiB,MAAM,IAAI2C,oBAAmB,GACjD,CAAC,iCAAiC,EAAEtD,KAAK,SAAS,CAACW,WAAW;gBAGhE,IAAI2C,sBACD3C,SAAiB,MAAM,GAAG2C;gBAG7B,MAAM,EAAEK,WAAW,EAAEC,UAAU,EAAE,GAC/BC,AAAAA,IAAAA,kCAAAA,qCAAAA,AAAAA,EACEP,wBAAwB,IACxB3C,UACA;oBACE4C,cAAc,IAAI;oBAClBA,cAAc,cAAc,IAAI;iBACjC;gBAGL,MAAMO,aAAa;oBACjB,GAAGF,UAAU;oBACb,QAAQD;gBACV;gBAEAhF,MACE,CAAC,eAAe,EAAE4E,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAEvD,KAAK,SAAS,CAAC8D,YAAY,MAAM,IAAI;gBAEtD,MAAMzD,MAAM,uBAAuB,CAACkD,cAAc,IAAI,EAAEO;YAC1D;QACF;QACA,IAAI,CAAC,UAAU,GAAGzD,MAAM,UAAU;QAClC,MAAM,IAAI,CAAC,uBAAuB;IACpC;IAEA,MAAM,MAAM;QACV,MAAM,EAAE0D,MAAM,EAAEC,GAAG,EAAEC,OAAO,EAAEC,GAAG,EAAEC,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM;QACxD,MAAMC,SAASJ,OAAOD;QACtB,MAAMM,aAAaJ;QACnB,MAAMK,SAASJ;QACf,MAAMK,WAAWH,UAAUC,cAAcC;QAEzC,IAAI,CAAC,eAAe,CAAC;QAErB,IAAIjE,QAAsB;QAC1B,IAAImE,SAAmB,EAAE;QACzB,IAAI;YACF,MAAM,EAAE,OAAOC,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAClEH;YAEF,IAAI,CAAC,WAAW,GAAG,MAAME,SAAS,cAAc;YAChDpE,QAAQoE;YACR,MAAME,yBAAyBtE,MAAM,cAAc;YACnDA,MAAM,cAAc,GAAG,CAACuE;gBACtB,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,cAAc,GAAGA;gBAExBD,QAAAA,0BAAAA,uBAAyBC;YAC3B;YACAJ,SAAS;mBACHE,aAAa,EAAE;gBACnB;oBACE,MAAM;oBACN,IAAI;wBACF,IAAIrE,OACFA,MAAM,cAAc,GAAGsE;oBAE3B;gBACF;aACD;QACH,EAAE,OAAOE,GAAG;YACV,IAAI,CAAC,eAAe,CAAC,SAASA;YAC9B;QACF;QACA,IAAI,CAAC,cAAc,GAAGxE;QAEtB,IAAIlB,YAAY;QAChB,IAAI,CAAC,eAAe,CAAC;QACrB,IAAI2F,YAAY;QAChB,MAAO3F,YAAYgF,MAAM,MAAM,CAAE;YAC/B,MAAM9E,aAAa,IAAI,CAAC,cAAc,CAACF,UAAU;YACjD,IAAI,CAAC,aAAa,CAACA,WAAW;YAC9B,IAAI,CAAC,YAAY,CAACA;YAElB,IAAI;gBACF,MAAM,IAAI,CAAC,QAAQ,CAACE,YAAY,IAAI,CAAC,cAAc;gBACnD,IAAI,CAAC,aAAa,CAACF,WAAW;YAChC,EAAE,OAAO0F,GAAG;gBACV,IAAI,CAAC,aAAa,CAAC1F,WAAW,SAAgB0F;gBAE9C,IAAIxF,WAAW,eAAe;qBAEvB;oBACL,IAAI,CAAC,UAAU,GAAGgB,MAAM,UAAU;oBAClCyE,YAAY;oBACZ;gBACF;YACF;YACA,IAAI,CAAC,UAAU,GAAGzE,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,UAAU;YACnClB;QACF;QAEA,IAAI2F,WACF,IAAI,CAAC,eAAe,CAAC;aAErB,IAAI,CAAC,eAAe,CAAC;QAEvB,IAAI,CAAC,cAAc,GAAG;QAGtB,KAAK,MAAMC,MAAMP,OACf,IAAI;YAEF,MAAMO,GAAG,EAAE;QAEb,EAAE,OAAOF,GAAG,CAEZ;IAEJ;IA/fA,YACUG,MAA0B,EAC1BC,UAGN,EACKC,kBAAiE,EACxEC,UAAmB,CACnB;YAaWC,cAgBOC,eAKPC;;;;QAxDb,uBAAO,oBAAP;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAO,cAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,sBAAR;QACA,uBAAO,UAAP;QACA,uBAAO,sBAAP;QACA,uBAAO,gBAAP;QACA,uBAAQ,kBAAR;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,eAAR;QACA,uBAAQ,cAAR;aAEUN,MAAM,GAANA;aACAC,UAAU,GAAVA;aAIDC,kBAAkB,GAAlBA;aAnBF,cAAc,GAA6B,EAAE;aAC7C,MAAM,GAA4B;aAGjC,kBAAkB,GAAG;aAIrB,cAAc,GAAiB;aAG/B,WAAW,GAAmB,EAAE;QAWtC,IAAI,CAAC,UAAU,GAAGC;QAClB,IAAI,CAAC,MAAM,GAAG,CAAC;QACf,IAAI,CAAC,MAAM,GACTH,OAAO,MAAM,IACbA,OAAO,GAAG,IACVA,OAAO,OAAO,IACdA,OAAO,GAAG,IACVA,OAAO,MAAM;QAEf,IAAIO,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAAE;YAC7B,IAAI,CAAC,MAAM,GAAGvF;YACdtB,MAAM;QACR,OAAO,IAAI,QAAAyG,CAAAA,eAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,MAAM,EAAE;YAC9B,IAAI,CAAC,MAAM,GAAG3F,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM;YACvDf,MAAM,mCAAmC,IAAI,CAAC,MAAM;QACtD,OAAO;YACL,MAAM8G,aAAa,IAAI,CAAC,UAAU,GAC9BC,AAAAA,IAAAA,mCAAAA,QAAAA,AAAAA,EAAS,IAAI,CAAC,UAAU,EAAE,SAAS,OAAO,CAAC,eAAe,MAC1D;YACJ,IAAI,CAAC,MAAM,GAAGC,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACZC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB,GAAGH,WAAW,CAAC,EAAEI,KAAK,GAAG,GAAG,KAAK,CAAC;YAEpClH,MAAM,iCAAiC,IAAI,CAAC,MAAM;QACpD;QAEA,IAAI4G,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAC3B,IAAI,CAAC,kBAAkB,GAAGvF;aACrB,IAAI,AAA2C,YAA3C,gBAAOoF,CAAAA,gBAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,kBAAkB,AAAD,GAC9C,IAAI,CAAC,kBAAkB,GAAG5F,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EACxBC,QAAQ,GAAG,IACX,IAAI,CAAC,MAAM,CAAC,kBAAkB;aAE3B,IAAI4F,AAAAA,SAAAA,CAAAA,gBAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,kBAAkB,AAAD,MAAM,MAC7C,IAAI,CAAC,kBAAkB,GAAGK,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACxBC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB;QAIJ,IAAI,CAAC,cAAc,GAAIZ,AAAAA,CAAAA,OAAO,KAAK,IAAI,EAAC,EAAG,GAAG,CAAC,CAACc,MAAM3G;gBAIxC4G;mBAJuD;gBACnE,GAAGD,IAAI;gBACP,OAAO3G;gBACP,QAAQ;gBACR,YAAY4G,AAAAA,SAAAA,CAAAA,aAAAA,KAAK,IAAI,AAAD,IAARA,KAAAA,IAAAA,WAAW,MAAM,AAAD,KAAK;YACnC;;IACF;AAycF"}
+{"version":3,"file":"yaml/player.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/yaml/player.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { basename, dirname, join, resolve } from 'node:path';\nimport { assert, ifInBrowser, ifInWorker } from '@midscene/shared/utils';\n\n// previous defined yaml flow, as a helper\ninterface MidsceneYamlFlowItemAIInput extends LocateOption {\n  // previous version\n  // aiInput: string; // value to input\n  // locate: TUserPrompt; // where to input\n  aiInput: TUserPrompt | undefined; // where to input\n  value: string | number; // value to input\n}\n\ninterface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {\n  // previous version\n  // aiKeyboardPress: string;\n  // locate?: TUserPrompt; // where to press, optional\n  aiKeyboardPress: TUserPrompt | undefined; // where to press\n  keyName: string; // key to press\n}\n\ninterface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {\n  // previous version\n  // aiScroll: null;\n  // locate?: TUserPrompt; // which area to scroll, optional\n  aiScroll: TUserPrompt | undefined; // which area to scroll\n}\n\nimport type { Agent } from '@/agent/agent';\nimport type { TUserPrompt } from '@/ai-model/common';\nimport type {\n  DeviceAction,\n  FreeFn,\n  LocateOption,\n  MidsceneYamlFlowItemAIAction,\n  MidsceneYamlFlowItemAIAsk,\n  MidsceneYamlFlowItemAIAssert,\n  MidsceneYamlFlowItemAIBoolean,\n  MidsceneYamlFlowItemAILocate,\n  MidsceneYamlFlowItemAINumber,\n  MidsceneYamlFlowItemAIQuery,\n  MidsceneYamlFlowItemAIString,\n  MidsceneYamlFlowItemAIWaitFor,\n  MidsceneYamlFlowItemEvaluateJavaScript,\n  MidsceneYamlFlowItemLogScreenshot,\n  MidsceneYamlFlowItemSleep,\n  MidsceneYamlScript,\n  MidsceneYamlScriptEnv,\n  ScriptPlayerStatusValue,\n  ScriptPlayerTaskStatus,\n  ScrollParam,\n} from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { getDebug } from '@midscene/shared/logger';\nimport {\n  buildDetailedLocateParam,\n  buildDetailedLocateParamAndRestParams,\n} from './utils';\n\nconst debug = getDebug('yaml-player');\nexport class ScriptPlayer<T extends MidsceneYamlScriptEnv> {\n  public currentTaskIndex?: number;\n  public taskStatusList: ScriptPlayerTaskStatus[] = [];\n  public status: ScriptPlayerStatusValue = 'init';\n  public reportFile?: string | null;\n  public result: Record<string, any>;\n  private unnamedResultIndex = 0;\n  public output?: string | null;\n  public unstableLogContent?: string | null;\n  public errorInSetup?: Error;\n  private interfaceAgent: Agent | null = null;\n  public agentStatusTip?: string;\n  public target?: MidsceneYamlScriptEnv;\n  private actionSpace: DeviceAction[] = [];\n  private scriptPath?: string;\n  constructor(\n    private script: MidsceneYamlScript,\n    private setupAgent: (platform: T) => Promise<{\n      agent: Agent;\n      freeFn: FreeFn[];\n    }>,\n    public onTaskStatusChange?: (taskStatus: ScriptPlayerTaskStatus) => void,\n    scriptPath?: string,\n  ) {\n    this.scriptPath = scriptPath;\n    this.result = {};\n    this.target =\n      script.target ||\n      script.web ||\n      script.android ||\n      script.ios ||\n      script.config;\n\n    if (ifInBrowser || ifInWorker) {\n      this.output = undefined;\n      debug('output is undefined in browser or worker');\n    } else if (this.target?.output) {\n      this.output = resolve(process.cwd(), this.target.output);\n      debug('setting output by config.output', this.output);\n    } else {\n      const scriptName = this.scriptPath\n        ? basename(this.scriptPath, '.yaml').replace(/\\.(ya?ml)$/i, '')\n        : 'script';\n      this.output = join(\n        getMidsceneRunSubDir('output'),\n        `${scriptName}-${Date.now()}.json`,\n      );\n      debug('setting output by script path', this.output);\n    }\n\n    if (ifInBrowser || ifInWorker) {\n      this.unstableLogContent = undefined;\n    } else if (typeof this.target?.unstableLogContent === 'string') {\n      this.unstableLogContent = resolve(\n        process.cwd(),\n        this.target.unstableLogContent,\n      );\n    } else if (this.target?.unstableLogContent === true) {\n      this.unstableLogContent = join(\n        getMidsceneRunSubDir('output'),\n        'unstableLogContent.json',\n      );\n    }\n\n    this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({\n      ...task,\n      index: taskIndex,\n      status: 'init',\n      totalSteps: task.flow?.length || 0,\n    }));\n  }\n\n  private setResult(key: string | undefined, value: any) {\n    const keyToUse = key || this.unnamedResultIndex++;\n    if (this.result[keyToUse]) {\n      console.warn(`result key ${keyToUse} already exists, will overwrite`);\n    }\n    this.result[keyToUse] = value;\n\n    return this.flushResult();\n  }\n\n  private setPlayerStatus(status: ScriptPlayerStatusValue, error?: Error) {\n    this.status = status;\n    this.errorInSetup = error;\n  }\n\n  private notifyCurrentTaskStatusChange(taskIndex?: number) {\n    const taskIndexToNotify =\n      typeof taskIndex === 'number' ? taskIndex : this.currentTaskIndex;\n\n    if (typeof taskIndexToNotify !== 'number') {\n      return;\n    }\n\n    const taskStatus = this.taskStatusList[taskIndexToNotify];\n    if (this.onTaskStatusChange) {\n      this.onTaskStatusChange(taskStatus);\n    }\n  }\n\n  private async setTaskStatus(\n    index: number,\n    statusValue: ScriptPlayerStatusValue,\n    error?: Error,\n  ) {\n    this.taskStatusList[index].status = statusValue;\n    if (error) {\n      this.taskStatusList[index].error = error;\n    }\n\n    this.notifyCurrentTaskStatusChange(index);\n  }\n\n  private setTaskIndex(taskIndex: number) {\n    this.currentTaskIndex = taskIndex;\n  }\n\n  private flushResult() {\n    if (this.output) {\n      const output = resolve(process.cwd(), this.output);\n      const outputDir = dirname(output);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(output, JSON.stringify(this.result || {}, undefined, 2));\n    }\n  }\n\n  private flushUnstableLogContent() {\n    if (this.unstableLogContent) {\n      const content = this.interfaceAgent?._unstableLogContent();\n      const filePath = resolve(process.cwd(), this.unstableLogContent);\n      const outputDir = dirname(filePath);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(filePath, JSON.stringify(content, null, 2));\n    }\n  }\n\n  async playTask(taskStatus: ScriptPlayerTaskStatus, agent: Agent) {\n    const { flow } = taskStatus;\n    assert(flow, 'missing flow in task');\n\n    for (const flowItemIndex in flow) {\n      const currentStep = Number.parseInt(flowItemIndex, 10);\n      taskStatus.currentStep = currentStep;\n      const flowItem = flow[flowItemIndex];\n      debug(\n        `playing step ${flowItemIndex}, flowItem=${JSON.stringify(flowItem)}`,\n      );\n      if (\n        'aiAct' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n        'aiAction' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n        'ai' in (flowItem as MidsceneYamlFlowItemAIAction)\n      ) {\n        const actionTask = flowItem as MidsceneYamlFlowItemAIAction;\n        const prompt = actionTask.aiAct || actionTask.aiAction || actionTask.ai;\n        assert(prompt, 'missing prompt for ai (aiAct)');\n        await agent.aiAct(prompt, {\n          cacheable: actionTask.cacheable,\n        });\n      } else if ('aiAssert' in (flowItem as MidsceneYamlFlowItemAIAssert)) {\n        const assertTask = flowItem as MidsceneYamlFlowItemAIAssert;\n        const prompt = assertTask.aiAssert;\n        const msg = assertTask.errorMessage;\n        assert(prompt, 'missing prompt for aiAssert');\n        const { pass, thought, message } =\n          (await agent.aiAssert(prompt, msg, {\n            keepRawResponse: true,\n          })) || {};\n\n        this.setResult(assertTask.name, {\n          pass,\n          thought,\n          message,\n        });\n\n        if (!pass) {\n          throw new Error(message);\n        }\n      } else if ('aiQuery' in (flowItem as MidsceneYamlFlowItemAIQuery)) {\n        const queryTask = flowItem as MidsceneYamlFlowItemAIQuery;\n        const prompt = queryTask.aiQuery;\n        const options = {\n          domIncluded: queryTask.domIncluded,\n          screenshotIncluded: queryTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiQuery');\n        const queryResult = await agent.aiQuery(prompt, options);\n        this.setResult(queryTask.name, queryResult);\n      } else if ('aiNumber' in (flowItem as MidsceneYamlFlowItemAINumber)) {\n        const numberTask = flowItem as MidsceneYamlFlowItemAINumber;\n        const prompt = numberTask.aiNumber;\n        const options = {\n          domIncluded: numberTask.domIncluded,\n          screenshotIncluded: numberTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiNumber');\n        const numberResult = await agent.aiNumber(prompt, options);\n        this.setResult(numberTask.name, numberResult);\n      } else if ('aiString' in (flowItem as MidsceneYamlFlowItemAIString)) {\n        const stringTask = flowItem as MidsceneYamlFlowItemAIString;\n        const prompt = stringTask.aiString;\n        const options = {\n          domIncluded: stringTask.domIncluded,\n          screenshotIncluded: stringTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiString');\n        const stringResult = await agent.aiString(prompt, options);\n        this.setResult(stringTask.name, stringResult);\n      } else if ('aiBoolean' in (flowItem as MidsceneYamlFlowItemAIBoolean)) {\n        const booleanTask = flowItem as MidsceneYamlFlowItemAIBoolean;\n        const prompt = booleanTask.aiBoolean;\n        const options = {\n          domIncluded: booleanTask.domIncluded,\n          screenshotIncluded: booleanTask.screenshotIncluded,\n        };\n        assert(prompt, 'missing prompt for aiBoolean');\n        const booleanResult = await agent.aiBoolean(prompt, options);\n        this.setResult(booleanTask.name, booleanResult);\n      } else if ('aiAsk' in (flowItem as MidsceneYamlFlowItemAIAsk)) {\n        const askTask = flowItem as MidsceneYamlFlowItemAIAsk;\n        const prompt = askTask.aiAsk;\n        assert(prompt, 'missing prompt for aiAsk');\n        const askResult = await agent.aiAsk(prompt);\n        this.setResult(askTask.name, askResult);\n      } else if ('aiLocate' in (flowItem as MidsceneYamlFlowItemAILocate)) {\n        const locateTask = flowItem as MidsceneYamlFlowItemAILocate;\n        const prompt = locateTask.aiLocate;\n        assert(prompt, 'missing prompt for aiLocate');\n        const locateResult = await agent.aiLocate(prompt, locateTask);\n        this.setResult(locateTask.name, locateResult);\n      } else if ('aiWaitFor' in (flowItem as MidsceneYamlFlowItemAIWaitFor)) {\n        const waitForTask = flowItem as MidsceneYamlFlowItemAIWaitFor;\n        const prompt = waitForTask.aiWaitFor;\n        assert(prompt, 'missing prompt for aiWaitFor');\n        const timeout = waitForTask.timeout;\n        await agent.aiWaitFor(prompt, { timeoutMs: timeout });\n      } else if ('sleep' in (flowItem as MidsceneYamlFlowItemSleep)) {\n        const sleepTask = flowItem as MidsceneYamlFlowItemSleep;\n        const ms = sleepTask.sleep;\n        let msNumber = ms;\n        if (typeof ms === 'string') {\n          msNumber = Number.parseInt(ms, 10);\n        }\n        assert(\n          msNumber && msNumber > 0,\n          `ms for sleep must be greater than 0, but got ${ms}`,\n        );\n        await new Promise((resolve) => setTimeout(resolve, msNumber));\n      } else if (\n        'javascript' in (flowItem as MidsceneYamlFlowItemEvaluateJavaScript)\n      ) {\n        const evaluateJavaScriptTask =\n          flowItem as MidsceneYamlFlowItemEvaluateJavaScript;\n\n        const result = await agent.evaluateJavaScript(\n          evaluateJavaScriptTask.javascript,\n        );\n        this.setResult(evaluateJavaScriptTask.name, result);\n      } else if (\n        'logScreenshot' in (flowItem as MidsceneYamlFlowItemLogScreenshot) ||\n        'recordToReport' in (flowItem as MidsceneYamlFlowItemLogScreenshot)\n      ) {\n        const recordTask = flowItem as MidsceneYamlFlowItemLogScreenshot;\n        const title =\n          recordTask.recordToReport ?? recordTask.logScreenshot ?? 'untitled';\n        const content = recordTask.content || '';\n        await agent.recordToReport(title, { content });\n      } else if ('aiInput' in (flowItem as MidsceneYamlFlowItemAIInput)) {\n        // may be input empty string ''\n        const { aiInput, value: rawValue, ...inputTask } =\n          flowItem as MidsceneYamlFlowItemAIInput;\n\n        // Compatibility with previous version:\n        // Old format: { aiInput: string (value), locate: TUserPrompt }\n        // New format - 1: { aiInput: TUserPrompt, value: string | number }\n        // New format - 2: { aiInput: undefined, locate: TUserPrompt, value: string | number }\n        let locatePrompt: TUserPrompt | undefined;\n        let value: string | number | undefined;\n        if ((inputTask as any).locate) {\n          // Old format - aiInput is the value, locate is the prompt\n          // Keep backward compatibility: empty string is treated as no value\n          value = (aiInput as string | number) || rawValue;\n          locatePrompt = (inputTask as any).locate;\n        } else {\n          // New format - aiInput is the prompt, value is the value\n          locatePrompt = aiInput || '';\n          value = rawValue;\n        }\n\n        // Convert value to string for Input action\n        await agent.callActionInActionSpace('Input', {\n          ...inputTask,\n          ...(value !== undefined ? { value: String(value) } : {}),\n          ...(locatePrompt\n            ? { locate: buildDetailedLocateParam(locatePrompt, inputTask) }\n            : {}),\n        });\n      } else if (\n        'aiKeyboardPress' in (flowItem as MidsceneYamlFlowItemAIKeyboardPress)\n      ) {\n        const { aiKeyboardPress, ...keyboardPressTask } =\n          flowItem as MidsceneYamlFlowItemAIKeyboardPress;\n\n        // Compatibility with previous version:\n        // Old format: { aiKeyboardPress: string (key), locate?: TUserPrompt }\n        // New format - 1: { aiKeyboardPress: TUserPrompt, keyName: string }\n        // New format - 2: { aiKeyboardPress: , locate?: TUserPrompt, keyName: string }\n        let locatePrompt: TUserPrompt | undefined;\n        let keyName: string | undefined;\n        if ((keyboardPressTask as any).locate) {\n          // Old format - aiKeyboardPress is the key, locate is the prompt\n          keyName = aiKeyboardPress as string;\n          locatePrompt = (keyboardPressTask as any).locate;\n        } else if (keyboardPressTask.keyName) {\n          // New format - aiKeyboardPress is the prompt, key is the key\n          keyName = keyboardPressTask.keyName;\n          locatePrompt = aiKeyboardPress;\n        } else {\n          keyName = aiKeyboardPress as string;\n        }\n\n        await agent.callActionInActionSpace('KeyboardPress', {\n          ...keyboardPressTask,\n          ...(keyName ? { keyName } : {}),\n          ...(locatePrompt\n            ? {\n                locate: buildDetailedLocateParam(\n                  locatePrompt,\n                  keyboardPressTask,\n                ),\n              }\n            : {}),\n        });\n      } else if ('aiScroll' in (flowItem as MidsceneYamlFlowItemAIScroll)) {\n        const { aiScroll, ...scrollTask } =\n          flowItem as MidsceneYamlFlowItemAIScroll;\n\n        // Compatibility with previous version:\n        // Old format: { aiScroll: null, locate?: TUserPrompt, direction, scrollType, distance? }\n        // New format - 1: { aiScroll: TUserPrompt, direction, scrollType, distance? }\n        // New format - 2: { aiScroll: undefined, locate: TUserPrompt, direction, scrollType, distance? }\n        let locatePrompt: TUserPrompt | undefined;\n        if ((scrollTask as any).locate) {\n          // Old format - locate is the prompt, aiScroll is null/ignored\n          locatePrompt = (scrollTask as any).locate;\n        } else {\n          // New format - aiScroll is the prompt, or no prompt for global scroll\n          locatePrompt = aiScroll;\n        }\n\n        await agent.callActionInActionSpace('Scroll', {\n          ...scrollTask,\n          ...(locatePrompt\n            ? { locate: buildDetailedLocateParam(locatePrompt, scrollTask) }\n            : {}),\n        });\n      } else {\n        // generic action, find the action in actionSpace\n\n        /* for aiTap, aiRightClick, the parameters are a flattened data for the 'locate', these are all valid data\n\n        - aiTap: 'search input box'\n        - aiTap: 'search input box'\n          deepThink: true\n          cacheable: false\n        - aiTap:\n          prompt: 'search input box'\n        - aiTap:\n          prompt: 'search input box'\n          deepThink: true\n          cacheable: false\n        */\n\n        const actionSpace = this.actionSpace;\n        let locatePromptShortcut: string | undefined;\n        const matchedAction = actionSpace.find((action) => {\n          const actionInterfaceAlias = action.interfaceAlias;\n          if (\n            actionInterfaceAlias &&\n            Object.prototype.hasOwnProperty.call(flowItem, actionInterfaceAlias)\n          ) {\n            locatePromptShortcut = flowItem[\n              actionInterfaceAlias as keyof typeof flowItem\n            ] as string;\n            return true;\n          }\n\n          const keyOfActionInActionSpace = action.name;\n          if (\n            Object.prototype.hasOwnProperty.call(\n              flowItem,\n              keyOfActionInActionSpace,\n            )\n          ) {\n            locatePromptShortcut = flowItem[\n              keyOfActionInActionSpace as keyof typeof flowItem\n            ] as string;\n            return true;\n          }\n\n          return false;\n        });\n\n        assert(\n          matchedAction,\n          `unknown flowItem in yaml: ${JSON.stringify(flowItem)}`,\n        );\n\n        // Create a new object instead of mutating the original flowItem\n        // This prevents issues when the same YAML script is executed multiple times\n        const flowItemForProcessing = locatePromptShortcut\n          ? { ...flowItem, prompt: locatePromptShortcut }\n          : flowItem;\n\n        const { locateParam, restParams } =\n          buildDetailedLocateParamAndRestParams(\n            locatePromptShortcut || '',\n            flowItemForProcessing as LocateOption,\n            [\n              matchedAction.name,\n              matchedAction.interfaceAlias || '_never_mind_',\n            ],\n          );\n\n        const flowParams = {\n          ...restParams,\n          locate: locateParam,\n        };\n\n        debug(\n          `matchedAction: ${matchedAction.name}`,\n          `flowParams: ${JSON.stringify(flowParams, null, 2)}`,\n        );\n        await agent.callActionInActionSpace(matchedAction.name, flowParams);\n      }\n    }\n    this.reportFile = agent.reportFile;\n    await this.flushUnstableLogContent();\n  }\n\n  async run() {\n    const { target, web, android, ios, tasks } = this.script;\n    const webEnv = web || target;\n    const androidEnv = android;\n    const iosEnv = ios;\n    const platform = webEnv || androidEnv || iosEnv;\n\n    this.setPlayerStatus('running');\n\n    let agent: Agent | null = null;\n    let freeFn: FreeFn[] = [];\n    try {\n      const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(\n        platform as T,\n      );\n      this.actionSpace = await newAgent.getActionSpace();\n      agent = newAgent;\n      const originalOnTaskStartTip = agent.onTaskStartTip;\n      agent.onTaskStartTip = (tip) => {\n        if (this.status === 'running') {\n          this.agentStatusTip = tip;\n        }\n        originalOnTaskStartTip?.(tip);\n      };\n      freeFn = [\n        ...(newFreeFn || []),\n        {\n          name: 'restore-agent-onTaskStartTip',\n          fn: () => {\n            if (agent) {\n              agent.onTaskStartTip = originalOnTaskStartTip;\n            }\n          },\n        },\n      ];\n    } catch (e) {\n      this.setPlayerStatus('error', e as Error);\n      return;\n    }\n    this.interfaceAgent = agent;\n\n    let taskIndex = 0;\n    this.setPlayerStatus('running');\n    let errorFlag = false;\n    while (taskIndex < tasks.length) {\n      const taskStatus = this.taskStatusList[taskIndex];\n      this.setTaskStatus(taskIndex, 'running' as any);\n      this.setTaskIndex(taskIndex);\n\n      try {\n        await this.playTask(taskStatus, this.interfaceAgent);\n        this.setTaskStatus(taskIndex, 'done' as any);\n      } catch (e) {\n        this.setTaskStatus(taskIndex, 'error' as any, e as Error);\n\n        if (taskStatus.continueOnError) {\n          // nothing more to do\n        } else {\n          this.reportFile = agent.reportFile;\n          errorFlag = true;\n          break;\n        }\n      }\n      this.reportFile = agent?.reportFile;\n      taskIndex++;\n    }\n\n    if (errorFlag) {\n      this.setPlayerStatus('error');\n    } else {\n      this.setPlayerStatus('done');\n    }\n    this.agentStatusTip = '';\n\n    // free the resources\n    for (const fn of freeFn) {\n      try {\n        // console.log('freeing', fn.name);\n        await fn.fn();\n        // console.log('freed', fn.name);\n      } catch (e) {\n        // console.error('error freeing', fn.name, e);\n      }\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","ScriptPlayer","value","keyToUse","console","status","error","taskIndex","taskIndexToNotify","taskStatus","index","statusValue","output","resolve","process","outputDir","dirname","existsSync","mkdirSync","writeFileSync","JSON","undefined","_this_interfaceAgent","content","filePath","agent","flow","assert","flowItemIndex","currentStep","Number","flowItem","actionTask","prompt","assertTask","msg","pass","thought","message","Error","queryTask","options","queryResult","numberTask","numberResult","stringTask","stringResult","booleanTask","booleanResult","askTask","askResult","locateTask","locateResult","waitForTask","timeout","sleepTask","ms","msNumber","Promise","setTimeout","evaluateJavaScriptTask","result","recordTask","title","aiInput","rawValue","inputTask","locatePrompt","String","buildDetailedLocateParam","aiKeyboardPress","keyboardPressTask","keyName","aiScroll","scrollTask","actionSpace","locatePromptShortcut","matchedAction","action","actionInterfaceAlias","keyOfActionInActionSpace","flowItemForProcessing","locateParam","restParams","buildDetailedLocateParamAndRestParams","flowParams","target","web","android","ios","tasks","webEnv","androidEnv","iosEnv","platform","freeFn","newAgent","newFreeFn","originalOnTaskStartTip","tip","e","errorFlag","fn","script","setupAgent","onTaskStatusChange","scriptPath","_this_target","_this_target1","_this_target2","ifInBrowser","ifInWorker","scriptName","basename","join","getMidsceneRunSubDir","Date","task","_task_flow"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;ACqDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAChB,MAAMC;IAwEH,UAAUP,GAAuB,EAAEQ,KAAU,EAAE;QACrD,MAAMC,WAAWT,OAAO,IAAI,CAAC,kBAAkB;QAC/C,IAAI,IAAI,CAAC,MAAM,CAACS,SAAS,EACvBC,QAAQ,IAAI,CAAC,CAAC,WAAW,EAAED,SAAS,+BAA+B,CAAC;QAEtE,IAAI,CAAC,MAAM,CAACA,SAAS,GAAGD;QAExB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEQ,gBAAgBG,MAA+B,EAAEC,KAAa,EAAE;QACtE,IAAI,CAAC,MAAM,GAAGD;QACd,IAAI,CAAC,YAAY,GAAGC;IACtB;IAEQ,8BAA8BC,SAAkB,EAAE;QACxD,MAAMC,oBACJ,AAAqB,YAArB,OAAOD,YAAyBA,YAAY,IAAI,CAAC,gBAAgB;QAEnE,IAAI,AAA6B,YAA7B,OAAOC,mBACT;QAGF,MAAMC,aAAa,IAAI,CAAC,cAAc,CAACD,kBAAkB;QACzD,IAAI,IAAI,CAAC,kBAAkB,EACzB,IAAI,CAAC,kBAAkB,CAACC;IAE5B;IAEA,MAAc,cACZC,KAAa,EACbC,WAAoC,EACpCL,KAAa,EACb;QACA,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,MAAM,GAAGC;QACpC,IAAIL,OACF,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,KAAK,GAAGJ;QAGrC,IAAI,CAAC,6BAA6B,CAACI;IACrC;IAEQ,aAAaH,SAAiB,EAAE;QACtC,IAAI,CAAC,gBAAgB,GAAGA;IAC1B;IAEQ,cAAc;QACpB,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,MAAMK,SAASC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM;YACjD,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQJ;YAC1B,IAAI,CAACK,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcP,QAAQQ,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,GAAGC,QAAW;QACrE;IACF;IAEQ,0BAA0B;QAChC,IAAI,IAAI,CAAC,kBAAkB,EAAE;gBACXC;YAAhB,MAAMC,UAAU,QAAAD,CAAAA,uBAAAA,IAAI,CAAC,cAAc,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,mBAAmB;YACxD,MAAME,WAAWX,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,kBAAkB;YAC/D,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQQ;YAC1B,IAAI,CAACP,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcK,UAAUJ,KAAK,SAAS,CAACG,SAAS,MAAM;QACxD;IACF;IAEA,MAAM,SAASd,UAAkC,EAAEgB,KAAY,EAAE;QAC/D,MAAM,EAAEC,IAAI,EAAE,GAAGjB;QACjBkB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,MAAM;QAEb,IAAK,MAAME,iBAAiBF,KAAM;YAChC,MAAMG,cAAcC,OAAO,QAAQ,CAACF,eAAe;YACnDnB,WAAW,WAAW,GAAGoB;YACzB,MAAME,WAAWL,IAAI,CAACE,cAAc;YACpC7B,MACE,CAAC,aAAa,EAAE6B,cAAc,WAAW,EAAER,KAAK,SAAS,CAACW,WAAW;YAEvE,IACE,WAAYA,YACZ,cAAeA,YACf,QAASA,UACT;gBACA,MAAMC,aAAaD;gBACnB,MAAME,SAASD,WAAW,KAAK,IAAIA,WAAW,QAAQ,IAAIA,WAAW,EAAE;gBACvEL,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMR,MAAM,KAAK,CAACQ,QAAQ;oBACxB,WAAWD,WAAW,SAAS;gBACjC;YACF,OAAO,IAAI,cAAeD,UAA2C;gBACnE,MAAMG,aAAaH;gBACnB,MAAME,SAASC,WAAW,QAAQ;gBAClC,MAAMC,MAAMD,WAAW,YAAY;gBACnCP,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAM,EAAEG,IAAI,EAAEC,OAAO,EAAEC,OAAO,EAAE,GAC7B,MAAMb,MAAM,QAAQ,CAACQ,QAAQE,KAAK;oBACjC,iBAAiB;gBACnB,MAAO,CAAC;gBAEV,IAAI,CAAC,SAAS,CAACD,WAAW,IAAI,EAAE;oBAC9BE;oBACAC;oBACAC;gBACF;gBAEA,IAAI,CAACF,MACH,MAAM,IAAIG,MAAMD;YAEpB,OAAO,IAAI,aAAcP,UAA0C;gBACjE,MAAMS,YAAYT;gBAClB,MAAME,SAASO,UAAU,OAAO;gBAChC,MAAMC,UAAU;oBACd,aAAaD,UAAU,WAAW;oBAClC,oBAAoBA,UAAU,kBAAkB;gBAClD;gBACAb,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMS,cAAc,MAAMjB,MAAM,OAAO,CAACQ,QAAQQ;gBAChD,IAAI,CAAC,SAAS,CAACD,UAAU,IAAI,EAAEE;YACjC,OAAO,IAAI,cAAeX,UAA2C;gBACnE,MAAMY,aAAaZ;gBACnB,MAAME,SAASU,WAAW,QAAQ;gBAClC,MAAMF,UAAU;oBACd,aAAaE,WAAW,WAAW;oBACnC,oBAAoBA,WAAW,kBAAkB;gBACnD;gBACAhB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMW,eAAe,MAAMnB,MAAM,QAAQ,CAACQ,QAAQQ;gBAClD,IAAI,CAAC,SAAS,CAACE,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,cAAeb,UAA2C;gBACnE,MAAMc,aAAad;gBACnB,MAAME,SAASY,WAAW,QAAQ;gBAClC,MAAMJ,UAAU;oBACd,aAAaI,WAAW,WAAW;oBACnC,oBAAoBA,WAAW,kBAAkB;gBACnD;gBACAlB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMa,eAAe,MAAMrB,MAAM,QAAQ,CAACQ,QAAQQ;gBAClD,IAAI,CAAC,SAAS,CAACI,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,eAAgBf,UAA4C;gBACrE,MAAMgB,cAAchB;gBACpB,MAAME,SAASc,YAAY,SAAS;gBACpC,MAAMN,UAAU;oBACd,aAAaM,YAAY,WAAW;oBACpC,oBAAoBA,YAAY,kBAAkB;gBACpD;gBACApB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMe,gBAAgB,MAAMvB,MAAM,SAAS,CAACQ,QAAQQ;gBACpD,IAAI,CAAC,SAAS,CAACM,YAAY,IAAI,EAAEC;YACnC,OAAO,IAAI,WAAYjB,UAAwC;gBAC7D,MAAMkB,UAAUlB;gBAChB,MAAME,SAASgB,QAAQ,KAAK;gBAC5BtB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMiB,YAAY,MAAMzB,MAAM,KAAK,CAACQ;gBACpC,IAAI,CAAC,SAAS,CAACgB,QAAQ,IAAI,EAAEC;YAC/B,OAAO,IAAI,cAAenB,UAA2C;gBACnE,MAAMoB,aAAapB;gBACnB,MAAME,SAASkB,WAAW,QAAQ;gBAClCxB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMmB,eAAe,MAAM3B,MAAM,QAAQ,CAACQ,QAAQkB;gBAClD,IAAI,CAAC,SAAS,CAACA,WAAW,IAAI,EAAEC;YAClC,OAAO,IAAI,eAAgBrB,UAA4C;gBACrE,MAAMsB,cAActB;gBACpB,MAAME,SAASoB,YAAY,SAAS;gBACpC1B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,QAAQ;gBACf,MAAMqB,UAAUD,YAAY,OAAO;gBACnC,MAAM5B,MAAM,SAAS,CAACQ,QAAQ;oBAAE,WAAWqB;gBAAQ;YACrD,OAAO,IAAI,WAAYvB,UAAwC;gBAC7D,MAAMwB,YAAYxB;gBAClB,MAAMyB,KAAKD,UAAU,KAAK;gBAC1B,IAAIE,WAAWD;gBACf,IAAI,AAAc,YAAd,OAAOA,IACTC,WAAW3B,OAAO,QAAQ,CAAC0B,IAAI;gBAEjC7B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8B,YAAYA,WAAW,GACvB,CAAC,6CAA6C,EAAED,IAAI;gBAEtD,MAAM,IAAIE,QAAQ,CAAC7C,UAAY8C,WAAW9C,SAAS4C;YACrD,OAAO,IACL,gBAAiB1B,UACjB;gBACA,MAAM6B,yBACJ7B;gBAEF,MAAM8B,SAAS,MAAMpC,MAAM,kBAAkB,CAC3CmC,uBAAuB,UAAU;gBAEnC,IAAI,CAAC,SAAS,CAACA,uBAAuB,IAAI,EAAEC;YAC9C,OAAO,IACL,mBAAoB9B,YACpB,oBAAqBA,UACrB;gBACA,MAAM+B,aAAa/B;gBACnB,MAAMgC,QACJD,WAAW,cAAc,IAAIA,WAAW,aAAa,IAAI;gBAC3D,MAAMvC,UAAUuC,WAAW,OAAO,IAAI;gBACtC,MAAMrC,MAAM,cAAc,CAACsC,OAAO;oBAAExC;gBAAQ;YAC9C,OAAO,IAAI,aAAcQ,UAA0C;gBAEjE,MAAM,EAAEiC,OAAO,EAAE,OAAOC,QAAQ,EAAE,GAAGC,WAAW,GAC9CnC;gBAMF,IAAIoC;gBACJ,IAAIjE;gBACJ,IAAKgE,UAAkB,MAAM,EAAE;oBAG7BhE,QAAS8D,WAA+BC;oBACxCE,eAAgBD,UAAkB,MAAM;gBAC1C,OAAO;oBAELC,eAAeH,WAAW;oBAC1B9D,QAAQ+D;gBACV;gBAGA,MAAMxC,MAAM,uBAAuB,CAAC,SAAS;oBAC3C,GAAGyC,SAAS;oBACZ,GAAIhE,AAAUmB,WAAVnB,QAAsB;wBAAE,OAAOkE,OAAOlE;oBAAO,IAAI,CAAC,CAAC;oBACvD,GAAIiE,eACA;wBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcD;oBAAW,IAC5D,CAAC,CAAC;gBACR;YACF,OAAO,IACL,qBAAsBnC,UACtB;gBACA,MAAM,EAAEuC,eAAe,EAAE,GAAGC,mBAAmB,GAC7CxC;gBAMF,IAAIoC;gBACJ,IAAIK;gBACJ,IAAKD,kBAA0B,MAAM,EAAE;oBAErCC,UAAUF;oBACVH,eAAgBI,kBAA0B,MAAM;gBAClD,OAAO,IAAIA,kBAAkB,OAAO,EAAE;oBAEpCC,UAAUD,kBAAkB,OAAO;oBACnCJ,eAAeG;gBACjB,OACEE,UAAUF;gBAGZ,MAAM7C,MAAM,uBAAuB,CAAC,iBAAiB;oBACnD,GAAG8C,iBAAiB;oBACpB,GAAIC,UAAU;wBAAEA;oBAAQ,IAAI,CAAC,CAAC;oBAC9B,GAAIL,eACA;wBACE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EACNF,cACAI;oBAEJ,IACA,CAAC,CAAC;gBACR;YACF,OAAO,IAAI,cAAexC,UAA2C;gBACnE,MAAM,EAAE0C,QAAQ,EAAE,GAAGC,YAAY,GAC/B3C;gBAMF,IAAIoC;gBAGFA,eAFGO,WAAmB,MAAM,GAEZA,WAAmB,MAAM,GAG1BD;gBAGjB,MAAMhD,MAAM,uBAAuB,CAAC,UAAU;oBAC5C,GAAGiD,UAAU;oBACb,GAAIP,eACA;wBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcO;oBAAY,IAC7D,CAAC,CAAC;gBACR;YACF,OAAO;gBAiBL,MAAMC,cAAc,IAAI,CAAC,WAAW;gBACpC,IAAIC;gBACJ,MAAMC,gBAAgBF,YAAY,IAAI,CAAC,CAACG;oBACtC,MAAMC,uBAAuBD,OAAO,cAAc;oBAClD,IACEC,wBACApF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACoC,UAAUgD,uBAC/C;wBACAH,uBAAuB7C,QAAQ,CAC7BgD,qBACD;wBACD,OAAO;oBACT;oBAEA,MAAMC,2BAA2BF,OAAO,IAAI;oBAC5C,IACEnF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAClCoC,UACAiD,2BAEF;wBACAJ,uBAAuB7C,QAAQ,CAC7BiD,yBACD;wBACD,OAAO;oBACT;oBAEA,OAAO;gBACT;gBAEArD,IAAAA,sBAAAA,MAAAA,AAAAA,EACEkD,eACA,CAAC,0BAA0B,EAAEzD,KAAK,SAAS,CAACW,WAAW;gBAKzD,MAAMkD,wBAAwBL,uBAC1B;oBAAE,GAAG7C,QAAQ;oBAAE,QAAQ6C;gBAAqB,IAC5C7C;gBAEJ,MAAM,EAAEmD,WAAW,EAAEC,UAAU,EAAE,GAC/BC,AAAAA,IAAAA,kCAAAA,qCAAAA,AAAAA,EACER,wBAAwB,IACxBK,uBACA;oBACEJ,cAAc,IAAI;oBAClBA,cAAc,cAAc,IAAI;iBACjC;gBAGL,MAAMQ,aAAa;oBACjB,GAAGF,UAAU;oBACb,QAAQD;gBACV;gBAEAnF,MACE,CAAC,eAAe,EAAE8E,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAEzD,KAAK,SAAS,CAACiE,YAAY,MAAM,IAAI;gBAEtD,MAAM5D,MAAM,uBAAuB,CAACoD,cAAc,IAAI,EAAEQ;YAC1D;QACF;QACA,IAAI,CAAC,UAAU,GAAG5D,MAAM,UAAU;QAClC,MAAM,IAAI,CAAC,uBAAuB;IACpC;IAEA,MAAM,MAAM;QACV,MAAM,EAAE6D,MAAM,EAAEC,GAAG,EAAEC,OAAO,EAAEC,GAAG,EAAEC,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM;QACxD,MAAMC,SAASJ,OAAOD;QACtB,MAAMM,aAAaJ;QACnB,MAAMK,SAASJ;QACf,MAAMK,WAAWH,UAAUC,cAAcC;QAEzC,IAAI,CAAC,eAAe,CAAC;QAErB,IAAIpE,QAAsB;QAC1B,IAAIsE,SAAmB,EAAE;QACzB,IAAI;YACF,MAAM,EAAE,OAAOC,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAClEH;YAEF,IAAI,CAAC,WAAW,GAAG,MAAME,SAAS,cAAc;YAChDvE,QAAQuE;YACR,MAAME,yBAAyBzE,MAAM,cAAc;YACnDA,MAAM,cAAc,GAAG,CAAC0E;gBACtB,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,cAAc,GAAGA;gBAExBD,QAAAA,0BAAAA,uBAAyBC;YAC3B;YACAJ,SAAS;mBACHE,aAAa,EAAE;gBACnB;oBACE,MAAM;oBACN,IAAI;wBACF,IAAIxE,OACFA,MAAM,cAAc,GAAGyE;oBAE3B;gBACF;aACD;QACH,EAAE,OAAOE,GAAG;YACV,IAAI,CAAC,eAAe,CAAC,SAASA;YAC9B;QACF;QACA,IAAI,CAAC,cAAc,GAAG3E;QAEtB,IAAIlB,YAAY;QAChB,IAAI,CAAC,eAAe,CAAC;QACrB,IAAI8F,YAAY;QAChB,MAAO9F,YAAYmF,MAAM,MAAM,CAAE;YAC/B,MAAMjF,aAAa,IAAI,CAAC,cAAc,CAACF,UAAU;YACjD,IAAI,CAAC,aAAa,CAACA,WAAW;YAC9B,IAAI,CAAC,YAAY,CAACA;YAElB,IAAI;gBACF,MAAM,IAAI,CAAC,QAAQ,CAACE,YAAY,IAAI,CAAC,cAAc;gBACnD,IAAI,CAAC,aAAa,CAACF,WAAW;YAChC,EAAE,OAAO6F,GAAG;gBACV,IAAI,CAAC,aAAa,CAAC7F,WAAW,SAAgB6F;gBAE9C,IAAI3F,WAAW,eAAe;qBAEvB;oBACL,IAAI,CAAC,UAAU,GAAGgB,MAAM,UAAU;oBAClC4E,YAAY;oBACZ;gBACF;YACF;YACA,IAAI,CAAC,UAAU,GAAG5E,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,UAAU;YACnClB;QACF;QAEA,IAAI8F,WACF,IAAI,CAAC,eAAe,CAAC;aAErB,IAAI,CAAC,eAAe,CAAC;QAEvB,IAAI,CAAC,cAAc,GAAG;QAGtB,KAAK,MAAMC,MAAMP,OACf,IAAI;YAEF,MAAMO,GAAG,EAAE;QAEb,EAAE,OAAOF,GAAG,CAEZ;IAEJ;IAjgBA,YACUG,MAA0B,EAC1BC,UAGN,EACKC,kBAAiE,EACxEC,UAAmB,CACnB;YAaWC,cAgBOC,eAKPC;;;;QAxDb,uBAAO,oBAAP;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAO,cAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,sBAAR;QACA,uBAAO,UAAP;QACA,uBAAO,sBAAP;QACA,uBAAO,gBAAP;QACA,uBAAQ,kBAAR;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,eAAR;QACA,uBAAQ,cAAR;aAEUN,MAAM,GAANA;aACAC,UAAU,GAAVA;aAIDC,kBAAkB,GAAlBA;aAnBF,cAAc,GAA6B,EAAE;aAC7C,MAAM,GAA4B;aAGjC,kBAAkB,GAAG;aAIrB,cAAc,GAAiB;aAG/B,WAAW,GAAmB,EAAE;QAWtC,IAAI,CAAC,UAAU,GAAGC;QAClB,IAAI,CAAC,MAAM,GAAG,CAAC;QACf,IAAI,CAAC,MAAM,GACTH,OAAO,MAAM,IACbA,OAAO,GAAG,IACVA,OAAO,OAAO,IACdA,OAAO,GAAG,IACVA,OAAO,MAAM;QAEf,IAAIO,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAAE;YAC7B,IAAI,CAAC,MAAM,GAAG1F;YACdtB,MAAM;QACR,OAAO,IAAI,QAAA4G,CAAAA,eAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,MAAM,EAAE;YAC9B,IAAI,CAAC,MAAM,GAAG9F,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM;YACvDf,MAAM,mCAAmC,IAAI,CAAC,MAAM;QACtD,OAAO;YACL,MAAMiH,aAAa,IAAI,CAAC,UAAU,GAC9BC,AAAAA,IAAAA,mCAAAA,QAAAA,AAAAA,EAAS,IAAI,CAAC,UAAU,EAAE,SAAS,OAAO,CAAC,eAAe,MAC1D;YACJ,IAAI,CAAC,MAAM,GAAGC,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACZC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB,GAAGH,WAAW,CAAC,EAAEI,KAAK,GAAG,GAAG,KAAK,CAAC;YAEpCrH,MAAM,iCAAiC,IAAI,CAAC,MAAM;QACpD;QAEA,IAAI+G,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAC3B,IAAI,CAAC,kBAAkB,GAAG1F;aACrB,IAAI,AAA2C,YAA3C,gBAAOuF,CAAAA,gBAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,kBAAkB,AAAD,GAC9C,IAAI,CAAC,kBAAkB,GAAG/F,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EACxBC,QAAQ,GAAG,IACX,IAAI,CAAC,MAAM,CAAC,kBAAkB;aAE3B,IAAI+F,AAAAA,SAAAA,CAAAA,gBAAAA,IAAI,CAAC,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,kBAAkB,AAAD,MAAM,MAC7C,IAAI,CAAC,kBAAkB,GAAGK,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACxBC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB;QAIJ,IAAI,CAAC,cAAc,GAAIZ,AAAAA,CAAAA,OAAO,KAAK,IAAI,EAAC,EAAG,GAAG,CAAC,CAACc,MAAM9G;gBAIxC+G;mBAJuD;gBACnE,GAAGD,IAAI;gBACP,OAAO9G;gBACP,QAAQ;gBACR,YAAY+G,AAAAA,SAAAA,CAAAA,aAAAA,KAAK,IAAI,AAAD,IAARA,KAAAA,IAAAA,WAAW,MAAM,AAAD,KAAK;YACnC;;IACF;AA2cF"}

package/dist/types/agent/agent.d.ts

@@ -90,6 +90,16 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
      * @deprecated Use aiScroll(locatePrompt, opt) instead where opt contains the scroll parameters
      */
     aiScroll(scrollParam: ScrollParam, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
+    aiAct(taskPrompt: string, opt?: {
+        cacheable?: boolean;
+    }): Promise<{
+        result: Record<string, any>;
+    } | {
+        yamlFlow?: import("../yaml").MidsceneYamlFlowItem[];
+    } | undefined>;
+    /**
+     * @deprecated Use {@link Agent.aiAct} instead.
+     */
     aiAction(taskPrompt: string, opt?: {
         cacheable?: boolean;
     }): Promise<{
@@ -123,6 +133,12 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     }>;
     evaluateJavaScript(script: string): Promise<any>;
     destroy(): Promise<void>;
+    recordToReport(title?: string, opt?: {
+        content: string;
+    }): Promise<void>;
+    /**
+     * @deprecated Use {@link Agent.recordToReport} instead.
+     */
     logScreenshot(title?: string, opt?: {
         content: string;
     }): Promise<void>;

package/dist/types/agent/execution-session.d.ts

@@ -0,0 +1,27 @@
+import { TaskRunner } from '../task-runner';
+import type { ExecutionTaskApply, ExecutionTaskProgressOptions, UIContext } from '../types';
+/**
+ * Thin wrapper around {@link TaskRunner} that represents a single linear execution run.
+ */
+export declare class ExecutionSession {
+    private readonly runner;
+    constructor(name: string, contextProvider: () => Promise<UIContext>, options?: ExecutionTaskProgressOptions & {
+        tasks?: ExecutionTaskApply[];
+    });
+    append(tasks: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void>;
+    appendAndRun(tasks: ExecutionTaskApply[] | ExecutionTaskApply): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    run(): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
+    isInErrorState(): boolean;
+    latestErrorTask(): import("../types").ExecutionTask | null;
+    appendErrorPlan(errorMsg: string): Promise<{
+        output: undefined;
+        runner: TaskRunner;
+    }>;
+    getRunner(): TaskRunner;
+}

package/dist/types/agent/task-builder.d.ts

@@ -0,0 +1,24 @@
+import type { AbstractInterface } from '../device';
+import type Insight from '../insight';
+import type { DetailedLocateParam, ExecutionTaskApply, PlanningAction, PlanningLocateParam } from '../types';
+import type { IModelConfig } from '@midscene/shared/env';
+import type { TaskCache } from './task-cache';
+export declare function locatePlanForLocate(param: string | DetailedLocateParam): PlanningAction<PlanningLocateParam>;
+interface TaskBuilderDeps {
+    interfaceInstance: AbstractInterface;
+    insight: Insight;
+    taskCache?: TaskCache;
+}
+interface BuildOptions {
+    cacheable?: boolean;
+}
+export declare class TaskBuilder {
+    private readonly interface;
+    private readonly insight;
+    private readonly taskCache?;
+    constructor({ interfaceInstance, insight, taskCache }: TaskBuilderDeps);
+    build(plans: PlanningAction[], modelConfig: IModelConfig, options?: BuildOptions): Promise<{
+        tasks: ExecutionTaskApply[];
+    }>;
+}
+export {};

package/dist/types/agent/tasks.d.ts

@@ -1,20 +1,21 @@
-import { Executor } from '../ai-model/action-executor';
 import type { TMultimodalPrompt, TUserPrompt } from '../ai-model/common';
 import type { AbstractInterface } from '../device';
+import type { TaskRunner } from '../task-runner';
 import type Insight from '../insight';
-import type { DetailedLocateParam, ExecutionTaskApply, ExecutionTaskProgressOptions, InsightExtractOption, InsightExtractParam, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor, PlanningLocateParam } from '../types';
+import type { ExecutionTaskApply, ExecutionTaskProgressOptions, InsightExtractOption, InsightExtractParam, MidsceneYamlFlowItem, PlanningAction, PlanningActionParamWaitFor } from '../types';
 import { type IModelConfig } from '@midscene/shared/env';
 import type { TaskCache } from './task-cache';
+export { locatePlanForLocate } from './task-builder';
 interface ExecutionResult<OutputType = any> {
     output: OutputType;
     thought?: string;
-    executor: Executor;
+    runner: TaskRunner;
 }
-export declare function locatePlanForLocate(param: string | DetailedLocateParam): PlanningAction<PlanningLocateParam>;
 export declare class TaskExecutor {
     interface: AbstractInterface;
     insight: Insight;
     taskCache?: TaskCache;
+    private readonly taskBuilder;
     private conversationHistory;
     onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];
     replanningCycleLimit?: number;
@@ -24,14 +25,12 @@ export declare class TaskExecutor {
         onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
         replanningCycleLimit?: number;
     });
-    private recordScreenshot;
-    private prependExecutorWithScreenshot;
+    private createExecutionSession;
     convertPlanToExecutable(plans: PlanningAction[], modelConfig: IModelConfig, cacheable?: boolean): Promise<{
-        tasks: ExecutionTaskApply<any, any, any, any>[];
+        tasks: ExecutionTaskApply[];
     }>;
-    private setupPlanningContext;
     loadYamlFlowAsPlanning(userInstruction: string, yamlString: string): Promise<{
-        executor: Executor;
+        runner: TaskRunner;
     }>;
     private createPlanningTask;
     runPlans(title: string, plans: PlanningAction[], modelConfig: IModelConfig): Promise<ExecutionResult>;
@@ -41,8 +40,6 @@ export declare class TaskExecutor {
     } | undefined>>;
     private createTypeQueryTask;
     createTypeQueryExecution<T>(type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert', demand: InsightExtractParam, modelConfig: IModelConfig, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<ExecutionResult<T>>;
-    private appendErrorPlan;
     taskForSleep(timeMs: number, modelConfig: IModelConfig): Promise<ExecutionTaskApply<any, any, any, any>>;
     waitFor(assertion: TUserPrompt, opt: PlanningActionParamWaitFor, modelConfig: IModelConfig): Promise<ExecutionResult<void>>;
 }
-export {};

package/dist/types/agent/ui-utils.d.ts

@@ -1,7 +1,7 @@
-import type { AndroidPullParam, DetailedLocateParam, ExecutionTask, ScrollParam } from '../types';
+import type { DetailedLocateParam, ExecutionTask, PullParam, ScrollParam } from '../types';
 export declare function typeStr(task: ExecutionTask): any;
 export declare function locateParamStr(locate?: DetailedLocateParam | string): string;
 export declare function scrollParamStr(scrollParam?: ScrollParam): string;
-export declare function pullParamStr(pullParam?: AndroidPullParam): string;
+export declare function pullParamStr(pullParam?: PullParam): string;
 export declare function taskTitleStr(type: 'Tap' | 'Hover' | 'Input' | 'RightClick' | 'KeyboardPress' | 'Scroll' | 'Action' | 'Query' | 'Assert' | 'WaitFor' | 'Locate' | 'Boolean' | 'Number' | 'String', prompt: string): string;
 export declare function paramStr(task: ExecutionTask): string;

package/dist/types/agent/utils.d.ts

@@ -1,7 +1,7 @@
 import type { TMultimodalPrompt, TUserPrompt } from '../ai-model/common';
 import type { AbstractInterface } from '../device';
 import type { BaseElement, ElementCacheFeature, ElementTreeNode, ExecutionDump, ExecutorContext, LocateResultElement, PlanningLocateParam, UIContext } from '../types';
-import type { TaskExecutor } from './tasks';
+import type { TaskCache } from './task-cache';
 export declare function commonContextParser(interfaceInstance: AbstractInterface, _opt: {
     uploadServerUrl?: string;
 }): Promise<UIContext>;
@@ -14,7 +14,10 @@ export declare function printReportMsg(filepath: string): void;
 export declare function getCurrentExecutionFile(trace?: string): string | false;
 export declare function generateCacheId(fileName?: string): string;
 export declare function matchElementFromPlan(planLocateParam: PlanningLocateParam, tree: ElementTreeNode<BaseElement>): any;
-export declare function matchElementFromCache(taskExecutor: TaskExecutor, cacheEntry: ElementCacheFeature | undefined, cachePrompt: TUserPrompt, cacheable: boolean | undefined): Promise<LocateResultElement | undefined>;
+export declare function matchElementFromCache(context: {
+    taskCache?: TaskCache;
+    interfaceInstance: AbstractInterface;
+}, cacheEntry: ElementCacheFeature | undefined, cachePrompt: TUserPrompt, cacheable: boolean | undefined): Promise<LocateResultElement | undefined>;
 export declare function trimContextByViewport(execution: ExecutionDump): {
     tasks: {
         type: any;

package/dist/types/ai-model/common.d.ts

@@ -19,7 +19,6 @@ export declare function adaptBbox(bbox: number[], width: number, height: number,
 export declare function normalized01000(bbox: number[], width: number, height: number): [number, number, number, number];
 export declare function adaptGeminiBbox(bbox: number[], width: number, height: number): [number, number, number, number];
 export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number, rightLimit?: number, bottomLimit?: number, vlMode?: TVlModeTypes | undefined): Rect;
-export declare function warnGPT4oSizeLimit(size: Size, modelName: string): void;
 export declare function mergeRects(rects: Rect[]): {
     left: number;
     top: number;

package/dist/types/ai-model/prompt/llm-locator.d.ts

@@ -1,8 +1,6 @@
 import { PromptTemplate } from '@langchain/core/prompts';
 import type { TVlModeTypes } from '@midscene/shared/env';
-import type { ResponseFormatJSONSchema } from 'openai/resources/index';
 export declare function systemPromptToLocateElement(vlMode: TVlModeTypes | undefined): string;
-export declare const locatorSchema: ResponseFormatJSONSchema;
 export declare const findElementPrompt: PromptTemplate<{
     pageDescription: any;
     targetElementDescription: any;

package/dist/types/index.d.ts

@@ -1,12 +1,13 @@
 import { z } from 'zod';
-import { Executor } from './ai-model/action-executor';
+import { TaskRunner } from './task-runner';
 import Insight from './insight/index';
 import { getVersion } from './utils';
 export { plan, describeUserPage, AiLocateElement, getMidsceneLocationSchema, type MidsceneLocationResultType, PointSchema, SizeSchema, RectSchema, TMultimodalPromptSchema, TUserPromptSchema, type TMultimodalPrompt, type TUserPrompt, } from './ai-model/index';
-export { MIDSCENE_MODEL_NAME } from '@midscene/shared/env';
+export { MIDSCENE_MODEL_NAME, type CreateOpenAIClientFn, } from '@midscene/shared/env';
 export type * from './types';
+export { InsightError } from './types';
 export { z };
 export default Insight;
-export { Executor, Insight, getVersion };
+export { TaskRunner, Insight, getVersion };
 export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlConfigResult, MidsceneYamlConfig, MidsceneYamlScriptWebEnv, MidsceneYamlScriptAndroidEnv, MidsceneYamlScriptIOSEnv, MidsceneYamlScriptEnv, LocateOption, DetailedLocateParam, } from './yaml';
 export { Agent, type AgentOpt, createAgent } from './agent';

package/dist/types/insight/index.d.ts

@@ -1,5 +1,5 @@
 import { callAIWithObjectResponse } from '../ai-model/index';
-import type { AIDescribeElementResponse, AIUsageInfo, BaseElement, DetailedLocateParam, DumpSubscriber, InsightAction, InsightExtractOption, InsightExtractParam, InsightTaskInfo, LocateResult, Rect, UIContext } from '../types';
+import type { AIDescribeElementResponse, BaseElement, DetailedLocateParam, InsightExtractOption, InsightExtractParam, InsightExtractResult, InsightTaskInfo, LocateResultWithDump, Rect, UIContext } from '../types';
 import { type IModelConfig } from '@midscene/shared/env';
 import type { TMultimodalPrompt } from '../ai-model/common';
 export interface LocateOpts {
@@ -13,17 +13,12 @@ interface InsightOptions {
     aiVendorFn?: typeof callAIWithObjectResponse;
 }
 export default class Insight<ElementType extends BaseElement = BaseElement, ContextType extends UIContext<ElementType> = UIContext<ElementType>> {
-    contextRetrieverFn: (action: InsightAction) => Promise<ContextType> | ContextType;
+    contextRetrieverFn: () => Promise<ContextType> | ContextType;
     aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined>;
-    onceDumpUpdatedFn?: DumpSubscriber;
     taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;
-    constructor(context: ContextType | ((action: InsightAction) => Promise<ContextType> | ContextType), opt?: InsightOptions);
-    locate(query: DetailedLocateParam, opt: LocateOpts, modelConfig: IModelConfig): Promise<LocateResult>;
-    extract<T>(dataDemand: InsightExtractParam, modelConfig: IModelConfig, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<{
-        data: T;
-        thought?: string;
-        usage?: AIUsageInfo;
-    }>;
+    constructor(context: ContextType | (() => Promise<ContextType> | ContextType), opt?: InsightOptions);
+    locate(query: DetailedLocateParam, opt: LocateOpts, modelConfig: IModelConfig): Promise<LocateResultWithDump>;
+    extract<T>(dataDemand: InsightExtractParam, modelConfig: IModelConfig, opt?: InsightExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<InsightExtractResult<T>>;
     describe(target: Rect | [number, number], modelConfig: IModelConfig, opt?: {
         deepThink?: boolean;
     }): Promise<Pick<AIDescribeElementResponse, 'description'>>;

package/dist/types/insight/utils.d.ts

@@ -1,2 +1,2 @@
-import type { DumpSubscriber, PartialInsightDumpFromSDK } from '../types';
-export declare function emitInsightDump(data: PartialInsightDumpFromSDK, dumpSubscriber?: DumpSubscriber): void;
+import type { InsightDump, PartialInsightDumpFromSDK } from '../types';
+export declare function createInsightDump(data: PartialInsightDumpFromSDK): InsightDump;

package/dist/types/{ai-model/action-executor.d.ts → task-runner.d.ts}

@@ -1,14 +1,21 @@
-import type { ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions } from '../types';
-export declare class Executor {
+import type { ExecutionDump, ExecutionTask, ExecutionTaskApply, ExecutionTaskProgressOptions, UIContext } from './types';
+export declare class TaskRunner {
     name: string;
     tasks: ExecutionTask[];
     status: 'init' | 'pending' | 'running' | 'completed' | 'error';
     onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];
-    constructor(name: string, options?: ExecutionTaskProgressOptions & {
+    private readonly uiContextBuilder;
+    constructor(name: string, uiContextBuilder: () => Promise<UIContext>, options?: ExecutionTaskProgressOptions & {
         tasks?: ExecutionTaskApply[];
     });
+    private captureScreenshot;
+    private attachRecorderItem;
     private markTaskAsPending;
     append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void>;
+    appendAndFlush(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<{
+        output: any;
+        thought?: string;
+    } | undefined>;
     flush(): Promise<{
         output: any;
         thought?: string;
@@ -16,4 +23,8 @@ export declare class Executor {
     isInErrorState(): boolean;
     latestErrorTask(): ExecutionTask | null;
     dump(): ExecutionDump;
+    appendErrorPlan(errorMsg: string): Promise<{
+        output: undefined;
+        runner: TaskRunner;
+    }>;
 }

package/dist/types/types.d.ts

@@ -1,5 +1,5 @@
 import type { NodeType } from '@midscene/shared/constants';
-import type { TModelConfigFn } from '@midscene/shared/env';
+import type { CreateOpenAIClientFn, TModelConfigFn } from '@midscene/shared/env';
 import type { BaseElement, ElementTreeNode, Rect, Size } from '@midscene/shared/types';
 import type { z } from 'zod';
 import type { TUserPrompt } from './ai-model/common';
@@ -154,7 +154,19 @@ export interface InsightDump extends DumpMeta {
     output?: any;
 }
 export type PartialInsightDumpFromSDK = Omit<InsightDump, 'logTime' | 'logId' | 'model_name'>;
-export type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;
+export interface InsightResultBase {
+    dump: InsightDump;
+}
+export type LocateResultWithDump = LocateResult & InsightResultBase;
+export interface InsightExtractResult<T> extends InsightResultBase {
+    data: T;
+    thought?: string;
+    usage?: AIUsageInfo;
+}
+export declare class InsightError extends Error {
+    dump: InsightDump;
+    constructor(message: string, dump: InsightDump);
+}
 export interface LiteUISection {
     name: string;
     description: string;
@@ -216,10 +228,10 @@ export interface PlanningActionParamError {
     thought: string;
 }
 export type PlanningActionParamWaitFor = AgentWaitForOpt & {};
-export interface AndroidLongPressParam {
+export interface LongPressParam {
     duration?: number;
 }
-export interface AndroidPullParam {
+export interface PullParam {
     direction: 'up' | 'down';
     distance?: number;
     duration?: number;
@@ -251,6 +263,7 @@ export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action' | 'Assertion'
 export interface ExecutorContext {
     task: ExecutionTask;
     element?: LocateResultElement | null;
+    uiContext?: UIContext;
 }
 export interface ExecutionTaskApply<Type extends ExecutionTaskType = any, TaskParam = any, TaskOutput = any, TaskLog = any> {
     type: Type;
@@ -398,6 +411,36 @@ export interface AgentOpt {
     modelConfig?: TModelConfigFn;
     cache?: Cache;
     replanningCycleLimit?: number;
+    /**
+     * Custom OpenAI client factory function
+     *
+     * If provided, this function will be called to create OpenAI client instances
+     * for each AI call, allowing you to:
+     * - Wrap clients with observability tools (langsmith, langfuse)
+     * - Use custom OpenAI-compatible clients
+     * - Apply different configurations based on intent
+     *
+     * @param config - Resolved model configuration
+     * @returns OpenAI client instance (original or wrapped)
+     *
+     * @example
+     * ```typescript
+     * createOpenAIClient: (config) => {
+     *   const openai = new OpenAI({
+     *     apiKey: config.openaiApiKey,
+     *     baseURL: config.openaiBaseURL,
+     *   });
+     *
+     *   // Wrap with langsmith for planning tasks
+     *   if (config.intent === 'planning') {
+     *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });
+     *   }
+     *
+     *   return openai;
+     * }
+     * ```
+     */
+    createOpenAIClient?: CreateOpenAIClientFn;
 }
 export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted';
 export interface ReportFileWithAttributes {

package/dist/types/yaml.d.ts

@@ -80,8 +80,9 @@ export interface MidsceneYamlScriptIOSEnv extends MidsceneYamlScriptConfig {
 }
 export type MidsceneYamlScriptEnv = MidsceneYamlScriptWebEnv | MidsceneYamlScriptAndroidEnv | MidsceneYamlScriptIOSEnv;
 export interface MidsceneYamlFlowItemAIAction {
-    ai?: string;
     aiAction?: string;
+    ai?: string;
+    aiAct?: string;
     aiActionProgressTips?: string[];
     cacheable?: boolean;
 }
@@ -127,6 +128,7 @@ export interface MidsceneYamlFlowItemSleep {
 }
 export interface MidsceneYamlFlowItemLogScreenshot {
     logScreenshot?: string;
+    recordToReport?: string;
     content?: string;
 }
 export type MidsceneYamlFlowItem = MidsceneYamlFlowItemAIAction | MidsceneYamlFlowItemAIAssert | MidsceneYamlFlowItemAIQuery | MidsceneYamlFlowItemAIWaitFor | MidsceneYamlFlowItemSleep | MidsceneYamlFlowItemLogScreenshot;