@midscene/core 1.9.8 → 1.10.0

package/dist/lib/yaml/player.js

@@ -175,6 +175,13 @@ class ScriptPlayer {
             else if ('string' == typeof actionPrompt && actionPrompt) promptForAI = actionPrompt;
             (0, utils_namespaceObject.assert)(promptForAI, 'missing prompt for ai (aiAct)');
             await agent.aiAct(promptForAI, actionOptions);
+        } else if ('runGherkinScenario' in flowItem) {
+            const gherkinScenarioTask = flowItem;
+            const { runGherkinScenario } = gherkinScenarioTask;
+            (0, utils_namespaceObject.assert)(runGherkinScenario, 'missing scenario for runGherkinScenario');
+            await agent.runGherkinScenario(runGherkinScenario, {
+                cacheable: false
+            });
         } else if ('aiAssert' in flowItem) {
             const assertTask = flowItem;
             const { aiAssert: prompt, errorMessage: msg, name, ...restOpts } = assertTask;

package/dist/lib/yaml/player.js.map

@@ -1 +1 @@
-{"version":3,"file":"yaml/player.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/yaml/player.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { basename, dirname, join, resolve } from 'node:path';\nimport { assert, ifInBrowser, ifInWorker } from '@midscene/shared/utils';\nimport { type ZodTypeAny, z } from 'zod';\n\n// previous defined yaml flow, as a helper\ninterface MidsceneYamlFlowItemAIInput extends LocateOption {\n  // previous version\n  // aiInput: string; // value to input\n  // locate: TUserPrompt; // where to input\n  aiInput: TUserPrompt | undefined; // where to input\n  value: string | number; // value to input\n}\n\ninterface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {\n  // previous version\n  // aiKeyboardPress: string;\n  // locate?: TUserPrompt; // where to press, optional\n  aiKeyboardPress: TUserPrompt | undefined; // where to press\n  keyName: string; // key to press\n}\n\ninterface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {\n  // previous version\n  // aiScroll: null;\n  // locate?: TUserPrompt; // which area to scroll, optional\n  aiScroll: TUserPrompt | undefined; // which area to scroll\n}\n\ntype RuntimeYamlFlowItem =\n  | MidsceneYamlFlowItem\n  | MidsceneYamlFlowItemAIInput\n  | MidsceneYamlFlowItemAIKeyboardPress\n  | MidsceneYamlFlowItemAIScroll;\n\nimport type { Agent } from '@/agent/agent';\nimport type { TUserPrompt } from '@/common';\nimport type {\n  DeviceAction,\n  FreeFn,\n  LocateOption,\n  MidsceneYamlFlowItem,\n  MidsceneYamlFlowItemAIAction,\n  MidsceneYamlFlowItemAIAssert,\n  MidsceneYamlFlowItemAIWaitFor,\n  MidsceneYamlFlowItemEvaluateJavaScript,\n  MidsceneYamlFlowItemLogScreenshot,\n  MidsceneYamlFlowItemSleep,\n  MidsceneYamlScript,\n  MidsceneYamlScriptEnv,\n  ScriptPlayerStatusValue,\n  ScriptPlayerTaskStatus,\n  ScrollParam,\n} from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { getDebug } from '@midscene/shared/logger';\nimport {\n  buildDetailedLocateParam,\n  buildDetailedLocateParamAndRestParams,\n} from './utils';\n\nconst debug = getDebug('yaml-player');\n\nconst aiTaskHandlerMap = {\n  aiQuery: 'aiQuery',\n  aiNumber: 'aiNumber',\n  aiString: 'aiString',\n  aiBoolean: 'aiBoolean',\n  aiAsk: 'aiAsk',\n  aiLocate: 'aiLocate',\n} as const;\n\ntype AISimpleTaskKey = keyof typeof aiTaskHandlerMap;\n\nconst isStringParamSchema = (schema?: ZodTypeAny): boolean => {\n  if (!schema) {\n    return false;\n  }\n\n  const schemaDef = (schema as any)?._def;\n  if (!schemaDef?.typeName) {\n    return false;\n  }\n\n  switch (schemaDef.typeName) {\n    case z.ZodFirstPartyTypeKind.ZodString:\n    case z.ZodFirstPartyTypeKind.ZodEnum:\n    case z.ZodFirstPartyTypeKind.ZodNativeEnum:\n      return true;\n    case z.ZodFirstPartyTypeKind.ZodLiteral:\n      return typeof schemaDef.value === 'string';\n    case z.ZodFirstPartyTypeKind.ZodOptional:\n    case z.ZodFirstPartyTypeKind.ZodNullable:\n    case z.ZodFirstPartyTypeKind.ZodDefault:\n      return isStringParamSchema(schemaDef.innerType);\n    case z.ZodFirstPartyTypeKind.ZodEffects:\n      return isStringParamSchema(schemaDef.schema);\n    case z.ZodFirstPartyTypeKind.ZodPipeline:\n      return isStringParamSchema(schemaDef.out);\n    case z.ZodFirstPartyTypeKind.ZodUnion: {\n      const options = schemaDef.options as ZodTypeAny[] | undefined;\n      return Array.isArray(options)\n        ? options.every((option) => isStringParamSchema(option))\n        : false;\n    }\n    default:\n      return false;\n  }\n};\n\nconst buildShortcutActionParam = (\n  actionName: string,\n  interfaceAlias: string | undefined,\n  value: string,\n) => {\n  if (actionName === 'Launch' || interfaceAlias === 'launch') {\n    return { uri: value };\n  }\n\n  if (actionName === 'Terminate' || interfaceAlias === 'terminate') {\n    return { uri: value };\n  }\n\n  if (\n    actionName === 'RunAdbShell' ||\n    interfaceAlias === 'runAdbShell' ||\n    actionName === 'RunHdcShell' ||\n    interfaceAlias === 'runHdcShell'\n  ) {\n    return { command: value };\n  }\n\n  return undefined;\n};\nexport class ScriptPlayer<T extends MidsceneYamlScriptEnv> {\n  public currentTaskIndex?: number;\n  public taskStatusList: ScriptPlayerTaskStatus[] = [];\n  public status: ScriptPlayerStatusValue = 'init';\n  public reportFile?: string | null;\n  public result: Record<string, any>;\n  private unnamedResultIndex = 0;\n  public output?: string | null;\n  public unstableLogContent?: string | null;\n  public errorInSetup?: Error;\n  private interfaceAgent: Agent | null = null;\n  public agentStatusTip?: string;\n  public target?: MidsceneYamlScriptEnv;\n  private actionSpace: DeviceAction[] = [];\n  private scriptPath?: string;\n  private failedReportExecutionInCurrentStep = false;\n  constructor(\n    private script: MidsceneYamlScript,\n    private setupAgent: (platform: T) => Promise<{\n      agent: Agent;\n      freeFn: FreeFn[];\n    }>,\n    public onTaskStatusChange?: (taskStatus: ScriptPlayerTaskStatus) => void,\n    scriptPath?: string,\n  ) {\n    this.scriptPath = scriptPath;\n    this.result = {};\n    const resolvedAiActContext =\n      script.agent?.aiActContext ?? script.agent?.aiActionContext;\n\n    if (resolvedAiActContext !== undefined && script.agent) {\n      if (\n        script.agent.aiActContext === undefined &&\n        script.agent.aiActionContext !== undefined\n      ) {\n        console.warn(\n          'agent.aiActionContext is deprecated, please use agent.aiActContext instead. The legacy name is still accepted for backward compatibility.',\n        );\n      }\n\n      script.agent.aiActContext = resolvedAiActContext;\n    }\n\n    this.target =\n      script.target ||\n      script.web ||\n      script.android ||\n      script.ios ||\n      script.computer ||\n      script.config;\n\n    if (ifInBrowser || ifInWorker) {\n      this.output = undefined;\n      debug('output is undefined in browser or worker');\n    } else if (this.target?.output) {\n      this.output = resolve(process.cwd(), this.target.output);\n      debug('setting output by config.output', this.output);\n    } else {\n      const scriptName = this.scriptPath\n        ? basename(this.scriptPath, '.yaml').replace(/\\.(ya?ml)$/i, '')\n        : 'script';\n      this.output = join(\n        getMidsceneRunSubDir('output'),\n        `${scriptName}-${Date.now()}.json`,\n      );\n      debug('setting output by script path', this.output);\n    }\n\n    if (ifInBrowser || ifInWorker) {\n      this.unstableLogContent = undefined;\n    } else if (typeof this.target?.unstableLogContent === 'string') {\n      this.unstableLogContent = resolve(\n        process.cwd(),\n        this.target.unstableLogContent,\n      );\n    } else if (this.target?.unstableLogContent === true) {\n      this.unstableLogContent = join(\n        getMidsceneRunSubDir('output'),\n        'unstableLogContent.json',\n      );\n    }\n\n    this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({\n      ...task,\n      index: taskIndex,\n      status: 'init',\n      totalSteps: task.flow?.length || 0,\n    }));\n  }\n\n  private setResult(key: string | undefined, value: any) {\n    const keyToUse = key || this.unnamedResultIndex++;\n    if (this.result[keyToUse]) {\n      console.warn(`result key ${keyToUse} already exists, will overwrite`);\n    }\n    this.result[keyToUse] = value;\n\n    return this.flushResult();\n  }\n\n  private setPlayerStatus(status: ScriptPlayerStatusValue, error?: Error) {\n    this.status = status;\n    this.errorInSetup = error;\n  }\n\n  private notifyCurrentTaskStatusChange(taskIndex?: number) {\n    const taskIndexToNotify =\n      typeof taskIndex === 'number' ? taskIndex : this.currentTaskIndex;\n\n    if (typeof taskIndexToNotify !== 'number') {\n      return;\n    }\n\n    const taskStatus = this.taskStatusList[taskIndexToNotify];\n    if (this.onTaskStatusChange) {\n      this.onTaskStatusChange(taskStatus);\n    }\n  }\n\n  private async setTaskStatus(\n    index: number,\n    statusValue: ScriptPlayerStatusValue,\n    error?: Error,\n  ) {\n    this.taskStatusList[index].status = statusValue;\n    if (error) {\n      this.taskStatusList[index].error = error;\n    }\n\n    this.notifyCurrentTaskStatusChange(index);\n  }\n\n  private setTaskIndex(taskIndex: number) {\n    this.currentTaskIndex = taskIndex;\n  }\n\n  private flushResult() {\n    if (this.output) {\n      const output = resolve(process.cwd(), this.output);\n      const outputDir = dirname(output);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(output, JSON.stringify(this.result || {}, undefined, 2));\n    }\n  }\n\n  private flushUnstableLogContent() {\n    if (this.unstableLogContent) {\n      const content = this.interfaceAgent?._unstableLogContent();\n      const filePath = resolve(process.cwd(), this.unstableLogContent);\n      const outputDir = dirname(filePath);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(filePath, JSON.stringify(content, null, 2));\n    }\n  }\n\n  async playTask(taskStatus: ScriptPlayerTaskStatus, agent: Agent) {\n    const { flow } = taskStatus;\n    assert(flow, 'missing flow in task');\n\n    for (const flowItemIndex in flow) {\n      const currentStep = Number.parseInt(flowItemIndex, 10);\n      taskStatus.currentStep = currentStep;\n      const flowItem = flow[flowItemIndex] as RuntimeYamlFlowItem;\n      const flowItemRecord = flowItem as Record<string, unknown>;\n      const executionCountBeforeStep = agent.dump?.executions?.length ?? 0;\n      this.failedReportExecutionInCurrentStep = false;\n\n      try {\n        await this.playFlowItem(agent, flowItem, flowItemRecord, flowItemIndex);\n      } catch (error) {\n        this.failedReportExecutionInCurrentStep =\n          this.hasFailedReportExecutionSince(agent, executionCountBeforeStep);\n        throw error;\n      }\n    }\n    this.reportFile = agent.reportFile;\n    await this.flushUnstableLogContent();\n  }\n\n  private hasFailedReportExecutionSince(\n    agent: Agent,\n    executionCountBefore: number,\n  ) {\n    return (agent.dump?.executions ?? [])\n      .slice(executionCountBefore)\n      .some((execution) =>\n        execution.tasks.some(\n          (task) =>\n            task.status === 'failed' ||\n            Boolean(task.error || task.errorMessage),\n        ),\n      );\n  }\n\n  private async playFlowItem(\n    agent: Agent,\n    flowItem: RuntimeYamlFlowItem,\n    flowItemRecord: Record<string, unknown>,\n    flowItemIndex: string,\n  ) {\n    // Skip Finalize action from cache - it's a planning-only marker\n    if ('Finalize' in flowItemRecord) {\n      return;\n    }\n\n    debug(\n      `playing step ${flowItemIndex}, flowItem=${JSON.stringify(flowItem)}`,\n    );\n    const simpleAIKey = (\n      Object.keys(aiTaskHandlerMap) as AISimpleTaskKey[]\n    ).find((key) => Object.prototype.hasOwnProperty.call(flowItemRecord, key));\n    if (\n      'aiAct' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n      'aiAction' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n      'ai' in (flowItem as MidsceneYamlFlowItemAIAction)\n    ) {\n      const actionTask = flowItem as MidsceneYamlFlowItemAIAction;\n      const { aiAct, aiAction, ai, instruction, ...actionOptions } =\n        actionTask as any;\n      const actionPrompt = aiAct ?? aiAction ?? ai;\n      let promptForAI: TUserPrompt | undefined;\n\n      if (typeof instruction === 'string' && instruction) {\n        promptForAI = instruction;\n      } else if (\n        instruction &&\n        typeof instruction === 'object' &&\n        typeof (instruction as { prompt?: unknown }).prompt === 'string' &&\n        (instruction as { prompt?: string }).prompt\n      ) {\n        promptForAI = instruction as TUserPrompt;\n      } else if (\n        actionPrompt &&\n        typeof actionPrompt === 'object' &&\n        typeof (actionPrompt as { prompt?: unknown }).prompt === 'string' &&\n        (actionPrompt as { prompt?: string }).prompt\n      ) {\n        promptForAI = actionPrompt as TUserPrompt;\n      } else if (typeof actionPrompt === 'string' && actionPrompt) {\n        promptForAI = actionPrompt;\n      }\n\n      assert(promptForAI, 'missing prompt for ai (aiAct)');\n      await agent.aiAct(promptForAI, actionOptions);\n    } else if ('aiAssert' in (flowItem as MidsceneYamlFlowItemAIAssert)) {\n      const assertTask = flowItem as MidsceneYamlFlowItemAIAssert;\n      const {\n        aiAssert: prompt,\n        errorMessage: msg,\n        name,\n        ...restOpts\n      } = assertTask;\n      assert(prompt, 'missing prompt for aiAssert');\n      const { pass, thought, message } =\n        (await agent.aiAssert(prompt, msg, {\n          ...restOpts,\n          keepRawResponse: true,\n        })) || {};\n\n      this.setResult(name, {\n        pass,\n        thought,\n        message,\n      });\n\n      if (!pass) {\n        throw new Error(message);\n      }\n    } else if (simpleAIKey) {\n      const {\n        [simpleAIKey]: prompt,\n        name,\n        ...options\n      } = flowItem as Record<string, any>;\n      assert(prompt, `missing prompt for ${simpleAIKey}`);\n      const agentMethod = (agent as any)[aiTaskHandlerMap[simpleAIKey]];\n      assert(\n        typeof agentMethod === 'function',\n        `missing agent method for ${simpleAIKey}`,\n      );\n      const aiResult = await agentMethod.call(agent, prompt, options);\n      this.setResult(name, aiResult);\n    } else if ('aiWaitFor' in (flowItem as MidsceneYamlFlowItemAIWaitFor)) {\n      const waitForTask = flowItem as MidsceneYamlFlowItemAIWaitFor;\n      const { aiWaitFor, timeout, ...restWaitForOpts } = waitForTask;\n      const prompt = aiWaitFor;\n      assert(prompt, 'missing prompt for aiWaitFor');\n      const waitForOptions = {\n        ...restWaitForOpts,\n        ...(timeout !== undefined ? { timeout, timeoutMs: timeout } : {}),\n      };\n      await agent.aiWaitFor(prompt, waitForOptions);\n    } else if ('sleep' in flowItem) {\n      const sleepTask = flowItem as unknown as MidsceneYamlFlowItemSleep;\n      const ms = sleepTask.sleep;\n      let msNumber = ms;\n      if (typeof ms === 'string') {\n        msNumber = Number.parseInt(ms, 10);\n      }\n      assert(\n        msNumber && msNumber > 0,\n        `ms for sleep must be greater than 0, but got ${ms}`,\n      );\n      await new Promise((resolve) => setTimeout(resolve, msNumber));\n    } else if ('javascript' in flowItem) {\n      const evaluateJavaScriptTask =\n        flowItem as unknown as MidsceneYamlFlowItemEvaluateJavaScript;\n\n      const result = await agent.evaluateJavaScript(\n        evaluateJavaScriptTask.javascript,\n      );\n      this.setResult(evaluateJavaScriptTask.name, result);\n    } else if (\n      'logScreenshot' in (flowItem as MidsceneYamlFlowItemLogScreenshot) ||\n      'recordToReport' in (flowItem as MidsceneYamlFlowItemLogScreenshot)\n    ) {\n      const recordTask = flowItem as MidsceneYamlFlowItemLogScreenshot;\n      const title =\n        recordTask.recordToReport ?? recordTask.logScreenshot ?? 'untitled';\n      const content = recordTask.content || '';\n      await agent.recordToReport(title, { content });\n    } else if ('aiInput' in flowItem) {\n      // may be input empty string ''\n      const {\n        aiInput,\n        value: rawValue,\n        ...inputTask\n      } = flowItem as unknown as MidsceneYamlFlowItemAIInput;\n\n      // Compatibility with previous version:\n      // Old format: { aiInput: string (value), locate: TUserPrompt }\n      // New format - 1: { aiInput: TUserPrompt, value: string | number }\n      // New format - 2: { aiInput: undefined, locate: TUserPrompt, value: string | number }\n      let locatePrompt: TUserPrompt | undefined;\n      let value: string | number | undefined;\n      if ((inputTask as any).locate) {\n        // Old format - aiInput is the value, locate is the prompt\n        // Keep backward compatibility: empty string is treated as no value\n        value = (aiInput as string | number) || rawValue;\n        locatePrompt = (inputTask as any).locate;\n      } else {\n        // New format - aiInput is the prompt, value is the value\n        locatePrompt = aiInput || '';\n        value = rawValue;\n      }\n\n      // Convert value to string for Input action\n      await agent.callActionInActionSpace('Input', {\n        ...inputTask,\n        ...(value !== undefined ? { value: String(value) } : {}),\n        ...(locatePrompt\n          ? { locate: buildDetailedLocateParam(locatePrompt, inputTask) }\n          : {}),\n      });\n    } else if ('aiKeyboardPress' in flowItem) {\n      const { aiKeyboardPress, ...keyboardPressTask } =\n        flowItem as unknown as MidsceneYamlFlowItemAIKeyboardPress;\n\n      // Compatibility with previous version:\n      // Old format: { aiKeyboardPress: string (key), locate?: TUserPrompt }\n      // New format - 1: { aiKeyboardPress: TUserPrompt, keyName: string }\n      // New format - 2: { aiKeyboardPress: , locate?: TUserPrompt, keyName: string }\n      let locatePrompt: TUserPrompt | undefined;\n      let keyName: string | undefined;\n      if ((keyboardPressTask as any).locate) {\n        // Old format - aiKeyboardPress is the key, locate is the prompt\n        keyName = aiKeyboardPress as string;\n        locatePrompt = (keyboardPressTask as any).locate;\n      } else if (keyboardPressTask.keyName) {\n        // New format - aiKeyboardPress is the prompt, key is the key\n        keyName = keyboardPressTask.keyName;\n        locatePrompt = aiKeyboardPress;\n      } else {\n        keyName = aiKeyboardPress as string;\n      }\n\n      await agent.callActionInActionSpace('KeyboardPress', {\n        ...keyboardPressTask,\n        ...(keyName ? { keyName } : {}),\n        ...(locatePrompt\n          ? {\n              locate: buildDetailedLocateParam(locatePrompt, keyboardPressTask),\n            }\n          : {}),\n      });\n    } else if ('aiScroll' in flowItem) {\n      const { aiScroll, ...scrollTask } =\n        flowItem as unknown as MidsceneYamlFlowItemAIScroll;\n\n      // Compatibility with previous version:\n      // Old format: { aiScroll: null, locate?: TUserPrompt, direction, scrollType, distance? }\n      // New format - 1: { aiScroll: TUserPrompt, direction, scrollType, distance? }\n      // New format - 2: { aiScroll: undefined, locate: TUserPrompt, direction, scrollType, distance? }\n      const { locate, ...scrollOptions } = scrollTask as any;\n      const locatePrompt: TUserPrompt | undefined =\n        locate ?? aiScroll ?? undefined;\n\n      await agent.aiScroll(locatePrompt, scrollOptions);\n    } else if ('aiTap' in flowItem) {\n      const { aiTap, prompt, locate, ...tapOptions } = flowItem as any;\n\n      let locatePrompt: TUserPrompt;\n      let opts = tapOptions;\n      // Support both formats:\n      // 1. { aiTap: null, locate: { prompt, images, ... } }  (locate as sibling key)\n      // 2. { aiTap: { locate: { prompt, images, ... } } }    (locate nested in aiTap)\n      const locateObj =\n        locate ??\n        (typeof aiTap === 'object' && aiTap !== null\n          ? aiTap.locate\n          : undefined);\n\n      if (typeof aiTap === 'string' && aiTap) {\n        // User YAML: aiTap: 'search input box'\n        locatePrompt = aiTap;\n      } else if (typeof locateObj === 'object' && locateObj?.prompt) {\n        // buildYamlFlowFromPlans: { aiTap: '', locate: { prompt, deepLocate, cacheable } }\n        const { prompt: lp, ...locateOpts } = locateObj;\n        locatePrompt = lp;\n        opts = { ...locateOpts, ...tapOptions };\n      } else {\n        // User YAML: aiTap: { prompt: '...' } or aiTap: null + prompt: '...'\n        locatePrompt = aiTap?.prompt || prompt || locateObj;\n      }\n\n      assert(locatePrompt, 'missing prompt for aiTap');\n      await agent.aiTap(locatePrompt, opts);\n    } else {\n      // generic action, find the action in actionSpace\n\n      /* for aiRightClick, the parameters are a flattened data for the 'locate', these are all valid data\n\n        - aiRightClick: 'search input box'\n        - aiRightClick: 'search input box'\n          deepLocate: true\n          cacheable: false\n        - aiRightClick:\n          prompt: 'search input box'\n        - aiRightClick:\n          prompt: 'search input box'\n          deepLocate: true\n          cacheable: false\n        */\n\n      const actionSpace = this.actionSpace;\n      let locatePromptShortcut: string | undefined;\n      let actionParamForMatchedAction: unknown;\n      const matchedAction = actionSpace.find((action) => {\n        const actionInterfaceAlias = action.interfaceAlias;\n        if (\n          actionInterfaceAlias &&\n          Object.prototype.hasOwnProperty.call(flowItem, actionInterfaceAlias)\n        ) {\n          actionParamForMatchedAction =\n            flowItem[actionInterfaceAlias as keyof typeof flowItem];\n          if (typeof actionParamForMatchedAction === 'string') {\n            locatePromptShortcut = actionParamForMatchedAction;\n          }\n          return true;\n        }\n\n        const keyOfActionInActionSpace = action.name;\n        if (\n          Object.prototype.hasOwnProperty.call(\n            flowItem,\n            keyOfActionInActionSpace,\n          )\n        ) {\n          actionParamForMatchedAction =\n            flowItem[keyOfActionInActionSpace as keyof typeof flowItem];\n          if (typeof actionParamForMatchedAction === 'string') {\n            locatePromptShortcut = actionParamForMatchedAction;\n          }\n          return true;\n        }\n\n        return false;\n      });\n\n      assert(\n        matchedAction,\n        `unknown flowItem in yaml: ${JSON.stringify(flowItem)}`,\n      );\n\n      const schemaIsStringParam = isStringParamSchema(\n        matchedAction.paramSchema,\n      );\n      let stringParamToCall: string | undefined;\n      const resultName = (flowItem as any).name;\n      const timeout = (flowItem as any).timeout;\n      const hasRunAdbShellAlias = Object.prototype.hasOwnProperty.call(\n        flowItem,\n        'runAdbShell',\n      );\n\n      if (\n        hasRunAdbShellAlias &&\n        typeof actionParamForMatchedAction === 'string' &&\n        typeof timeout === 'number' &&\n        typeof (agent as any).runAdbShell === 'function'\n      ) {\n        const result = await (agent as any).runAdbShell(\n          actionParamForMatchedAction,\n          { timeout },\n        );\n        if (result !== undefined) {\n          this.setResult(resultName, result);\n        }\n        return;\n      }\n\n      const specialActionParamToCall =\n        typeof actionParamForMatchedAction === 'string'\n          ? buildShortcutActionParam(\n              matchedAction.name,\n              matchedAction.interfaceAlias,\n              actionParamForMatchedAction,\n            )\n          : undefined;\n      if (specialActionParamToCall) {\n        debug(\n          `matchedAction: ${matchedAction.name}`,\n          `flowParams: ${JSON.stringify(specialActionParamToCall)}`,\n        );\n        const result = await agent.callActionInActionSpace(\n          matchedAction.name,\n          specialActionParamToCall,\n        );\n\n        if (result !== undefined) {\n          this.setResult(resultName, result);\n        }\n      } else if (\n        typeof actionParamForMatchedAction === 'string' &&\n        schemaIsStringParam\n      ) {\n        if (matchedAction.paramSchema) {\n          const parseResult = matchedAction.paramSchema.safeParse(\n            actionParamForMatchedAction,\n          );\n          if (parseResult.success && typeof parseResult.data === 'string') {\n            stringParamToCall = parseResult.data;\n          } else if (!parseResult.success) {\n            debug(\n              `parse failed for action ${matchedAction.name} with string param`,\n              parseResult.error,\n            );\n            stringParamToCall = actionParamForMatchedAction;\n          }\n        } else {\n          stringParamToCall = actionParamForMatchedAction;\n        }\n\n        if (stringParamToCall !== undefined) {\n          debug(\n            `matchedAction: ${matchedAction.name}`,\n            `flowParams: ${JSON.stringify(stringParamToCall)}`,\n          );\n          const result = await agent.callActionInActionSpace(\n            matchedAction.name,\n            stringParamToCall,\n          );\n\n          // Store result if there's a name property in flowItem\n          const resultName = (flowItem as any).name;\n          if (result !== undefined) {\n            this.setResult(resultName, result);\n          }\n        }\n      } else {\n        // Determine the source for parameter extraction:\n        // - If we have a locatePromptShortcut, use the flowItem (for actions like aiTap with prompt)\n        // - Otherwise, use actionParamForMatchedAction (for actions like runWdaRequest with structured params)\n        const sourceForParams =\n          locatePromptShortcut &&\n          typeof actionParamForMatchedAction === 'string'\n            ? { ...flowItem, prompt: locatePromptShortcut }\n            : typeof actionParamForMatchedAction === 'object' &&\n                actionParamForMatchedAction !== null\n              ? actionParamForMatchedAction\n              : flowItem;\n\n        const { locateParam, restParams } =\n          buildDetailedLocateParamAndRestParams(\n            locatePromptShortcut || '',\n            sourceForParams as LocateOption,\n            [\n              matchedAction.name,\n              matchedAction.interfaceAlias || '_never_mind_',\n            ],\n          );\n\n        const flowParams = {\n          ...restParams,\n          locate: locateParam,\n        };\n\n        debug(\n          `matchedAction: ${matchedAction.name}`,\n          `flowParams: ${JSON.stringify(flowParams, null, 2)}`,\n        );\n        const result = await agent.callActionInActionSpace(\n          matchedAction.name,\n          flowParams,\n        );\n\n        // Store result if there's a name property in flowItem\n        const resultName = (flowItem as any).name;\n        if (result !== undefined) {\n          this.setResult(resultName, result);\n        }\n      }\n    }\n  }\n\n  async run() {\n    const { target, web, android, ios, harmony, computer, tasks } = this.script;\n    const webEnv = web || target;\n    const androidEnv = android;\n    const iosEnv = ios;\n    const harmonyEnv = harmony;\n    const computerEnv = computer;\n    const platform =\n      webEnv || androidEnv || iosEnv || harmonyEnv || computerEnv;\n\n    this.setPlayerStatus('running');\n\n    let agent: Agent | null = null;\n    let freeFn: FreeFn[] = [];\n    try {\n      const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(\n        platform as T,\n      );\n      this.actionSpace = await newAgent.getActionSpace();\n      agent = newAgent;\n      const originalOnTaskStartTip = agent.onTaskStartTip;\n      agent.onTaskStartTip = (tip) => {\n        if (this.status === 'running') {\n          this.agentStatusTip = tip;\n        }\n        originalOnTaskStartTip?.(tip);\n      };\n      freeFn = [\n        ...(newFreeFn || []),\n        {\n          name: 'restore-agent-onTaskStartTip',\n          fn: () => {\n            if (agent) {\n              agent.onTaskStartTip = originalOnTaskStartTip;\n            }\n          },\n        },\n      ];\n    } catch (e) {\n      this.setPlayerStatus('error', e as Error);\n      return;\n    }\n    this.interfaceAgent = agent;\n\n    let taskIndex = 0;\n    this.setPlayerStatus('running');\n    let errorFlag = false;\n    while (taskIndex < tasks.length) {\n      const taskStatus = this.taskStatusList[taskIndex];\n      this.setTaskStatus(taskIndex, 'running' as any);\n      this.setTaskIndex(taskIndex);\n      this.failedReportExecutionInCurrentStep = false;\n\n      try {\n        await this.playTask(taskStatus, this.interfaceAgent);\n        this.setTaskStatus(taskIndex, 'done' as any);\n      } catch (e) {\n        this.setTaskStatus(taskIndex, 'error' as any, e as Error);\n        const recordErrorToReport = (agent as any).recordErrorToReport;\n        if (\n          !this.failedReportExecutionInCurrentStep &&\n          typeof recordErrorToReport === 'function'\n        ) {\n          try {\n            await recordErrorToReport.call(\n              agent,\n              `YAML task failed - ${taskStatus.name}`,\n              {\n                error: e as Error,\n                content: `Step ${taskStatus.currentStep ?? 0} failed while running YAML task \"${taskStatus.name}\".`,\n              },\n            );\n          } catch (reportError) {\n            debug('failed to record yaml error to report', reportError);\n          }\n        }\n\n        if (taskStatus.continueOnError) {\n          // nothing more to do\n        } else {\n          this.reportFile = agent.reportFile;\n          errorFlag = true;\n          break;\n        }\n      }\n      this.reportFile = agent?.reportFile;\n      taskIndex++;\n    }\n\n    if (errorFlag) {\n      this.setPlayerStatus('error');\n    } else {\n      this.setPlayerStatus('done');\n    }\n    this.agentStatusTip = '';\n\n    // free the resources\n    for (const fn of freeFn) {\n      try {\n        // console.log('freeing', fn.name);\n        await fn.fn();\n        // console.log('freed', fn.name);\n      } catch (e) {\n        // console.error('error freeing', fn.name, e);\n      }\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","aiTaskHandlerMap","isStringParamSchema","schema","schemaDef","z","options","Array","option","buildShortcutActionParam","actionName","interfaceAlias","value","ScriptPlayer","keyToUse","console","status","error","taskIndex","taskIndexToNotify","taskStatus","index","statusValue","output","resolve","process","outputDir","dirname","existsSync","mkdirSync","writeFileSync","JSON","undefined","content","filePath","agent","flow","assert","flowItemIndex","currentStep","Number","flowItem","flowItemRecord","executionCountBeforeStep","executionCountBefore","execution","task","Boolean","simpleAIKey","actionTask","aiAct","aiAction","ai","instruction","actionOptions","actionPrompt","promptForAI","assertTask","prompt","msg","name","restOpts","pass","thought","message","Error","agentMethod","aiResult","waitForTask","aiWaitFor","timeout","restWaitForOpts","waitForOptions","sleepTask","ms","msNumber","Promise","setTimeout","evaluateJavaScriptTask","result","recordTask","title","aiInput","rawValue","inputTask","locatePrompt","String","buildDetailedLocateParam","aiKeyboardPress","keyboardPressTask","keyName","aiScroll","scrollTask","locate","scrollOptions","aiTap","tapOptions","opts","locateObj","lp","locateOpts","actionSpace","locatePromptShortcut","actionParamForMatchedAction","matchedAction","action","actionInterfaceAlias","keyOfActionInActionSpace","schemaIsStringParam","stringParamToCall","resultName","hasRunAdbShellAlias","specialActionParamToCall","parseResult","sourceForParams","locateParam","restParams","buildDetailedLocateParamAndRestParams","flowParams","target","web","android","ios","harmony","computer","tasks","webEnv","androidEnv","iosEnv","harmonyEnv","computerEnv","platform","freeFn","newAgent","newFreeFn","originalOnTaskStartTip","tip","e","errorFlag","recordErrorToReport","reportError","fn","script","setupAgent","onTaskStatusChange","scriptPath","resolvedAiActContext","ifInBrowser","ifInWorker","scriptName","basename","join","getMidsceneRunSubDir","Date"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACuDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEvB,MAAMC,mBAAmB;IACvB,SAAS;IACT,UAAU;IACV,UAAU;IACV,WAAW;IACX,OAAO;IACP,UAAU;AACZ;AAIA,MAAMC,sBAAsB,CAACC;IAC3B,IAAI,CAACA,QACH,OAAO;IAGT,MAAMC,YAAaD,QAAgB;IACnC,IAAI,CAACC,WAAW,UACd,OAAO;IAGT,OAAQA,UAAU,QAAQ;QACxB,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,SAAiC;QACtC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,OAA+B;QACpC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,aAAqC;YACxC,OAAO;QACT,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,UAAkC;YACrC,OAAO,AAA2B,YAA3B,OAAOD,UAAU,KAAK;QAC/B,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,WAAmC;QACxC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,WAAmC;QACxC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,UAAkC;YACrC,OAAOH,oBAAoBE,UAAU,SAAS;QAChD,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,UAAkC;YACrC,OAAOH,oBAAoBE,UAAU,MAAM;QAC7C,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,WAAmC;YACtC,OAAOH,oBAAoBE,UAAU,GAAG;QAC1C,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,QAAgC;YAAE;gBACrC,MAAMC,UAAUF,UAAU,OAAO;gBACjC,OAAOG,MAAM,OAAO,CAACD,WACjBA,QAAQ,KAAK,CAAC,CAACE,SAAWN,oBAAoBM,WAC9C;YACN;QACA;YACE,OAAO;IACX;AACF;AAEA,MAAMC,2BAA2B,CAC/BC,YACAC,gBACAC;IAEA,IAAIF,AAAe,aAAfA,cAA2BC,AAAmB,aAAnBA,gBAC7B,OAAO;QAAE,KAAKC;IAAM;IAGtB,IAAIF,AAAe,gBAAfA,cAA8BC,AAAmB,gBAAnBA,gBAChC,OAAO;QAAE,KAAKC;IAAM;IAGtB,IACEF,AAAe,kBAAfA,cACAC,AAAmB,kBAAnBA,kBACAD,AAAe,kBAAfA,cACAC,AAAmB,kBAAnBA,gBAEA,OAAO;QAAE,SAASC;IAAM;AAI5B;AACO,MAAMC;IA0FH,UAAUnB,GAAuB,EAAEkB,KAAU,EAAE;QACrD,MAAME,WAAWpB,OAAO,IAAI,CAAC,kBAAkB;QAC/C,IAAI,IAAI,CAAC,MAAM,CAACoB,SAAS,EACvBC,QAAQ,IAAI,CAAC,CAAC,WAAW,EAAED,SAAS,+BAA+B,CAAC;QAEtE,IAAI,CAAC,MAAM,CAACA,SAAS,GAAGF;QAExB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEQ,gBAAgBI,MAA+B,EAAEC,KAAa,EAAE;QACtE,IAAI,CAAC,MAAM,GAAGD;QACd,IAAI,CAAC,YAAY,GAAGC;IACtB;IAEQ,8BAA8BC,SAAkB,EAAE;QACxD,MAAMC,oBACJ,AAAqB,YAArB,OAAOD,YAAyBA,YAAY,IAAI,CAAC,gBAAgB;QAEnE,IAAI,AAA6B,YAA7B,OAAOC,mBACT;QAGF,MAAMC,aAAa,IAAI,CAAC,cAAc,CAACD,kBAAkB;QACzD,IAAI,IAAI,CAAC,kBAAkB,EACzB,IAAI,CAAC,kBAAkB,CAACC;IAE5B;IAEA,MAAc,cACZC,KAAa,EACbC,WAAoC,EACpCL,KAAa,EACb;QACA,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,MAAM,GAAGC;QACpC,IAAIL,OACF,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,KAAK,GAAGJ;QAGrC,IAAI,CAAC,6BAA6B,CAACI;IACrC;IAEQ,aAAaH,SAAiB,EAAE;QACtC,IAAI,CAAC,gBAAgB,GAAGA;IAC1B;IAEQ,cAAc;QACpB,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,MAAMK,SAASC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM;YACjD,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQJ;YAC1B,IAAI,CAACK,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcP,QAAQQ,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,GAAGC,QAAW;QACrE;IACF;IAEQ,0BAA0B;QAChC,IAAI,IAAI,CAAC,kBAAkB,EAAE;YAC3B,MAAMC,UAAU,IAAI,CAAC,cAAc,EAAE;YACrC,MAAMC,WAAWV,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,kBAAkB;YAC/D,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQO;YAC1B,IAAI,CAACN,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcI,UAAUH,KAAK,SAAS,CAACE,SAAS,MAAM;QACxD;IACF;IAEA,MAAM,SAASb,UAAkC,EAAEe,KAAY,EAAE;QAC/D,MAAM,EAAEC,IAAI,EAAE,GAAGhB;QACjBiB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,MAAM;QAEb,IAAK,MAAME,iBAAiBF,KAAM;YAChC,MAAMG,cAAcC,OAAO,QAAQ,CAACF,eAAe;YACnDlB,WAAW,WAAW,GAAGmB;YACzB,MAAME,WAAWL,IAAI,CAACE,cAAc;YACpC,MAAMI,iBAAiBD;YACvB,MAAME,2BAA2BR,MAAM,IAAI,EAAE,YAAY,UAAU;YACnE,IAAI,CAAC,kCAAkC,GAAG;YAE1C,IAAI;gBACF,MAAM,IAAI,CAAC,YAAY,CAACA,OAAOM,UAAUC,gBAAgBJ;YAC3D,EAAE,OAAOrB,OAAO;gBACd,IAAI,CAAC,kCAAkC,GACrC,IAAI,CAAC,6BAA6B,CAACkB,OAAOQ;gBAC5C,MAAM1B;YACR;QACF;QACA,IAAI,CAAC,UAAU,GAAGkB,MAAM,UAAU;QAClC,MAAM,IAAI,CAAC,uBAAuB;IACpC;IAEQ,8BACNA,KAAY,EACZS,oBAA4B,EAC5B;QACA,OAAQT,AAAAA,CAAAA,MAAM,IAAI,EAAE,cAAc,EAAC,EAChC,KAAK,CAACS,sBACN,IAAI,CAAC,CAACC,YACLA,UAAU,KAAK,CAAC,IAAI,CAClB,CAACC,OACCA,AAAgB,aAAhBA,KAAK,MAAM,IACXC,QAAQD,KAAK,KAAK,IAAIA,KAAK,YAAY;IAGjD;IAEA,MAAc,aACZX,KAAY,EACZM,QAA6B,EAC7BC,cAAuC,EACvCJ,aAAqB,EACrB;QAEA,IAAI,cAAcI,gBAChB;QAGF3C,MACE,CAAC,aAAa,EAAEuC,cAAc,WAAW,EAAEP,KAAK,SAAS,CAACU,WAAW;QAEvE,MAAMO,cACJrD,OAAO,IAAI,CAACM,kBACZ,IAAI,CAAC,CAACP,MAAQC,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC+C,gBAAgBhD;QACrE,IACE,WAAY+C,YACZ,cAAeA,YACf,QAASA,UACT;YACA,MAAMQ,aAAaR;YACnB,MAAM,EAAES,KAAK,EAAEC,QAAQ,EAAEC,EAAE,EAAEC,WAAW,EAAE,GAAGC,eAAe,GAC1DL;YACF,MAAMM,eAAeL,SAASC,YAAYC;YAC1C,IAAII;YAEJ,IAAI,AAAuB,YAAvB,OAAOH,eAA4BA,aACrCG,cAAcH;iBACT,IACLA,eACA,AAAuB,YAAvB,OAAOA,eACP,AAAwD,YAAxD,OAAQA,YAAqC,MAAM,IAClDA,YAAoC,MAAM,EAE3CG,cAAcH;iBACT,IACLE,gBACA,AAAwB,YAAxB,OAAOA,gBACP,AAAyD,YAAzD,OAAQA,aAAsC,MAAM,IACnDA,aAAqC,MAAM,EAE5CC,cAAcD;iBACT,IAAI,AAAwB,YAAxB,OAAOA,gBAA6BA,cAC7CC,cAAcD;YAGhBlB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOmB,aAAa;YACpB,MAAMrB,MAAM,KAAK,CAACqB,aAAaF;QACjC,OAAO,IAAI,cAAeb,UAA2C;YACnE,MAAMgB,aAAahB;YACnB,MAAM,EACJ,UAAUiB,MAAM,EAChB,cAAcC,GAAG,EACjBC,IAAI,EACJ,GAAGC,UACJ,GAAGJ;YACJpB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqB,QAAQ;YACf,MAAM,EAAEI,IAAI,EAAEC,OAAO,EAAEC,OAAO,EAAE,GAC7B,MAAM7B,MAAM,QAAQ,CAACuB,QAAQC,KAAK;gBACjC,GAAGE,QAAQ;gBACX,iBAAiB;YACnB,MAAO,CAAC;YAEV,IAAI,CAAC,SAAS,CAACD,MAAM;gBACnBE;gBACAC;gBACAC;YACF;YAEA,IAAI,CAACF,MACH,MAAM,IAAIG,MAAMD;QAEpB,OAAO,IAAIhB,aAAa;YACtB,MAAM,EACJ,CAACA,YAAY,EAAEU,MAAM,EACrBE,IAAI,EACJ,GAAGtD,SACJ,GAAGmC;YACJJ,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqB,QAAQ,CAAC,mBAAmB,EAAEV,aAAa;YAClD,MAAMkB,cAAe/B,KAAa,CAAClC,gBAAgB,CAAC+C,YAAY,CAAC;YACjEX,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAuB,cAAvB,OAAO6B,aACP,CAAC,yBAAyB,EAAElB,aAAa;YAE3C,MAAMmB,WAAW,MAAMD,YAAY,IAAI,CAAC/B,OAAOuB,QAAQpD;YACvD,IAAI,CAAC,SAAS,CAACsD,MAAMO;QACvB,OAAO,IAAI,eAAgB1B,UAA4C;YACrE,MAAM2B,cAAc3B;YACpB,MAAM,EAAE4B,SAAS,EAAEC,OAAO,EAAE,GAAGC,iBAAiB,GAAGH;YACnD,MAAMV,SAASW;YACfhC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqB,QAAQ;YACf,MAAMc,iBAAiB;gBACrB,GAAGD,eAAe;gBAClB,GAAID,AAAYtC,WAAZsC,UAAwB;oBAAEA;oBAAS,WAAWA;gBAAQ,IAAI,CAAC,CAAC;YAClE;YACA,MAAMnC,MAAM,SAAS,CAACuB,QAAQc;QAChC,OAAO,IAAI,WAAW/B,UAAU;YAC9B,MAAMgC,YAAYhC;YAClB,MAAMiC,KAAKD,UAAU,KAAK;YAC1B,IAAIE,WAAWD;YACf,IAAI,AAAc,YAAd,OAAOA,IACTC,WAAWnC,OAAO,QAAQ,CAACkC,IAAI;YAEjCrC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEsC,YAAYA,WAAW,GACvB,CAAC,6CAA6C,EAAED,IAAI;YAEtD,MAAM,IAAIE,QAAQ,CAACpD,UAAYqD,WAAWrD,SAASmD;QACrD,OAAO,IAAI,gBAAgBlC,UAAU;YACnC,MAAMqC,yBACJrC;YAEF,MAAMsC,SAAS,MAAM5C,MAAM,kBAAkB,CAC3C2C,uBAAuB,UAAU;YAEnC,IAAI,CAAC,SAAS,CAACA,uBAAuB,IAAI,EAAEC;QAC9C,OAAO,IACL,mBAAoBtC,YACpB,oBAAqBA,UACrB;YACA,MAAMuC,aAAavC;YACnB,MAAMwC,QACJD,WAAW,cAAc,IAAIA,WAAW,aAAa,IAAI;YAC3D,MAAM/C,UAAU+C,WAAW,OAAO,IAAI;YACtC,MAAM7C,MAAM,cAAc,CAAC8C,OAAO;gBAAEhD;YAAQ;QAC9C,OAAO,IAAI,aAAaQ,UAAU;YAEhC,MAAM,EACJyC,OAAO,EACP,OAAOC,QAAQ,EACf,GAAGC,WACJ,GAAG3C;YAMJ,IAAI4C;YACJ,IAAIzE;YACJ,IAAKwE,UAAkB,MAAM,EAAE;gBAG7BxE,QAASsE,WAA+BC;gBACxCE,eAAgBD,UAAkB,MAAM;YAC1C,OAAO;gBAELC,eAAeH,WAAW;gBAC1BtE,QAAQuE;YACV;YAGA,MAAMhD,MAAM,uBAAuB,CAAC,SAAS;gBAC3C,GAAGiD,SAAS;gBACZ,GAAIxE,AAAUoB,WAAVpB,QAAsB;oBAAE,OAAO0E,OAAO1E;gBAAO,IAAI,CAAC,CAAC;gBACvD,GAAIyE,eACA;oBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcD;gBAAW,IAC5D,CAAC,CAAC;YACR;QACF,OAAO,IAAI,qBAAqB3C,UAAU;YACxC,MAAM,EAAE+C,eAAe,EAAE,GAAGC,mBAAmB,GAC7ChD;YAMF,IAAI4C;YACJ,IAAIK;YACJ,IAAKD,kBAA0B,MAAM,EAAE;gBAErCC,UAAUF;gBACVH,eAAgBI,kBAA0B,MAAM;YAClD,OAAO,IAAIA,kBAAkB,OAAO,EAAE;gBAEpCC,UAAUD,kBAAkB,OAAO;gBACnCJ,eAAeG;YACjB,OACEE,UAAUF;YAGZ,MAAMrD,MAAM,uBAAuB,CAAC,iBAAiB;gBACnD,GAAGsD,iBAAiB;gBACpB,GAAIC,UAAU;oBAAEA;gBAAQ,IAAI,CAAC,CAAC;gBAC9B,GAAIL,eACA;oBACE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcI;gBACjD,IACA,CAAC,CAAC;YACR;QACF,OAAO,IAAI,cAAchD,UAAU;YACjC,MAAM,EAAEkD,QAAQ,EAAE,GAAGC,YAAY,GAC/BnD;YAMF,MAAM,EAAEoD,MAAM,EAAE,GAAGC,eAAe,GAAGF;YACrC,MAAMP,eACJQ,UAAUF,YAAY3D;YAExB,MAAMG,MAAM,QAAQ,CAACkD,cAAcS;QACrC,OAAO,IAAI,WAAWrD,UAAU;YAC9B,MAAM,EAAEsD,KAAK,EAAErC,MAAM,EAAEmC,MAAM,EAAE,GAAGG,YAAY,GAAGvD;YAEjD,IAAI4C;YACJ,IAAIY,OAAOD;YAIX,MAAME,YACJL,UACC,CAAiB,YAAjB,OAAOE,SAAsBA,AAAU,SAAVA,QAC1BA,MAAM,MAAM,GACZ/D,MAAQ;YAEd,IAAI,AAAiB,YAAjB,OAAO+D,SAAsBA,OAE/BV,eAAeU;iBACV,IAAI,AAAqB,YAArB,OAAOG,aAA0BA,WAAW,QAAQ;gBAE7D,MAAM,EAAE,QAAQC,EAAE,EAAE,GAAGC,YAAY,GAAGF;gBACtCb,eAAec;gBACfF,OAAO;oBAAE,GAAGG,UAAU;oBAAE,GAAGJ,UAAU;gBAAC;YACxC,OAEEX,eAAeU,OAAO,UAAUrC,UAAUwC;YAG5C7D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOgD,cAAc;YACrB,MAAMlD,MAAM,KAAK,CAACkD,cAAcY;QAClC,OAAO;YAiBL,MAAMI,cAAc,IAAI,CAAC,WAAW;YACpC,IAAIC;YACJ,IAAIC;YACJ,MAAMC,gBAAgBH,YAAY,IAAI,CAAC,CAACI;gBACtC,MAAMC,uBAAuBD,OAAO,cAAc;gBAClD,IACEC,wBACA/G,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC8C,UAAUiE,uBAC/C;oBACAH,8BACE9D,QAAQ,CAACiE,qBAA8C;oBACzD,IAAI,AAAuC,YAAvC,OAAOH,6BACTD,uBAAuBC;oBAEzB,OAAO;gBACT;gBAEA,MAAMI,2BAA2BF,OAAO,IAAI;gBAC5C,IACE9G,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAClC8C,UACAkE,2BAEF;oBACAJ,8BACE9D,QAAQ,CAACkE,yBAAkD;oBAC7D,IAAI,AAAuC,YAAvC,OAAOJ,6BACTD,uBAAuBC;oBAEzB,OAAO;gBACT;gBAEA,OAAO;YACT;YAEAlE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEmE,eACA,CAAC,0BAA0B,EAAEzE,KAAK,SAAS,CAACU,WAAW;YAGzD,MAAMmE,sBAAsB1G,oBAC1BsG,cAAc,WAAW;YAE3B,IAAIK;YACJ,MAAMC,aAAcrE,SAAiB,IAAI;YACzC,MAAM6B,UAAW7B,SAAiB,OAAO;YACzC,MAAMsE,sBAAsBpH,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAC9D8C,UACA;YAGF,IACEsE,uBACA,AAAuC,YAAvC,OAAOR,+BACP,AAAmB,YAAnB,OAAOjC,WACP,AAAsC,cAAtC,OAAQnC,MAAc,WAAW,EACjC;gBACA,MAAM4C,SAAS,MAAO5C,MAAc,WAAW,CAC7CoE,6BACA;oBAAEjC;gBAAQ;gBAEZ,IAAIS,AAAW/C,WAAX+C,QACF,IAAI,CAAC,SAAS,CAAC+B,YAAY/B;gBAE7B;YACF;YAEA,MAAMiC,2BACJ,AAAuC,YAAvC,OAAOT,8BACH9F,yBACE+F,cAAc,IAAI,EAClBA,cAAc,cAAc,EAC5BD,+BAEFvE;YACN,IAAIgF,0BAA0B;gBAC5BjH,MACE,CAAC,eAAe,EAAEyG,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAEzE,KAAK,SAAS,CAACiF,2BAA2B;gBAE3D,MAAMjC,SAAS,MAAM5C,MAAM,uBAAuB,CAChDqE,cAAc,IAAI,EAClBQ;gBAGF,IAAIjC,AAAW/C,WAAX+C,QACF,IAAI,CAAC,SAAS,CAAC+B,YAAY/B;YAE/B,OAAO,IACL,AAAuC,YAAvC,OAAOwB,+BACPK,qBACA;gBACA,IAAIJ,cAAc,WAAW,EAAE;oBAC7B,MAAMS,cAAcT,cAAc,WAAW,CAAC,SAAS,CACrDD;oBAEF,IAAIU,YAAY,OAAO,IAAI,AAA4B,YAA5B,OAAOA,YAAY,IAAI,EAChDJ,oBAAoBI,YAAY,IAAI;yBAC/B,IAAI,CAACA,YAAY,OAAO,EAAE;wBAC/BlH,MACE,CAAC,wBAAwB,EAAEyG,cAAc,IAAI,CAAC,kBAAkB,CAAC,EACjES,YAAY,KAAK;wBAEnBJ,oBAAoBN;oBACtB;gBACF,OACEM,oBAAoBN;gBAGtB,IAAIM,AAAsB7E,WAAtB6E,mBAAiC;oBACnC9G,MACE,CAAC,eAAe,EAAEyG,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAEzE,KAAK,SAAS,CAAC8E,oBAAoB;oBAEpD,MAAM9B,SAAS,MAAM5C,MAAM,uBAAuB,CAChDqE,cAAc,IAAI,EAClBK;oBAIF,MAAMC,aAAcrE,SAAiB,IAAI;oBACzC,IAAIsC,AAAW/C,WAAX+C,QACF,IAAI,CAAC,SAAS,CAAC+B,YAAY/B;gBAE/B;YACF,OAAO;gBAIL,MAAMmC,kBACJZ,wBACA,AAAuC,YAAvC,OAAOC,8BACH;oBAAE,GAAG9D,QAAQ;oBAAE,QAAQ6D;gBAAqB,IAC5C,AAAuC,YAAvC,OAAOC,+BACLA,AAAgC,SAAhCA,8BACAA,8BACA9D;gBAER,MAAM,EAAE0E,WAAW,EAAEC,UAAU,EAAE,GAC/BC,AAAAA,IAAAA,kCAAAA,qCAAAA,AAAAA,EACEf,wBAAwB,IACxBY,iBACA;oBACEV,cAAc,IAAI;oBAClBA,cAAc,cAAc,IAAI;iBACjC;gBAGL,MAAMc,aAAa;oBACjB,GAAGF,UAAU;oBACb,QAAQD;gBACV;gBAEApH,MACE,CAAC,eAAe,EAAEyG,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAEzE,KAAK,SAAS,CAACuF,YAAY,MAAM,IAAI;gBAEtD,MAAMvC,SAAS,MAAM5C,MAAM,uBAAuB,CAChDqE,cAAc,IAAI,EAClBc;gBAIF,MAAMR,aAAcrE,SAAiB,IAAI;gBACzC,IAAIsC,AAAW/C,WAAX+C,QACF,IAAI,CAAC,SAAS,CAAC+B,YAAY/B;YAE/B;QACF;IACF;IAEA,MAAM,MAAM;QACV,MAAM,EAAEwC,MAAM,EAAEC,GAAG,EAAEC,OAAO,EAAEC,GAAG,EAAEC,OAAO,EAAEC,QAAQ,EAAEC,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM;QAC3E,MAAMC,SAASN,OAAOD;QACtB,MAAMQ,aAAaN;QACnB,MAAMO,SAASN;QACf,MAAMO,aAAaN;QACnB,MAAMO,cAAcN;QACpB,MAAMO,WACJL,UAAUC,cAAcC,UAAUC,cAAcC;QAElD,IAAI,CAAC,eAAe,CAAC;QAErB,IAAI/F,QAAsB;QAC1B,IAAIiG,SAAmB,EAAE;QACzB,IAAI;YACF,MAAM,EAAE,OAAOC,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAClEH;YAEF,IAAI,CAAC,WAAW,GAAG,MAAME,SAAS,cAAc;YAChDlG,QAAQkG;YACR,MAAME,yBAAyBpG,MAAM,cAAc;YACnDA,MAAM,cAAc,GAAG,CAACqG;gBACtB,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,cAAc,GAAGA;gBAExBD,yBAAyBC;YAC3B;YACAJ,SAAS;mBACHE,aAAa,EAAE;gBACnB;oBACE,MAAM;oBACN,IAAI;wBACF,IAAInG,OACFA,MAAM,cAAc,GAAGoG;oBAE3B;gBACF;aACD;QACH,EAAE,OAAOE,GAAG;YACV,IAAI,CAAC,eAAe,CAAC,SAASA;YAC9B;QACF;QACA,IAAI,CAAC,cAAc,GAAGtG;QAEtB,IAAIjB,YAAY;QAChB,IAAI,CAAC,eAAe,CAAC;QACrB,IAAIwH,YAAY;QAChB,MAAOxH,YAAY2G,MAAM,MAAM,CAAE;YAC/B,MAAMzG,aAAa,IAAI,CAAC,cAAc,CAACF,UAAU;YACjD,IAAI,CAAC,aAAa,CAACA,WAAW;YAC9B,IAAI,CAAC,YAAY,CAACA;YAClB,IAAI,CAAC,kCAAkC,GAAG;YAE1C,IAAI;gBACF,MAAM,IAAI,CAAC,QAAQ,CAACE,YAAY,IAAI,CAAC,cAAc;gBACnD,IAAI,CAAC,aAAa,CAACF,WAAW;YAChC,EAAE,OAAOuH,GAAG;gBACV,IAAI,CAAC,aAAa,CAACvH,WAAW,SAAgBuH;gBAC9C,MAAME,sBAAuBxG,MAAc,mBAAmB;gBAC9D,IACE,CAAC,IAAI,CAAC,kCAAkC,IACxC,AAA+B,cAA/B,OAAOwG,qBAEP,IAAI;oBACF,MAAMA,oBAAoB,IAAI,CAC5BxG,OACA,CAAC,mBAAmB,EAAEf,WAAW,IAAI,EAAE,EACvC;wBACE,OAAOqH;wBACP,SAAS,CAAC,KAAK,EAAErH,WAAW,WAAW,IAAI,EAAE,iCAAiC,EAAEA,WAAW,IAAI,CAAC,EAAE,CAAC;oBACrG;gBAEJ,EAAE,OAAOwH,aAAa;oBACpB7I,MAAM,yCAAyC6I;gBACjD;gBAGF,IAAIxH,WAAW,eAAe;qBAEvB;oBACL,IAAI,CAAC,UAAU,GAAGe,MAAM,UAAU;oBAClCuG,YAAY;oBACZ;gBACF;YACF;YACA,IAAI,CAAC,UAAU,GAAGvG,OAAO;YACzBjB;QACF;QAEA,IAAIwH,WACF,IAAI,CAAC,eAAe,CAAC;aAErB,IAAI,CAAC,eAAe,CAAC;QAEvB,IAAI,CAAC,cAAc,GAAG;QAGtB,KAAK,MAAMG,MAAMT,OACf,IAAI;YAEF,MAAMS,GAAG,EAAE;QAEb,EAAE,OAAOJ,GAAG,CAEZ;IAEJ;IArsBA,YACUK,MAA0B,EAC1BC,UAGN,EACKC,kBAAiE,EACxEC,UAAmB,CACnB;;;;QAvBF,uBAAO,oBAAP;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAO,cAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,sBAAR;QACA,uBAAO,UAAP;QACA,uBAAO,sBAAP;QACA,uBAAO,gBAAP;QACA,uBAAQ,kBAAR;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,eAAR;QACA,uBAAQ,cAAR;QACA,uBAAQ,sCAAR;aAEUH,MAAM,GAANA;aACAC,UAAU,GAAVA;aAIDC,kBAAkB,GAAlBA;aApBF,cAAc,GAA6B,EAAE;aAC7C,MAAM,GAA4B;aAGjC,kBAAkB,GAAG;aAIrB,cAAc,GAAiB;aAG/B,WAAW,GAAmB,EAAE;aAEhC,kCAAkC,GAAG;QAU3C,IAAI,CAAC,UAAU,GAAGC;QAClB,IAAI,CAAC,MAAM,GAAG,CAAC;QACf,MAAMC,uBACJJ,OAAO,KAAK,EAAE,gBAAgBA,OAAO,KAAK,EAAE;QAE9C,IAAII,AAAyBlH,WAAzBkH,wBAAsCJ,OAAO,KAAK,EAAE;YACtD,IACEA,AAA8B9G,WAA9B8G,OAAO,KAAK,CAAC,YAAY,IACzBA,AAAiC9G,WAAjC8G,OAAO,KAAK,CAAC,eAAe,EAE5B/H,QAAQ,IAAI,CACV;YAIJ+H,OAAO,KAAK,CAAC,YAAY,GAAGI;QAC9B;QAEA,IAAI,CAAC,MAAM,GACTJ,OAAO,MAAM,IACbA,OAAO,GAAG,IACVA,OAAO,OAAO,IACdA,OAAO,GAAG,IACVA,OAAO,QAAQ,IACfA,OAAO,MAAM;QAEf,IAAIK,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAAE;YAC7B,IAAI,CAAC,MAAM,GAAGpH;YACdjC,MAAM;QACR,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,QAAQ;YAC9B,IAAI,CAAC,MAAM,GAAGyB,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM;YACvD1B,MAAM,mCAAmC,IAAI,CAAC,MAAM;QACtD,OAAO;YACL,MAAMsJ,aAAa,IAAI,CAAC,UAAU,GAC9BC,AAAAA,IAAAA,mCAAAA,QAAAA,AAAAA,EAAS,IAAI,CAAC,UAAU,EAAE,SAAS,OAAO,CAAC,eAAe,MAC1D;YACJ,IAAI,CAAC,MAAM,GAAGC,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACZC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB,GAAGH,WAAW,CAAC,EAAEI,KAAK,GAAG,GAAG,KAAK,CAAC;YAEpC1J,MAAM,iCAAiC,IAAI,CAAC,MAAM;QACpD;QAEA,IAAIoJ,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAC3B,IAAI,CAAC,kBAAkB,GAAGpH;aACrB,IAAI,AAA2C,YAA3C,OAAO,IAAI,CAAC,MAAM,EAAE,oBAC7B,IAAI,CAAC,kBAAkB,GAAGR,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EACxBC,QAAQ,GAAG,IACX,IAAI,CAAC,MAAM,CAAC,kBAAkB;aAE3B,IAAI,IAAI,CAAC,MAAM,EAAE,uBAAuB,MAC7C,IAAI,CAAC,kBAAkB,GAAG8H,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACxBC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB;QAIJ,IAAI,CAAC,cAAc,GAAIV,AAAAA,CAAAA,OAAO,KAAK,IAAI,EAAC,EAAG,GAAG,CAAC,CAAChG,MAAM5B,YAAe;gBACnE,GAAG4B,IAAI;gBACP,OAAO5B;gBACP,QAAQ;gBACR,YAAY4B,KAAK,IAAI,EAAE,UAAU;YACnC;IACF;AA8nBF"}
+{"version":3,"file":"yaml/player.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/yaml/player.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { existsSync, mkdirSync, writeFileSync } from 'node:fs';\nimport { basename, dirname, join, resolve } from 'node:path';\nimport { assert, ifInBrowser, ifInWorker } from '@midscene/shared/utils';\nimport { type ZodTypeAny, z } from 'zod';\n\n// previous defined yaml flow, as a helper\ninterface MidsceneYamlFlowItemAIInput extends LocateOption {\n  // previous version\n  // aiInput: string; // value to input\n  // locate: TUserPrompt; // where to input\n  aiInput: TUserPrompt | undefined; // where to input\n  value: string | number; // value to input\n}\n\ninterface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {\n  // previous version\n  // aiKeyboardPress: string;\n  // locate?: TUserPrompt; // where to press, optional\n  aiKeyboardPress: TUserPrompt | undefined; // where to press\n  keyName: string; // key to press\n}\n\ninterface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {\n  // previous version\n  // aiScroll: null;\n  // locate?: TUserPrompt; // which area to scroll, optional\n  aiScroll: TUserPrompt | undefined; // which area to scroll\n}\n\ntype RuntimeYamlFlowItem =\n  | MidsceneYamlFlowItem\n  | MidsceneYamlFlowItemAIInput\n  | MidsceneYamlFlowItemAIKeyboardPress\n  | MidsceneYamlFlowItemAIScroll;\n\nimport type { Agent } from '@/agent/agent';\nimport type { TUserPrompt } from '@/common';\nimport type {\n  DeviceAction,\n  FreeFn,\n  LocateOption,\n  MidsceneYamlFlowItem,\n  MidsceneYamlFlowItemAIAction,\n  MidsceneYamlFlowItemAIAssert,\n  MidsceneYamlFlowItemAIWaitFor,\n  MidsceneYamlFlowItemEvaluateJavaScript,\n  MidsceneYamlFlowItemLogScreenshot,\n  MidsceneYamlFlowItemRunGherkinScenario,\n  MidsceneYamlFlowItemSleep,\n  MidsceneYamlScript,\n  MidsceneYamlScriptEnv,\n  ScriptPlayerStatusValue,\n  ScriptPlayerTaskStatus,\n  ScrollParam,\n} from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { getDebug } from '@midscene/shared/logger';\nimport {\n  buildDetailedLocateParam,\n  buildDetailedLocateParamAndRestParams,\n} from './utils';\n\nconst debug = getDebug('yaml-player');\n\nconst aiTaskHandlerMap = {\n  aiQuery: 'aiQuery',\n  aiNumber: 'aiNumber',\n  aiString: 'aiString',\n  aiBoolean: 'aiBoolean',\n  aiAsk: 'aiAsk',\n  aiLocate: 'aiLocate',\n} as const;\n\ntype AISimpleTaskKey = keyof typeof aiTaskHandlerMap;\n\nconst isStringParamSchema = (schema?: ZodTypeAny): boolean => {\n  if (!schema) {\n    return false;\n  }\n\n  const schemaDef = (schema as any)?._def;\n  if (!schemaDef?.typeName) {\n    return false;\n  }\n\n  switch (schemaDef.typeName) {\n    case z.ZodFirstPartyTypeKind.ZodString:\n    case z.ZodFirstPartyTypeKind.ZodEnum:\n    case z.ZodFirstPartyTypeKind.ZodNativeEnum:\n      return true;\n    case z.ZodFirstPartyTypeKind.ZodLiteral:\n      return typeof schemaDef.value === 'string';\n    case z.ZodFirstPartyTypeKind.ZodOptional:\n    case z.ZodFirstPartyTypeKind.ZodNullable:\n    case z.ZodFirstPartyTypeKind.ZodDefault:\n      return isStringParamSchema(schemaDef.innerType);\n    case z.ZodFirstPartyTypeKind.ZodEffects:\n      return isStringParamSchema(schemaDef.schema);\n    case z.ZodFirstPartyTypeKind.ZodPipeline:\n      return isStringParamSchema(schemaDef.out);\n    case z.ZodFirstPartyTypeKind.ZodUnion: {\n      const options = schemaDef.options as ZodTypeAny[] | undefined;\n      return Array.isArray(options)\n        ? options.every((option) => isStringParamSchema(option))\n        : false;\n    }\n    default:\n      return false;\n  }\n};\n\nconst buildShortcutActionParam = (\n  actionName: string,\n  interfaceAlias: string | undefined,\n  value: string,\n) => {\n  if (actionName === 'Launch' || interfaceAlias === 'launch') {\n    return { uri: value };\n  }\n\n  if (actionName === 'Terminate' || interfaceAlias === 'terminate') {\n    return { uri: value };\n  }\n\n  if (\n    actionName === 'RunAdbShell' ||\n    interfaceAlias === 'runAdbShell' ||\n    actionName === 'RunHdcShell' ||\n    interfaceAlias === 'runHdcShell'\n  ) {\n    return { command: value };\n  }\n\n  return undefined;\n};\nexport class ScriptPlayer<T extends MidsceneYamlScriptEnv> {\n  public currentTaskIndex?: number;\n  public taskStatusList: ScriptPlayerTaskStatus[] = [];\n  public status: ScriptPlayerStatusValue = 'init';\n  public reportFile?: string | null;\n  public result: Record<string, any>;\n  private unnamedResultIndex = 0;\n  public output?: string | null;\n  public unstableLogContent?: string | null;\n  public errorInSetup?: Error;\n  private interfaceAgent: Agent | null = null;\n  public agentStatusTip?: string;\n  public target?: MidsceneYamlScriptEnv;\n  private actionSpace: DeviceAction[] = [];\n  private scriptPath?: string;\n  private failedReportExecutionInCurrentStep = false;\n  constructor(\n    private script: MidsceneYamlScript,\n    private setupAgent: (platform: T) => Promise<{\n      agent: Agent;\n      freeFn: FreeFn[];\n    }>,\n    public onTaskStatusChange?: (taskStatus: ScriptPlayerTaskStatus) => void,\n    scriptPath?: string,\n  ) {\n    this.scriptPath = scriptPath;\n    this.result = {};\n    const resolvedAiActContext =\n      script.agent?.aiActContext ?? script.agent?.aiActionContext;\n\n    if (resolvedAiActContext !== undefined && script.agent) {\n      if (\n        script.agent.aiActContext === undefined &&\n        script.agent.aiActionContext !== undefined\n      ) {\n        console.warn(\n          'agent.aiActionContext is deprecated, please use agent.aiActContext instead. The legacy name is still accepted for backward compatibility.',\n        );\n      }\n\n      script.agent.aiActContext = resolvedAiActContext;\n    }\n\n    this.target =\n      script.target ||\n      script.web ||\n      script.android ||\n      script.ios ||\n      script.computer ||\n      script.config;\n\n    if (ifInBrowser || ifInWorker) {\n      this.output = undefined;\n      debug('output is undefined in browser or worker');\n    } else if (this.target?.output) {\n      this.output = resolve(process.cwd(), this.target.output);\n      debug('setting output by config.output', this.output);\n    } else {\n      const scriptName = this.scriptPath\n        ? basename(this.scriptPath, '.yaml').replace(/\\.(ya?ml)$/i, '')\n        : 'script';\n      this.output = join(\n        getMidsceneRunSubDir('output'),\n        `${scriptName}-${Date.now()}.json`,\n      );\n      debug('setting output by script path', this.output);\n    }\n\n    if (ifInBrowser || ifInWorker) {\n      this.unstableLogContent = undefined;\n    } else if (typeof this.target?.unstableLogContent === 'string') {\n      this.unstableLogContent = resolve(\n        process.cwd(),\n        this.target.unstableLogContent,\n      );\n    } else if (this.target?.unstableLogContent === true) {\n      this.unstableLogContent = join(\n        getMidsceneRunSubDir('output'),\n        'unstableLogContent.json',\n      );\n    }\n\n    this.taskStatusList = (script.tasks || []).map((task, taskIndex) => ({\n      ...task,\n      index: taskIndex,\n      status: 'init',\n      totalSteps: task.flow?.length || 0,\n    }));\n  }\n\n  private setResult(key: string | undefined, value: any) {\n    const keyToUse = key || this.unnamedResultIndex++;\n    if (this.result[keyToUse]) {\n      console.warn(`result key ${keyToUse} already exists, will overwrite`);\n    }\n    this.result[keyToUse] = value;\n\n    return this.flushResult();\n  }\n\n  private setPlayerStatus(status: ScriptPlayerStatusValue, error?: Error) {\n    this.status = status;\n    this.errorInSetup = error;\n  }\n\n  private notifyCurrentTaskStatusChange(taskIndex?: number) {\n    const taskIndexToNotify =\n      typeof taskIndex === 'number' ? taskIndex : this.currentTaskIndex;\n\n    if (typeof taskIndexToNotify !== 'number') {\n      return;\n    }\n\n    const taskStatus = this.taskStatusList[taskIndexToNotify];\n    if (this.onTaskStatusChange) {\n      this.onTaskStatusChange(taskStatus);\n    }\n  }\n\n  private async setTaskStatus(\n    index: number,\n    statusValue: ScriptPlayerStatusValue,\n    error?: Error,\n  ) {\n    this.taskStatusList[index].status = statusValue;\n    if (error) {\n      this.taskStatusList[index].error = error;\n    }\n\n    this.notifyCurrentTaskStatusChange(index);\n  }\n\n  private setTaskIndex(taskIndex: number) {\n    this.currentTaskIndex = taskIndex;\n  }\n\n  private flushResult() {\n    if (this.output) {\n      const output = resolve(process.cwd(), this.output);\n      const outputDir = dirname(output);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(output, JSON.stringify(this.result || {}, undefined, 2));\n    }\n  }\n\n  private flushUnstableLogContent() {\n    if (this.unstableLogContent) {\n      const content = this.interfaceAgent?._unstableLogContent();\n      const filePath = resolve(process.cwd(), this.unstableLogContent);\n      const outputDir = dirname(filePath);\n      if (!existsSync(outputDir)) {\n        mkdirSync(outputDir, { recursive: true });\n      }\n      writeFileSync(filePath, JSON.stringify(content, null, 2));\n    }\n  }\n\n  async playTask(taskStatus: ScriptPlayerTaskStatus, agent: Agent) {\n    const { flow } = taskStatus;\n    assert(flow, 'missing flow in task');\n\n    for (const flowItemIndex in flow) {\n      const currentStep = Number.parseInt(flowItemIndex, 10);\n      taskStatus.currentStep = currentStep;\n      const flowItem = flow[flowItemIndex] as RuntimeYamlFlowItem;\n      const flowItemRecord = flowItem as Record<string, unknown>;\n      const executionCountBeforeStep = agent.dump?.executions?.length ?? 0;\n      this.failedReportExecutionInCurrentStep = false;\n\n      try {\n        await this.playFlowItem(agent, flowItem, flowItemRecord, flowItemIndex);\n      } catch (error) {\n        this.failedReportExecutionInCurrentStep =\n          this.hasFailedReportExecutionSince(agent, executionCountBeforeStep);\n        throw error;\n      }\n    }\n    this.reportFile = agent.reportFile;\n    await this.flushUnstableLogContent();\n  }\n\n  private hasFailedReportExecutionSince(\n    agent: Agent,\n    executionCountBefore: number,\n  ) {\n    return (agent.dump?.executions ?? [])\n      .slice(executionCountBefore)\n      .some((execution) =>\n        execution.tasks.some(\n          (task) =>\n            task.status === 'failed' ||\n            Boolean(task.error || task.errorMessage),\n        ),\n      );\n  }\n\n  private async playFlowItem(\n    agent: Agent,\n    flowItem: RuntimeYamlFlowItem,\n    flowItemRecord: Record<string, unknown>,\n    flowItemIndex: string,\n  ) {\n    // Skip Finalize action from cache - it's a planning-only marker\n    if ('Finalize' in flowItemRecord) {\n      return;\n    }\n\n    debug(\n      `playing step ${flowItemIndex}, flowItem=${JSON.stringify(flowItem)}`,\n    );\n    const simpleAIKey = (\n      Object.keys(aiTaskHandlerMap) as AISimpleTaskKey[]\n    ).find((key) => Object.prototype.hasOwnProperty.call(flowItemRecord, key));\n    if (\n      'aiAct' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n      'aiAction' in (flowItem as MidsceneYamlFlowItemAIAction) ||\n      'ai' in (flowItem as MidsceneYamlFlowItemAIAction)\n    ) {\n      const actionTask = flowItem as MidsceneYamlFlowItemAIAction;\n      const { aiAct, aiAction, ai, instruction, ...actionOptions } =\n        actionTask as any;\n      const actionPrompt = aiAct ?? aiAction ?? ai;\n      let promptForAI: TUserPrompt | undefined;\n\n      if (typeof instruction === 'string' && instruction) {\n        promptForAI = instruction;\n      } else if (\n        instruction &&\n        typeof instruction === 'object' &&\n        typeof (instruction as { prompt?: unknown }).prompt === 'string' &&\n        (instruction as { prompt?: string }).prompt\n      ) {\n        promptForAI = instruction as TUserPrompt;\n      } else if (\n        actionPrompt &&\n        typeof actionPrompt === 'object' &&\n        typeof (actionPrompt as { prompt?: unknown }).prompt === 'string' &&\n        (actionPrompt as { prompt?: string }).prompt\n      ) {\n        promptForAI = actionPrompt as TUserPrompt;\n      } else if (typeof actionPrompt === 'string' && actionPrompt) {\n        promptForAI = actionPrompt;\n      }\n\n      assert(promptForAI, 'missing prompt for ai (aiAct)');\n      await agent.aiAct(promptForAI, actionOptions);\n    } else if (\n      'runGherkinScenario' in\n      (flowItem as MidsceneYamlFlowItemRunGherkinScenario)\n    ) {\n      const gherkinScenarioTask =\n        flowItem as MidsceneYamlFlowItemRunGherkinScenario;\n      const { runGherkinScenario } = gherkinScenarioTask;\n      assert(runGherkinScenario, 'missing scenario for runGherkinScenario');\n      await agent.runGherkinScenario(runGherkinScenario, {\n        cacheable: false,\n      });\n    } else if ('aiAssert' in (flowItem as MidsceneYamlFlowItemAIAssert)) {\n      const assertTask = flowItem as MidsceneYamlFlowItemAIAssert;\n      const {\n        aiAssert: prompt,\n        errorMessage: msg,\n        name,\n        ...restOpts\n      } = assertTask;\n      assert(prompt, 'missing prompt for aiAssert');\n      const { pass, thought, message } =\n        (await agent.aiAssert(prompt, msg, {\n          ...restOpts,\n          keepRawResponse: true,\n        })) || {};\n\n      this.setResult(name, {\n        pass,\n        thought,\n        message,\n      });\n\n      if (!pass) {\n        throw new Error(message);\n      }\n    } else if (simpleAIKey) {\n      const {\n        [simpleAIKey]: prompt,\n        name,\n        ...options\n      } = flowItem as Record<string, any>;\n      assert(prompt, `missing prompt for ${simpleAIKey}`);\n      const agentMethod = (agent as any)[aiTaskHandlerMap[simpleAIKey]];\n      assert(\n        typeof agentMethod === 'function',\n        `missing agent method for ${simpleAIKey}`,\n      );\n      const aiResult = await agentMethod.call(agent, prompt, options);\n      this.setResult(name, aiResult);\n    } else if ('aiWaitFor' in (flowItem as MidsceneYamlFlowItemAIWaitFor)) {\n      const waitForTask = flowItem as MidsceneYamlFlowItemAIWaitFor;\n      const { aiWaitFor, timeout, ...restWaitForOpts } = waitForTask;\n      const prompt = aiWaitFor;\n      assert(prompt, 'missing prompt for aiWaitFor');\n      const waitForOptions = {\n        ...restWaitForOpts,\n        ...(timeout !== undefined ? { timeout, timeoutMs: timeout } : {}),\n      };\n      await agent.aiWaitFor(prompt, waitForOptions);\n    } else if ('sleep' in flowItem) {\n      const sleepTask = flowItem as unknown as MidsceneYamlFlowItemSleep;\n      const ms = sleepTask.sleep;\n      let msNumber = ms;\n      if (typeof ms === 'string') {\n        msNumber = Number.parseInt(ms, 10);\n      }\n      assert(\n        msNumber && msNumber > 0,\n        `ms for sleep must be greater than 0, but got ${ms}`,\n      );\n      await new Promise((resolve) => setTimeout(resolve, msNumber));\n    } else if ('javascript' in flowItem) {\n      const evaluateJavaScriptTask =\n        flowItem as unknown as MidsceneYamlFlowItemEvaluateJavaScript;\n\n      const result = await agent.evaluateJavaScript(\n        evaluateJavaScriptTask.javascript,\n      );\n      this.setResult(evaluateJavaScriptTask.name, result);\n    } else if (\n      'logScreenshot' in (flowItem as MidsceneYamlFlowItemLogScreenshot) ||\n      'recordToReport' in (flowItem as MidsceneYamlFlowItemLogScreenshot)\n    ) {\n      const recordTask = flowItem as MidsceneYamlFlowItemLogScreenshot;\n      const title =\n        recordTask.recordToReport ?? recordTask.logScreenshot ?? 'untitled';\n      const content = recordTask.content || '';\n      await agent.recordToReport(title, { content });\n    } else if ('aiInput' in flowItem) {\n      // may be input empty string ''\n      const {\n        aiInput,\n        value: rawValue,\n        ...inputTask\n      } = flowItem as unknown as MidsceneYamlFlowItemAIInput;\n\n      // Compatibility with previous version:\n      // Old format: { aiInput: string (value), locate: TUserPrompt }\n      // New format - 1: { aiInput: TUserPrompt, value: string | number }\n      // New format - 2: { aiInput: undefined, locate: TUserPrompt, value: string | number }\n      let locatePrompt: TUserPrompt | undefined;\n      let value: string | number | undefined;\n      if ((inputTask as any).locate) {\n        // Old format - aiInput is the value, locate is the prompt\n        // Keep backward compatibility: empty string is treated as no value\n        value = (aiInput as string | number) || rawValue;\n        locatePrompt = (inputTask as any).locate;\n      } else {\n        // New format - aiInput is the prompt, value is the value\n        locatePrompt = aiInput || '';\n        value = rawValue;\n      }\n\n      // Convert value to string for Input action\n      await agent.callActionInActionSpace('Input', {\n        ...inputTask,\n        ...(value !== undefined ? { value: String(value) } : {}),\n        ...(locatePrompt\n          ? { locate: buildDetailedLocateParam(locatePrompt, inputTask) }\n          : {}),\n      });\n    } else if ('aiKeyboardPress' in flowItem) {\n      const { aiKeyboardPress, ...keyboardPressTask } =\n        flowItem as unknown as MidsceneYamlFlowItemAIKeyboardPress;\n\n      // Compatibility with previous version:\n      // Old format: { aiKeyboardPress: string (key), locate?: TUserPrompt }\n      // New format - 1: { aiKeyboardPress: TUserPrompt, keyName: string }\n      // New format - 2: { aiKeyboardPress: , locate?: TUserPrompt, keyName: string }\n      let locatePrompt: TUserPrompt | undefined;\n      let keyName: string | undefined;\n      if ((keyboardPressTask as any).locate) {\n        // Old format - aiKeyboardPress is the key, locate is the prompt\n        keyName = aiKeyboardPress as string;\n        locatePrompt = (keyboardPressTask as any).locate;\n      } else if (keyboardPressTask.keyName) {\n        // New format - aiKeyboardPress is the prompt, key is the key\n        keyName = keyboardPressTask.keyName;\n        locatePrompt = aiKeyboardPress;\n      } else {\n        keyName = aiKeyboardPress as string;\n      }\n\n      await agent.callActionInActionSpace('KeyboardPress', {\n        ...keyboardPressTask,\n        ...(keyName ? { keyName } : {}),\n        ...(locatePrompt\n          ? {\n              locate: buildDetailedLocateParam(locatePrompt, keyboardPressTask),\n            }\n          : {}),\n      });\n    } else if ('aiScroll' in flowItem) {\n      const { aiScroll, ...scrollTask } =\n        flowItem as unknown as MidsceneYamlFlowItemAIScroll;\n\n      // Compatibility with previous version:\n      // Old format: { aiScroll: null, locate?: TUserPrompt, direction, scrollType, distance? }\n      // New format - 1: { aiScroll: TUserPrompt, direction, scrollType, distance? }\n      // New format - 2: { aiScroll: undefined, locate: TUserPrompt, direction, scrollType, distance? }\n      const { locate, ...scrollOptions } = scrollTask as any;\n      const locatePrompt: TUserPrompt | undefined =\n        locate ?? aiScroll ?? undefined;\n\n      await agent.aiScroll(locatePrompt, scrollOptions);\n    } else if ('aiTap' in flowItem) {\n      const { aiTap, prompt, locate, ...tapOptions } = flowItem as any;\n\n      let locatePrompt: TUserPrompt;\n      let opts = tapOptions;\n      // Support both formats:\n      // 1. { aiTap: null, locate: { prompt, images, ... } }  (locate as sibling key)\n      // 2. { aiTap: { locate: { prompt, images, ... } } }    (locate nested in aiTap)\n      const locateObj =\n        locate ??\n        (typeof aiTap === 'object' && aiTap !== null\n          ? aiTap.locate\n          : undefined);\n\n      if (typeof aiTap === 'string' && aiTap) {\n        // User YAML: aiTap: 'search input box'\n        locatePrompt = aiTap;\n      } else if (typeof locateObj === 'object' && locateObj?.prompt) {\n        // buildYamlFlowFromPlans: { aiTap: '', locate: { prompt, deepLocate, cacheable } }\n        const { prompt: lp, ...locateOpts } = locateObj;\n        locatePrompt = lp;\n        opts = { ...locateOpts, ...tapOptions };\n      } else {\n        // User YAML: aiTap: { prompt: '...' } or aiTap: null + prompt: '...'\n        locatePrompt = aiTap?.prompt || prompt || locateObj;\n      }\n\n      assert(locatePrompt, 'missing prompt for aiTap');\n      await agent.aiTap(locatePrompt, opts);\n    } else {\n      // generic action, find the action in actionSpace\n\n      /* for aiRightClick, the parameters are a flattened data for the 'locate', these are all valid data\n\n        - aiRightClick: 'search input box'\n        - aiRightClick: 'search input box'\n          deepLocate: true\n          cacheable: false\n        - aiRightClick:\n          prompt: 'search input box'\n        - aiRightClick:\n          prompt: 'search input box'\n          deepLocate: true\n          cacheable: false\n        */\n\n      const actionSpace = this.actionSpace;\n      let locatePromptShortcut: string | undefined;\n      let actionParamForMatchedAction: unknown;\n      const matchedAction = actionSpace.find((action) => {\n        const actionInterfaceAlias = action.interfaceAlias;\n        if (\n          actionInterfaceAlias &&\n          Object.prototype.hasOwnProperty.call(flowItem, actionInterfaceAlias)\n        ) {\n          actionParamForMatchedAction =\n            flowItem[actionInterfaceAlias as keyof typeof flowItem];\n          if (typeof actionParamForMatchedAction === 'string') {\n            locatePromptShortcut = actionParamForMatchedAction;\n          }\n          return true;\n        }\n\n        const keyOfActionInActionSpace = action.name;\n        if (\n          Object.prototype.hasOwnProperty.call(\n            flowItem,\n            keyOfActionInActionSpace,\n          )\n        ) {\n          actionParamForMatchedAction =\n            flowItem[keyOfActionInActionSpace as keyof typeof flowItem];\n          if (typeof actionParamForMatchedAction === 'string') {\n            locatePromptShortcut = actionParamForMatchedAction;\n          }\n          return true;\n        }\n\n        return false;\n      });\n\n      assert(\n        matchedAction,\n        `unknown flowItem in yaml: ${JSON.stringify(flowItem)}`,\n      );\n\n      const schemaIsStringParam = isStringParamSchema(\n        matchedAction.paramSchema,\n      );\n      let stringParamToCall: string | undefined;\n      const resultName = (flowItem as any).name;\n      const timeout = (flowItem as any).timeout;\n      const hasRunAdbShellAlias = Object.prototype.hasOwnProperty.call(\n        flowItem,\n        'runAdbShell',\n      );\n\n      if (\n        hasRunAdbShellAlias &&\n        typeof actionParamForMatchedAction === 'string' &&\n        typeof timeout === 'number' &&\n        typeof (agent as any).runAdbShell === 'function'\n      ) {\n        const result = await (agent as any).runAdbShell(\n          actionParamForMatchedAction,\n          { timeout },\n        );\n        if (result !== undefined) {\n          this.setResult(resultName, result);\n        }\n        return;\n      }\n\n      const specialActionParamToCall =\n        typeof actionParamForMatchedAction === 'string'\n          ? buildShortcutActionParam(\n              matchedAction.name,\n              matchedAction.interfaceAlias,\n              actionParamForMatchedAction,\n            )\n          : undefined;\n      if (specialActionParamToCall) {\n        debug(\n          `matchedAction: ${matchedAction.name}`,\n          `flowParams: ${JSON.stringify(specialActionParamToCall)}`,\n        );\n        const result = await agent.callActionInActionSpace(\n          matchedAction.name,\n          specialActionParamToCall,\n        );\n\n        if (result !== undefined) {\n          this.setResult(resultName, result);\n        }\n      } else if (\n        typeof actionParamForMatchedAction === 'string' &&\n        schemaIsStringParam\n      ) {\n        if (matchedAction.paramSchema) {\n          const parseResult = matchedAction.paramSchema.safeParse(\n            actionParamForMatchedAction,\n          );\n          if (parseResult.success && typeof parseResult.data === 'string') {\n            stringParamToCall = parseResult.data;\n          } else if (!parseResult.success) {\n            debug(\n              `parse failed for action ${matchedAction.name} with string param`,\n              parseResult.error,\n            );\n            stringParamToCall = actionParamForMatchedAction;\n          }\n        } else {\n          stringParamToCall = actionParamForMatchedAction;\n        }\n\n        if (stringParamToCall !== undefined) {\n          debug(\n            `matchedAction: ${matchedAction.name}`,\n            `flowParams: ${JSON.stringify(stringParamToCall)}`,\n          );\n          const result = await agent.callActionInActionSpace(\n            matchedAction.name,\n            stringParamToCall,\n          );\n\n          // Store result if there's a name property in flowItem\n          const resultName = (flowItem as any).name;\n          if (result !== undefined) {\n            this.setResult(resultName, result);\n          }\n        }\n      } else {\n        // Determine the source for parameter extraction:\n        // - If we have a locatePromptShortcut, use the flowItem (for actions like aiTap with prompt)\n        // - Otherwise, use actionParamForMatchedAction (for actions like runWdaRequest with structured params)\n        const sourceForParams =\n          locatePromptShortcut &&\n          typeof actionParamForMatchedAction === 'string'\n            ? { ...flowItem, prompt: locatePromptShortcut }\n            : typeof actionParamForMatchedAction === 'object' &&\n                actionParamForMatchedAction !== null\n              ? actionParamForMatchedAction\n              : flowItem;\n\n        const { locateParam, restParams } =\n          buildDetailedLocateParamAndRestParams(\n            locatePromptShortcut || '',\n            sourceForParams as LocateOption,\n            [\n              matchedAction.name,\n              matchedAction.interfaceAlias || '_never_mind_',\n            ],\n          );\n\n        const flowParams = {\n          ...restParams,\n          locate: locateParam,\n        };\n\n        debug(\n          `matchedAction: ${matchedAction.name}`,\n          `flowParams: ${JSON.stringify(flowParams, null, 2)}`,\n        );\n        const result = await agent.callActionInActionSpace(\n          matchedAction.name,\n          flowParams,\n        );\n\n        // Store result if there's a name property in flowItem\n        const resultName = (flowItem as any).name;\n        if (result !== undefined) {\n          this.setResult(resultName, result);\n        }\n      }\n    }\n  }\n\n  async run() {\n    const { target, web, android, ios, harmony, computer, tasks } = this.script;\n    const webEnv = web || target;\n    const androidEnv = android;\n    const iosEnv = ios;\n    const harmonyEnv = harmony;\n    const computerEnv = computer;\n    const platform =\n      webEnv || androidEnv || iosEnv || harmonyEnv || computerEnv;\n\n    this.setPlayerStatus('running');\n\n    let agent: Agent | null = null;\n    let freeFn: FreeFn[] = [];\n    try {\n      const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(\n        platform as T,\n      );\n      this.actionSpace = await newAgent.getActionSpace();\n      agent = newAgent;\n      const originalOnTaskStartTip = agent.onTaskStartTip;\n      agent.onTaskStartTip = (tip) => {\n        if (this.status === 'running') {\n          this.agentStatusTip = tip;\n        }\n        originalOnTaskStartTip?.(tip);\n      };\n      freeFn = [\n        ...(newFreeFn || []),\n        {\n          name: 'restore-agent-onTaskStartTip',\n          fn: () => {\n            if (agent) {\n              agent.onTaskStartTip = originalOnTaskStartTip;\n            }\n          },\n        },\n      ];\n    } catch (e) {\n      this.setPlayerStatus('error', e as Error);\n      return;\n    }\n    this.interfaceAgent = agent;\n\n    let taskIndex = 0;\n    this.setPlayerStatus('running');\n    let errorFlag = false;\n    while (taskIndex < tasks.length) {\n      const taskStatus = this.taskStatusList[taskIndex];\n      this.setTaskStatus(taskIndex, 'running' as any);\n      this.setTaskIndex(taskIndex);\n      this.failedReportExecutionInCurrentStep = false;\n\n      try {\n        await this.playTask(taskStatus, this.interfaceAgent);\n        this.setTaskStatus(taskIndex, 'done' as any);\n      } catch (e) {\n        this.setTaskStatus(taskIndex, 'error' as any, e as Error);\n        const recordErrorToReport = (agent as any).recordErrorToReport;\n        if (\n          !this.failedReportExecutionInCurrentStep &&\n          typeof recordErrorToReport === 'function'\n        ) {\n          try {\n            await recordErrorToReport.call(\n              agent,\n              `YAML task failed - ${taskStatus.name}`,\n              {\n                error: e as Error,\n                content: `Step ${taskStatus.currentStep ?? 0} failed while running YAML task \"${taskStatus.name}\".`,\n              },\n            );\n          } catch (reportError) {\n            debug('failed to record yaml error to report', reportError);\n          }\n        }\n\n        if (taskStatus.continueOnError) {\n          // nothing more to do\n        } else {\n          this.reportFile = agent.reportFile;\n          errorFlag = true;\n          break;\n        }\n      }\n      this.reportFile = agent?.reportFile;\n      taskIndex++;\n    }\n\n    if (errorFlag) {\n      this.setPlayerStatus('error');\n    } else {\n      this.setPlayerStatus('done');\n    }\n    this.agentStatusTip = '';\n\n    // free the resources\n    for (const fn of freeFn) {\n      try {\n        // console.log('freeing', fn.name);\n        await fn.fn();\n        // console.log('freed', fn.name);\n      } catch (e) {\n        // console.error('error freeing', fn.name, e);\n      }\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","aiTaskHandlerMap","isStringParamSchema","schema","schemaDef","z","options","Array","option","buildShortcutActionParam","actionName","interfaceAlias","value","ScriptPlayer","keyToUse","console","status","error","taskIndex","taskIndexToNotify","taskStatus","index","statusValue","output","resolve","process","outputDir","dirname","existsSync","mkdirSync","writeFileSync","JSON","undefined","content","filePath","agent","flow","assert","flowItemIndex","currentStep","Number","flowItem","flowItemRecord","executionCountBeforeStep","executionCountBefore","execution","task","Boolean","simpleAIKey","actionTask","aiAct","aiAction","ai","instruction","actionOptions","actionPrompt","promptForAI","gherkinScenarioTask","runGherkinScenario","assertTask","prompt","msg","name","restOpts","pass","thought","message","Error","agentMethod","aiResult","waitForTask","aiWaitFor","timeout","restWaitForOpts","waitForOptions","sleepTask","ms","msNumber","Promise","setTimeout","evaluateJavaScriptTask","result","recordTask","title","aiInput","rawValue","inputTask","locatePrompt","String","buildDetailedLocateParam","aiKeyboardPress","keyboardPressTask","keyName","aiScroll","scrollTask","locate","scrollOptions","aiTap","tapOptions","opts","locateObj","lp","locateOpts","actionSpace","locatePromptShortcut","actionParamForMatchedAction","matchedAction","action","actionInterfaceAlias","keyOfActionInActionSpace","schemaIsStringParam","stringParamToCall","resultName","hasRunAdbShellAlias","specialActionParamToCall","parseResult","sourceForParams","locateParam","restParams","buildDetailedLocateParamAndRestParams","flowParams","target","web","android","ios","harmony","computer","tasks","webEnv","androidEnv","iosEnv","harmonyEnv","computerEnv","platform","freeFn","newAgent","newFreeFn","originalOnTaskStartTip","tip","e","errorFlag","recordErrorToReport","reportError","fn","script","setupAgent","onTaskStatusChange","scriptPath","resolvedAiActContext","ifInBrowser","ifInWorker","scriptName","basename","join","getMidsceneRunSubDir","Date"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACwDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEvB,MAAMC,mBAAmB;IACvB,SAAS;IACT,UAAU;IACV,UAAU;IACV,WAAW;IACX,OAAO;IACP,UAAU;AACZ;AAIA,MAAMC,sBAAsB,CAACC;IAC3B,IAAI,CAACA,QACH,OAAO;IAGT,MAAMC,YAAaD,QAAgB;IACnC,IAAI,CAACC,WAAW,UACd,OAAO;IAGT,OAAQA,UAAU,QAAQ;QACxB,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,SAAiC;QACtC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,OAA+B;QACpC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,aAAqC;YACxC,OAAO;QACT,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,UAAkC;YACrC,OAAO,AAA2B,YAA3B,OAAOD,UAAU,KAAK;QAC/B,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,WAAmC;QACxC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,WAAmC;QACxC,KAAKA,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,UAAkC;YACrC,OAAOH,oBAAoBE,UAAU,SAAS;QAChD,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,UAAkC;YACrC,OAAOH,oBAAoBE,UAAU,MAAM;QAC7C,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,WAAmC;YACtC,OAAOH,oBAAoBE,UAAU,GAAG;QAC1C,KAAKC,6BAAAA,CAAAA,CAAAA,qBAAAA,CAAAA,QAAgC;YAAE;gBACrC,MAAMC,UAAUF,UAAU,OAAO;gBACjC,OAAOG,MAAM,OAAO,CAACD,WACjBA,QAAQ,KAAK,CAAC,CAACE,SAAWN,oBAAoBM,WAC9C;YACN;QACA;YACE,OAAO;IACX;AACF;AAEA,MAAMC,2BAA2B,CAC/BC,YACAC,gBACAC;IAEA,IAAIF,AAAe,aAAfA,cAA2BC,AAAmB,aAAnBA,gBAC7B,OAAO;QAAE,KAAKC;IAAM;IAGtB,IAAIF,AAAe,gBAAfA,cAA8BC,AAAmB,gBAAnBA,gBAChC,OAAO;QAAE,KAAKC;IAAM;IAGtB,IACEF,AAAe,kBAAfA,cACAC,AAAmB,kBAAnBA,kBACAD,AAAe,kBAAfA,cACAC,AAAmB,kBAAnBA,gBAEA,OAAO;QAAE,SAASC;IAAM;AAI5B;AACO,MAAMC;IA0FH,UAAUnB,GAAuB,EAAEkB,KAAU,EAAE;QACrD,MAAME,WAAWpB,OAAO,IAAI,CAAC,kBAAkB;QAC/C,IAAI,IAAI,CAAC,MAAM,CAACoB,SAAS,EACvBC,QAAQ,IAAI,CAAC,CAAC,WAAW,EAAED,SAAS,+BAA+B,CAAC;QAEtE,IAAI,CAAC,MAAM,CAACA,SAAS,GAAGF;QAExB,OAAO,IAAI,CAAC,WAAW;IACzB;IAEQ,gBAAgBI,MAA+B,EAAEC,KAAa,EAAE;QACtE,IAAI,CAAC,MAAM,GAAGD;QACd,IAAI,CAAC,YAAY,GAAGC;IACtB;IAEQ,8BAA8BC,SAAkB,EAAE;QACxD,MAAMC,oBACJ,AAAqB,YAArB,OAAOD,YAAyBA,YAAY,IAAI,CAAC,gBAAgB;QAEnE,IAAI,AAA6B,YAA7B,OAAOC,mBACT;QAGF,MAAMC,aAAa,IAAI,CAAC,cAAc,CAACD,kBAAkB;QACzD,IAAI,IAAI,CAAC,kBAAkB,EACzB,IAAI,CAAC,kBAAkB,CAACC;IAE5B;IAEA,MAAc,cACZC,KAAa,EACbC,WAAoC,EACpCL,KAAa,EACb;QACA,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,MAAM,GAAGC;QACpC,IAAIL,OACF,IAAI,CAAC,cAAc,CAACI,MAAM,CAAC,KAAK,GAAGJ;QAGrC,IAAI,CAAC,6BAA6B,CAACI;IACrC;IAEQ,aAAaH,SAAiB,EAAE;QACtC,IAAI,CAAC,gBAAgB,GAAGA;IAC1B;IAEQ,cAAc;QACpB,IAAI,IAAI,CAAC,MAAM,EAAE;YACf,MAAMK,SAASC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM;YACjD,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQJ;YAC1B,IAAI,CAACK,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcP,QAAQQ,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,GAAGC,QAAW;QACrE;IACF;IAEQ,0BAA0B;QAChC,IAAI,IAAI,CAAC,kBAAkB,EAAE;YAC3B,MAAMC,UAAU,IAAI,CAAC,cAAc,EAAE;YACrC,MAAMC,WAAWV,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,kBAAkB;YAC/D,MAAMC,YAAYC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQO;YAC1B,IAAI,CAACN,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YACdG,AAAAA,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUH,WAAW;gBAAE,WAAW;YAAK;YAEzCI,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcI,UAAUH,KAAK,SAAS,CAACE,SAAS,MAAM;QACxD;IACF;IAEA,MAAM,SAASb,UAAkC,EAAEe,KAAY,EAAE;QAC/D,MAAM,EAAEC,IAAI,EAAE,GAAGhB;QACjBiB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,MAAM;QAEb,IAAK,MAAME,iBAAiBF,KAAM;YAChC,MAAMG,cAAcC,OAAO,QAAQ,CAACF,eAAe;YACnDlB,WAAW,WAAW,GAAGmB;YACzB,MAAME,WAAWL,IAAI,CAACE,cAAc;YACpC,MAAMI,iBAAiBD;YACvB,MAAME,2BAA2BR,MAAM,IAAI,EAAE,YAAY,UAAU;YACnE,IAAI,CAAC,kCAAkC,GAAG;YAE1C,IAAI;gBACF,MAAM,IAAI,CAAC,YAAY,CAACA,OAAOM,UAAUC,gBAAgBJ;YAC3D,EAAE,OAAOrB,OAAO;gBACd,IAAI,CAAC,kCAAkC,GACrC,IAAI,CAAC,6BAA6B,CAACkB,OAAOQ;gBAC5C,MAAM1B;YACR;QACF;QACA,IAAI,CAAC,UAAU,GAAGkB,MAAM,UAAU;QAClC,MAAM,IAAI,CAAC,uBAAuB;IACpC;IAEQ,8BACNA,KAAY,EACZS,oBAA4B,EAC5B;QACA,OAAQT,AAAAA,CAAAA,MAAM,IAAI,EAAE,cAAc,EAAC,EAChC,KAAK,CAACS,sBACN,IAAI,CAAC,CAACC,YACLA,UAAU,KAAK,CAAC,IAAI,CAClB,CAACC,OACCA,AAAgB,aAAhBA,KAAK,MAAM,IACXC,QAAQD,KAAK,KAAK,IAAIA,KAAK,YAAY;IAGjD;IAEA,MAAc,aACZX,KAAY,EACZM,QAA6B,EAC7BC,cAAuC,EACvCJ,aAAqB,EACrB;QAEA,IAAI,cAAcI,gBAChB;QAGF3C,MACE,CAAC,aAAa,EAAEuC,cAAc,WAAW,EAAEP,KAAK,SAAS,CAACU,WAAW;QAEvE,MAAMO,cACJrD,OAAO,IAAI,CAACM,kBACZ,IAAI,CAAC,CAACP,MAAQC,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC+C,gBAAgBhD;QACrE,IACE,WAAY+C,YACZ,cAAeA,YACf,QAASA,UACT;YACA,MAAMQ,aAAaR;YACnB,MAAM,EAAES,KAAK,EAAEC,QAAQ,EAAEC,EAAE,EAAEC,WAAW,EAAE,GAAGC,eAAe,GAC1DL;YACF,MAAMM,eAAeL,SAASC,YAAYC;YAC1C,IAAII;YAEJ,IAAI,AAAuB,YAAvB,OAAOH,eAA4BA,aACrCG,cAAcH;iBACT,IACLA,eACA,AAAuB,YAAvB,OAAOA,eACP,AAAwD,YAAxD,OAAQA,YAAqC,MAAM,IAClDA,YAAoC,MAAM,EAE3CG,cAAcH;iBACT,IACLE,gBACA,AAAwB,YAAxB,OAAOA,gBACP,AAAyD,YAAzD,OAAQA,aAAsC,MAAM,IACnDA,aAAqC,MAAM,EAE5CC,cAAcD;iBACT,IAAI,AAAwB,YAAxB,OAAOA,gBAA6BA,cAC7CC,cAAcD;YAGhBlB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOmB,aAAa;YACpB,MAAMrB,MAAM,KAAK,CAACqB,aAAaF;QACjC,OAAO,IACL,wBACCb,UACD;YACA,MAAMgB,sBACJhB;YACF,MAAM,EAAEiB,kBAAkB,EAAE,GAAGD;YAC/BpB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqB,oBAAoB;YAC3B,MAAMvB,MAAM,kBAAkB,CAACuB,oBAAoB;gBACjD,WAAW;YACb;QACF,OAAO,IAAI,cAAejB,UAA2C;YACnE,MAAMkB,aAAalB;YACnB,MAAM,EACJ,UAAUmB,MAAM,EAChB,cAAcC,GAAG,EACjBC,IAAI,EACJ,GAAGC,UACJ,GAAGJ;YACJtB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuB,QAAQ;YACf,MAAM,EAAEI,IAAI,EAAEC,OAAO,EAAEC,OAAO,EAAE,GAC7B,MAAM/B,MAAM,QAAQ,CAACyB,QAAQC,KAAK;gBACjC,GAAGE,QAAQ;gBACX,iBAAiB;YACnB,MAAO,CAAC;YAEV,IAAI,CAAC,SAAS,CAACD,MAAM;gBACnBE;gBACAC;gBACAC;YACF;YAEA,IAAI,CAACF,MACH,MAAM,IAAIG,MAAMD;QAEpB,OAAO,IAAIlB,aAAa;YACtB,MAAM,EACJ,CAACA,YAAY,EAAEY,MAAM,EACrBE,IAAI,EACJ,GAAGxD,SACJ,GAAGmC;YACJJ,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuB,QAAQ,CAAC,mBAAmB,EAAEZ,aAAa;YAClD,MAAMoB,cAAejC,KAAa,CAAClC,gBAAgB,CAAC+C,YAAY,CAAC;YACjEX,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAuB,cAAvB,OAAO+B,aACP,CAAC,yBAAyB,EAAEpB,aAAa;YAE3C,MAAMqB,WAAW,MAAMD,YAAY,IAAI,CAACjC,OAAOyB,QAAQtD;YACvD,IAAI,CAAC,SAAS,CAACwD,MAAMO;QACvB,OAAO,IAAI,eAAgB5B,UAA4C;YACrE,MAAM6B,cAAc7B;YACpB,MAAM,EAAE8B,SAAS,EAAEC,OAAO,EAAE,GAAGC,iBAAiB,GAAGH;YACnD,MAAMV,SAASW;YACflC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuB,QAAQ;YACf,MAAMc,iBAAiB;gBACrB,GAAGD,eAAe;gBAClB,GAAID,AAAYxC,WAAZwC,UAAwB;oBAAEA;oBAAS,WAAWA;gBAAQ,IAAI,CAAC,CAAC;YAClE;YACA,MAAMrC,MAAM,SAAS,CAACyB,QAAQc;QAChC,OAAO,IAAI,WAAWjC,UAAU;YAC9B,MAAMkC,YAAYlC;YAClB,MAAMmC,KAAKD,UAAU,KAAK;YAC1B,IAAIE,WAAWD;YACf,IAAI,AAAc,YAAd,OAAOA,IACTC,WAAWrC,OAAO,QAAQ,CAACoC,IAAI;YAEjCvC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEwC,YAAYA,WAAW,GACvB,CAAC,6CAA6C,EAAED,IAAI;YAEtD,MAAM,IAAIE,QAAQ,CAACtD,UAAYuD,WAAWvD,SAASqD;QACrD,OAAO,IAAI,gBAAgBpC,UAAU;YACnC,MAAMuC,yBACJvC;YAEF,MAAMwC,SAAS,MAAM9C,MAAM,kBAAkB,CAC3C6C,uBAAuB,UAAU;YAEnC,IAAI,CAAC,SAAS,CAACA,uBAAuB,IAAI,EAAEC;QAC9C,OAAO,IACL,mBAAoBxC,YACpB,oBAAqBA,UACrB;YACA,MAAMyC,aAAazC;YACnB,MAAM0C,QACJD,WAAW,cAAc,IAAIA,WAAW,aAAa,IAAI;YAC3D,MAAMjD,UAAUiD,WAAW,OAAO,IAAI;YACtC,MAAM/C,MAAM,cAAc,CAACgD,OAAO;gBAAElD;YAAQ;QAC9C,OAAO,IAAI,aAAaQ,UAAU;YAEhC,MAAM,EACJ2C,OAAO,EACP,OAAOC,QAAQ,EACf,GAAGC,WACJ,GAAG7C;YAMJ,IAAI8C;YACJ,IAAI3E;YACJ,IAAK0E,UAAkB,MAAM,EAAE;gBAG7B1E,QAASwE,WAA+BC;gBACxCE,eAAgBD,UAAkB,MAAM;YAC1C,OAAO;gBAELC,eAAeH,WAAW;gBAC1BxE,QAAQyE;YACV;YAGA,MAAMlD,MAAM,uBAAuB,CAAC,SAAS;gBAC3C,GAAGmD,SAAS;gBACZ,GAAI1E,AAAUoB,WAAVpB,QAAsB;oBAAE,OAAO4E,OAAO5E;gBAAO,IAAI,CAAC,CAAC;gBACvD,GAAI2E,eACA;oBAAE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcD;gBAAW,IAC5D,CAAC,CAAC;YACR;QACF,OAAO,IAAI,qBAAqB7C,UAAU;YACxC,MAAM,EAAEiD,eAAe,EAAE,GAAGC,mBAAmB,GAC7ClD;YAMF,IAAI8C;YACJ,IAAIK;YACJ,IAAKD,kBAA0B,MAAM,EAAE;gBAErCC,UAAUF;gBACVH,eAAgBI,kBAA0B,MAAM;YAClD,OAAO,IAAIA,kBAAkB,OAAO,EAAE;gBAEpCC,UAAUD,kBAAkB,OAAO;gBACnCJ,eAAeG;YACjB,OACEE,UAAUF;YAGZ,MAAMvD,MAAM,uBAAuB,CAAC,iBAAiB;gBACnD,GAAGwD,iBAAiB;gBACpB,GAAIC,UAAU;oBAAEA;gBAAQ,IAAI,CAAC,CAAC;gBAC9B,GAAIL,eACA;oBACE,QAAQE,AAAAA,IAAAA,kCAAAA,wBAAAA,AAAAA,EAAyBF,cAAcI;gBACjD,IACA,CAAC,CAAC;YACR;QACF,OAAO,IAAI,cAAclD,UAAU;YACjC,MAAM,EAAEoD,QAAQ,EAAE,GAAGC,YAAY,GAC/BrD;YAMF,MAAM,EAAEsD,MAAM,EAAE,GAAGC,eAAe,GAAGF;YACrC,MAAMP,eACJQ,UAAUF,YAAY7D;YAExB,MAAMG,MAAM,QAAQ,CAACoD,cAAcS;QACrC,OAAO,IAAI,WAAWvD,UAAU;YAC9B,MAAM,EAAEwD,KAAK,EAAErC,MAAM,EAAEmC,MAAM,EAAE,GAAGG,YAAY,GAAGzD;YAEjD,IAAI8C;YACJ,IAAIY,OAAOD;YAIX,MAAME,YACJL,UACC,CAAiB,YAAjB,OAAOE,SAAsBA,AAAU,SAAVA,QAC1BA,MAAM,MAAM,GACZjE,MAAQ;YAEd,IAAI,AAAiB,YAAjB,OAAOiE,SAAsBA,OAE/BV,eAAeU;iBACV,IAAI,AAAqB,YAArB,OAAOG,aAA0BA,WAAW,QAAQ;gBAE7D,MAAM,EAAE,QAAQC,EAAE,EAAE,GAAGC,YAAY,GAAGF;gBACtCb,eAAec;gBACfF,OAAO;oBAAE,GAAGG,UAAU;oBAAE,GAAGJ,UAAU;gBAAC;YACxC,OAEEX,eAAeU,OAAO,UAAUrC,UAAUwC;YAG5C/D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOkD,cAAc;YACrB,MAAMpD,MAAM,KAAK,CAACoD,cAAcY;QAClC,OAAO;YAiBL,MAAMI,cAAc,IAAI,CAAC,WAAW;YACpC,IAAIC;YACJ,IAAIC;YACJ,MAAMC,gBAAgBH,YAAY,IAAI,CAAC,CAACI;gBACtC,MAAMC,uBAAuBD,OAAO,cAAc;gBAClD,IACEC,wBACAjH,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC8C,UAAUmE,uBAC/C;oBACAH,8BACEhE,QAAQ,CAACmE,qBAA8C;oBACzD,IAAI,AAAuC,YAAvC,OAAOH,6BACTD,uBAAuBC;oBAEzB,OAAO;gBACT;gBAEA,MAAMI,2BAA2BF,OAAO,IAAI;gBAC5C,IACEhH,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAClC8C,UACAoE,2BAEF;oBACAJ,8BACEhE,QAAQ,CAACoE,yBAAkD;oBAC7D,IAAI,AAAuC,YAAvC,OAAOJ,6BACTD,uBAAuBC;oBAEzB,OAAO;gBACT;gBAEA,OAAO;YACT;YAEApE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEqE,eACA,CAAC,0BAA0B,EAAE3E,KAAK,SAAS,CAACU,WAAW;YAGzD,MAAMqE,sBAAsB5G,oBAC1BwG,cAAc,WAAW;YAE3B,IAAIK;YACJ,MAAMC,aAAcvE,SAAiB,IAAI;YACzC,MAAM+B,UAAW/B,SAAiB,OAAO;YACzC,MAAMwE,sBAAsBtH,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAC9D8C,UACA;YAGF,IACEwE,uBACA,AAAuC,YAAvC,OAAOR,+BACP,AAAmB,YAAnB,OAAOjC,WACP,AAAsC,cAAtC,OAAQrC,MAAc,WAAW,EACjC;gBACA,MAAM8C,SAAS,MAAO9C,MAAc,WAAW,CAC7CsE,6BACA;oBAAEjC;gBAAQ;gBAEZ,IAAIS,AAAWjD,WAAXiD,QACF,IAAI,CAAC,SAAS,CAAC+B,YAAY/B;gBAE7B;YACF;YAEA,MAAMiC,2BACJ,AAAuC,YAAvC,OAAOT,8BACHhG,yBACEiG,cAAc,IAAI,EAClBA,cAAc,cAAc,EAC5BD,+BAEFzE;YACN,IAAIkF,0BAA0B;gBAC5BnH,MACE,CAAC,eAAe,EAAE2G,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAE3E,KAAK,SAAS,CAACmF,2BAA2B;gBAE3D,MAAMjC,SAAS,MAAM9C,MAAM,uBAAuB,CAChDuE,cAAc,IAAI,EAClBQ;gBAGF,IAAIjC,AAAWjD,WAAXiD,QACF,IAAI,CAAC,SAAS,CAAC+B,YAAY/B;YAE/B,OAAO,IACL,AAAuC,YAAvC,OAAOwB,+BACPK,qBACA;gBACA,IAAIJ,cAAc,WAAW,EAAE;oBAC7B,MAAMS,cAAcT,cAAc,WAAW,CAAC,SAAS,CACrDD;oBAEF,IAAIU,YAAY,OAAO,IAAI,AAA4B,YAA5B,OAAOA,YAAY,IAAI,EAChDJ,oBAAoBI,YAAY,IAAI;yBAC/B,IAAI,CAACA,YAAY,OAAO,EAAE;wBAC/BpH,MACE,CAAC,wBAAwB,EAAE2G,cAAc,IAAI,CAAC,kBAAkB,CAAC,EACjES,YAAY,KAAK;wBAEnBJ,oBAAoBN;oBACtB;gBACF,OACEM,oBAAoBN;gBAGtB,IAAIM,AAAsB/E,WAAtB+E,mBAAiC;oBACnChH,MACE,CAAC,eAAe,EAAE2G,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAE3E,KAAK,SAAS,CAACgF,oBAAoB;oBAEpD,MAAM9B,SAAS,MAAM9C,MAAM,uBAAuB,CAChDuE,cAAc,IAAI,EAClBK;oBAIF,MAAMC,aAAcvE,SAAiB,IAAI;oBACzC,IAAIwC,AAAWjD,WAAXiD,QACF,IAAI,CAAC,SAAS,CAAC+B,YAAY/B;gBAE/B;YACF,OAAO;gBAIL,MAAMmC,kBACJZ,wBACA,AAAuC,YAAvC,OAAOC,8BACH;oBAAE,GAAGhE,QAAQ;oBAAE,QAAQ+D;gBAAqB,IAC5C,AAAuC,YAAvC,OAAOC,+BACLA,AAAgC,SAAhCA,8BACAA,8BACAhE;gBAER,MAAM,EAAE4E,WAAW,EAAEC,UAAU,EAAE,GAC/BC,AAAAA,IAAAA,kCAAAA,qCAAAA,AAAAA,EACEf,wBAAwB,IACxBY,iBACA;oBACEV,cAAc,IAAI;oBAClBA,cAAc,cAAc,IAAI;iBACjC;gBAGL,MAAMc,aAAa;oBACjB,GAAGF,UAAU;oBACb,QAAQD;gBACV;gBAEAtH,MACE,CAAC,eAAe,EAAE2G,cAAc,IAAI,EAAE,EACtC,CAAC,YAAY,EAAE3E,KAAK,SAAS,CAACyF,YAAY,MAAM,IAAI;gBAEtD,MAAMvC,SAAS,MAAM9C,MAAM,uBAAuB,CAChDuE,cAAc,IAAI,EAClBc;gBAIF,MAAMR,aAAcvE,SAAiB,IAAI;gBACzC,IAAIwC,AAAWjD,WAAXiD,QACF,IAAI,CAAC,SAAS,CAAC+B,YAAY/B;YAE/B;QACF;IACF;IAEA,MAAM,MAAM;QACV,MAAM,EAAEwC,MAAM,EAAEC,GAAG,EAAEC,OAAO,EAAEC,GAAG,EAAEC,OAAO,EAAEC,QAAQ,EAAEC,KAAK,EAAE,GAAG,IAAI,CAAC,MAAM;QAC3E,MAAMC,SAASN,OAAOD;QACtB,MAAMQ,aAAaN;QACnB,MAAMO,SAASN;QACf,MAAMO,aAAaN;QACnB,MAAMO,cAAcN;QACpB,MAAMO,WACJL,UAAUC,cAAcC,UAAUC,cAAcC;QAElD,IAAI,CAAC,eAAe,CAAC;QAErB,IAAIjG,QAAsB;QAC1B,IAAImG,SAAmB,EAAE;QACzB,IAAI;YACF,MAAM,EAAE,OAAOC,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,UAAU,CAClEH;YAEF,IAAI,CAAC,WAAW,GAAG,MAAME,SAAS,cAAc;YAChDpG,QAAQoG;YACR,MAAME,yBAAyBtG,MAAM,cAAc;YACnDA,MAAM,cAAc,GAAG,CAACuG;gBACtB,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,cAAc,GAAGA;gBAExBD,yBAAyBC;YAC3B;YACAJ,SAAS;mBACHE,aAAa,EAAE;gBACnB;oBACE,MAAM;oBACN,IAAI;wBACF,IAAIrG,OACFA,MAAM,cAAc,GAAGsG;oBAE3B;gBACF;aACD;QACH,EAAE,OAAOE,GAAG;YACV,IAAI,CAAC,eAAe,CAAC,SAASA;YAC9B;QACF;QACA,IAAI,CAAC,cAAc,GAAGxG;QAEtB,IAAIjB,YAAY;QAChB,IAAI,CAAC,eAAe,CAAC;QACrB,IAAI0H,YAAY;QAChB,MAAO1H,YAAY6G,MAAM,MAAM,CAAE;YAC/B,MAAM3G,aAAa,IAAI,CAAC,cAAc,CAACF,UAAU;YACjD,IAAI,CAAC,aAAa,CAACA,WAAW;YAC9B,IAAI,CAAC,YAAY,CAACA;YAClB,IAAI,CAAC,kCAAkC,GAAG;YAE1C,IAAI;gBACF,MAAM,IAAI,CAAC,QAAQ,CAACE,YAAY,IAAI,CAAC,cAAc;gBACnD,IAAI,CAAC,aAAa,CAACF,WAAW;YAChC,EAAE,OAAOyH,GAAG;gBACV,IAAI,CAAC,aAAa,CAACzH,WAAW,SAAgByH;gBAC9C,MAAME,sBAAuB1G,MAAc,mBAAmB;gBAC9D,IACE,CAAC,IAAI,CAAC,kCAAkC,IACxC,AAA+B,cAA/B,OAAO0G,qBAEP,IAAI;oBACF,MAAMA,oBAAoB,IAAI,CAC5B1G,OACA,CAAC,mBAAmB,EAAEf,WAAW,IAAI,EAAE,EACvC;wBACE,OAAOuH;wBACP,SAAS,CAAC,KAAK,EAAEvH,WAAW,WAAW,IAAI,EAAE,iCAAiC,EAAEA,WAAW,IAAI,CAAC,EAAE,CAAC;oBACrG;gBAEJ,EAAE,OAAO0H,aAAa;oBACpB/I,MAAM,yCAAyC+I;gBACjD;gBAGF,IAAI1H,WAAW,eAAe;qBAEvB;oBACL,IAAI,CAAC,UAAU,GAAGe,MAAM,UAAU;oBAClCyG,YAAY;oBACZ;gBACF;YACF;YACA,IAAI,CAAC,UAAU,GAAGzG,OAAO;YACzBjB;QACF;QAEA,IAAI0H,WACF,IAAI,CAAC,eAAe,CAAC;aAErB,IAAI,CAAC,eAAe,CAAC;QAEvB,IAAI,CAAC,cAAc,GAAG;QAGtB,KAAK,MAAMG,MAAMT,OACf,IAAI;YAEF,MAAMS,GAAG,EAAE;QAEb,EAAE,OAAOJ,GAAG,CAEZ;IAEJ;IAhtBA,YACUK,MAA0B,EAC1BC,UAGN,EACKC,kBAAiE,EACxEC,UAAmB,CACnB;;;;QAvBF,uBAAO,oBAAP;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAO,cAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,sBAAR;QACA,uBAAO,UAAP;QACA,uBAAO,sBAAP;QACA,uBAAO,gBAAP;QACA,uBAAQ,kBAAR;QACA,uBAAO,kBAAP;QACA,uBAAO,UAAP;QACA,uBAAQ,eAAR;QACA,uBAAQ,cAAR;QACA,uBAAQ,sCAAR;aAEUH,MAAM,GAANA;aACAC,UAAU,GAAVA;aAIDC,kBAAkB,GAAlBA;aApBF,cAAc,GAA6B,EAAE;aAC7C,MAAM,GAA4B;aAGjC,kBAAkB,GAAG;aAIrB,cAAc,GAAiB;aAG/B,WAAW,GAAmB,EAAE;aAEhC,kCAAkC,GAAG;QAU3C,IAAI,CAAC,UAAU,GAAGC;QAClB,IAAI,CAAC,MAAM,GAAG,CAAC;QACf,MAAMC,uBACJJ,OAAO,KAAK,EAAE,gBAAgBA,OAAO,KAAK,EAAE;QAE9C,IAAII,AAAyBpH,WAAzBoH,wBAAsCJ,OAAO,KAAK,EAAE;YACtD,IACEA,AAA8BhH,WAA9BgH,OAAO,KAAK,CAAC,YAAY,IACzBA,AAAiChH,WAAjCgH,OAAO,KAAK,CAAC,eAAe,EAE5BjI,QAAQ,IAAI,CACV;YAIJiI,OAAO,KAAK,CAAC,YAAY,GAAGI;QAC9B;QAEA,IAAI,CAAC,MAAM,GACTJ,OAAO,MAAM,IACbA,OAAO,GAAG,IACVA,OAAO,OAAO,IACdA,OAAO,GAAG,IACVA,OAAO,QAAQ,IACfA,OAAO,MAAM;QAEf,IAAIK,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAAE;YAC7B,IAAI,CAAC,MAAM,GAAGtH;YACdjC,MAAM;QACR,OAAO,IAAI,IAAI,CAAC,MAAM,EAAE,QAAQ;YAC9B,IAAI,CAAC,MAAM,GAAGyB,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQC,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM;YACvD1B,MAAM,mCAAmC,IAAI,CAAC,MAAM;QACtD,OAAO;YACL,MAAMwJ,aAAa,IAAI,CAAC,UAAU,GAC9BC,AAAAA,IAAAA,mCAAAA,QAAAA,AAAAA,EAAS,IAAI,CAAC,UAAU,EAAE,SAAS,OAAO,CAAC,eAAe,MAC1D;YACJ,IAAI,CAAC,MAAM,GAAGC,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACZC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB,GAAGH,WAAW,CAAC,EAAEI,KAAK,GAAG,GAAG,KAAK,CAAC;YAEpC5J,MAAM,iCAAiC,IAAI,CAAC,MAAM;QACpD;QAEA,IAAIsJ,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,EAC3B,IAAI,CAAC,kBAAkB,GAAGtH;aACrB,IAAI,AAA2C,YAA3C,OAAO,IAAI,CAAC,MAAM,EAAE,oBAC7B,IAAI,CAAC,kBAAkB,GAAGR,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EACxBC,QAAQ,GAAG,IACX,IAAI,CAAC,MAAM,CAAC,kBAAkB;aAE3B,IAAI,IAAI,CAAC,MAAM,EAAE,uBAAuB,MAC7C,IAAI,CAAC,kBAAkB,GAAGgI,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACxBC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB;QAIJ,IAAI,CAAC,cAAc,GAAIV,AAAAA,CAAAA,OAAO,KAAK,IAAI,EAAC,EAAG,GAAG,CAAC,CAAClG,MAAM5B,YAAe;gBACnE,GAAG4B,IAAI;gBACP,OAAO5B;gBACP,QAAQ;gBACR,YAAY4B,KAAK,IAAI,EAAE,UAAU;YACnC;IACF;AAyoBF"}

package/dist/types/agent/agent.d.ts

@@ -1,32 +1,19 @@
 import type { TUserPrompt } from '../ai-model/index';
 import Service from '../service/index';
-import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, type LocateOption, type LocateResultElement, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, type RecordToReportOptions, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type Size, type UIContext } from '../types';
+import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, type LocateOption, type LocateResultElement, type OnTaskStartTip, type RecordToReportOptions, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type UIContext } from '../types';
 import type { AbstractInterface } from '../device';
 import type { TaskRunner } from '../task-runner';
 import { ModelConfigManager } from '@midscene/shared/env';
+import { type RunGherkinScenarioOptions } from './run-gherkin-scenario';
 import { TaskCache } from './task-cache';
 import { TaskExecutor } from './tasks';
-export type DescribeElementCoordinateSpace = 'screenshot' | 'logical';
-export type DescribeElementAtPointOptions = {
-    verifyPrompt?: boolean;
-    retryLimit?: number;
-    deepLocate?: boolean;
-    screenshotBase64?: string;
-    screenshotSize?: Size;
-    coordinateSpace?: DescribeElementCoordinateSpace;
-    logicalSize?: Size;
-    onProgress?: (progress: {
-        prompt?: string;
-        deepLocate?: boolean;
-        verifyResult?: LocateValidatorResult;
-    }) => void;
-} & LocatorValidatorOption;
 export type AiActOptions = {
     cacheable?: boolean;
     fileChooserAccept?: string | string[];
     deepThink?: DeepThinkOption;
     deepLocate?: boolean;
     abortSignal?: AbortSignal;
+    context?: string;
 };
 export declare class Agent<InterfaceType extends AbstractInterface = AbstractInterface> {
     interface: InterfaceType;
@@ -148,6 +135,7 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     aiClearInput(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<void>;
     aiAct(taskPrompt: TUserPrompt, opt?: AiActOptions): Promise<string | undefined>;
     runMarkdown(markdownPath: string, opt?: AiActOptions): Promise<string | undefined>;
+    runGherkinScenario(scenarioText: string, opt?: RunGherkinScenarioOptions): Promise<void>;
     /**
      * @deprecated Use {@link Agent.aiAct} instead.
      */
@@ -157,8 +145,6 @@ export declare class Agent<InterfaceType extends AbstractInterface = AbstractInt
     aiNumber(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<number>;
     aiString(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
     aiAsk(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
-    describeElementAtPoint(center: [number, number], opt?: DescribeElementAtPointOptions): Promise<AgentDescribeElementAtPointResult>;
-    verifyLocator(prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption): Promise<LocateValidatorResult>;
     /**
      * Locate an element and return both its center point and an approximate rect.
      *

package/dist/types/agent/index.d.ts

@@ -1,10 +1,11 @@
 export { Agent, createAgent } from './agent';
 export { commonContextParser } from './utils';
-export { getReportFileName, printReportMsg, } from './utils';
+export { getReportFileName, printReportMsg } from './utils';
 export { extractInsightParam, locateParamStr, paramStr, taskTitleStr, typeStr, } from './ui-utils';
 export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
 export { cacheFileExt } from './task-cache';
 export { TaskExecutor } from './tasks';
+export type { GherkinStepKeyword, RunGherkinScenarioOptions, } from './run-gherkin-scenario';
 export type { AgentOpt } from '../types';
 export type { RecordToReportOptions, RecordToReportScreenshot } from '../types';
-export type { AiActOptions, DescribeElementAtPointOptions, DescribeElementCoordinateSpace, } from './agent';
+export type { AiActOptions } from './agent';

package/dist/types/agent/prompt-context.d.ts

@@ -0,0 +1,2 @@
+import type { TUserPrompt } from '../ai-model';
+export declare const buildPromptWithContext: (prompt: TUserPrompt, context: string | undefined) => TUserPrompt;

package/dist/types/agent/run-gherkin-scenario.d.ts

@@ -0,0 +1,26 @@
+import type { AiActOptions } from './agent';
+export type GherkinStepKeyword = 'Given' | 'When' | 'Then' | 'And' | 'But';
+export type RunGherkinScenarioOptions = AiActOptions & {
+    context?: string;
+};
+type GherkinPrimaryKeyword = 'Given' | 'When' | 'Then';
+type ParsedGherkinStep = {
+    keyword: GherkinStepKeyword;
+    effectiveKeyword: GherkinPrimaryKeyword;
+    text: string;
+    lineNumber: number;
+};
+type ParsedGherkinScenario = {
+    scenario?: string;
+    steps: ParsedGherkinStep[];
+};
+type GherkinScenarioAgent = {
+    aiAct: (taskPrompt: string, opt?: AiActOptions) => Promise<unknown>;
+    aiAssert: (assertion: string, msg?: string, opt?: {
+        context?: string;
+        abortSignal?: AbortSignal;
+    }) => Promise<unknown>;
+};
+export declare const parseGherkinScenario: (scenarioText: string) => ParsedGherkinScenario;
+export declare const runGherkinScenario: (agent: GherkinScenarioAgent, scenarioText: string, opt?: RunGherkinScenarioOptions) => Promise<void>;
+export {};

package/dist/types/agent/tasks.d.ts

@@ -97,7 +97,9 @@ export declare class TaskExecutor {
     private invalidateFailedCacheHitLocates;
     private runAction;
     private createTypeQueryTask;
-    createTypeQueryExecution<T>(type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert', demand: ServiceExtractParam, modelRuntime: ModelRuntime, opt?: ServiceExtractOption, multimodalPrompt?: TMultimodalPrompt): Promise<ExecutionResult<T>>;
+    createTypeQueryExecution<T>(type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert', demand: ServiceExtractParam, modelRuntime: ModelRuntime, opt?: ServiceExtractOption, multimodalPrompt?: TMultimodalPrompt, executionOptions?: {
+        abortSignal?: AbortSignal;
+    }): Promise<ExecutionResult<T>>;
     waitFor(assertion: TUserPrompt, opt: PlanningActionParamWaitFor, modelRuntime: ModelRuntime): Promise<ExecutionResult<void>>;
 }
 export declare function withFileChooser<T>(interfaceInstance: AbstractInterface, fileChooserAccept: string[] | undefined, action: () => Promise<T>): Promise<T>;

package/dist/types/ai-model/index.d.ts

@@ -9,8 +9,6 @@ export { generateRecorderSessionMetadata } from './prompt/recorder-metadata-gene
 export type { RecorderGeneratedMetadata, RecorderMetadataGenerationInput, } from './prompt/recorder-metadata-generator';
 export { generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
 export type { RecorderYamlGenerationInput, YamlGenerationOptions, } from './prompt/yaml-generator';
-export { describeRecorderUIEvent, describeRecorderUIEvents, getRecorderUIEventTargetRect, } from '../recorder-ui-describer';
-export type { DescribeRecorderUIEventInput, DescribeRecorderUIEventOptions, DescribeRecorderUIEventResult, } from '../recorder-ui-describer';
 export type { ChatCompletionMessageParam } from 'openai/resources/index';
 export { AiLocateElement, AiExtractElementInfo, AiLocateSection, AiJudgeOrderSensitive, } from './inspect';
 export { plan } from './llm-planning';

package/dist/types/ai-model/inspect.d.ts

@@ -2,7 +2,7 @@ import type { AIDataExtractionResponse, AIUsageInfo, Rect, ServiceExtractOption,
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
 import type { TMultimodalPrompt, TUserPrompt } from '../common';
 import type { ModelRuntime } from './models';
-import type { LocateOptions, LocateResult, SearchAreaConfig } from './workflows/inspect/types';
+import type { LocateModelResponse, LocateOptions, LocateRequestContext, LocateResult, SearchAreaConfig } from './workflows/inspect/types';
 export type InspectAIArgs = [
     ChatCompletionSystemMessageParam,
     ...ChatCompletionUserMessageParam[]
@@ -15,7 +15,7 @@ export declare function buildSearchAreaConfig(options: {
 export declare function AiLocateElement(options: LocateOptions & {
     targetElementDescription: TUserPrompt;
 }): Promise<LocateResult>;
-export declare function genericLocate(elementDescription: TUserPrompt, options: LocateOptions): Promise<LocateResult>;
+export declare function genericLocate(_elementDescription: TUserPrompt, options: LocateOptions, locateRequest: LocateRequestContext): Promise<LocateModelResponse>;
 export declare function AiLocateSection(options: {
     context: UIContext;
     sectionDescription: TUserPrompt;
@@ -35,6 +35,7 @@ export declare function AiExtractElementInfo<T>(options: {
     pageDescription?: string;
     extractOption?: ServiceExtractOption;
     modelRuntime: ModelRuntime;
+    abortSignal?: AbortSignal;
 }): Promise<{
     parseResult: AIDataExtractionResponse<T>;
     rawResponse: string;

package/dist/types/ai-model/model-adapter/chat-completion.d.ts

@@ -0,0 +1,3 @@
+import type { ChatCompletionAdapter, ChatCompletionContentSource, ContentAndReasoning, ModelAdapterDefinition } from './types';
+export declare function defaultExtractContentAndReasoning(message: ChatCompletionContentSource | undefined): ContentAndReasoning;
+export declare function resolveChatCompletion(chatCompletion: ModelAdapterDefinition['chatCompletion']): ChatCompletionAdapter;

package/dist/types/ai-model/model-adapter/custom-planning-types.d.ts

@@ -0,0 +1,32 @@
+import type { TUserPrompt } from '../../common';
+import type { PlanningAction } from '../../types';
+import type { LocateResultAdapter, LocateResultCoordinates, ResolvedLocateResultCoordinates } from '../shared/model-locate-result/types';
+import type { PlanOptions } from '../workflows/planning/types';
+export interface CustomPlanningInput {
+    userInstruction: TUserPrompt;
+    userInstructionText: string;
+    options: PlanOptions;
+    coordinateSystem?: ResolvedLocateResultCoordinates;
+}
+interface CustomPlanningLifecycle<TParsed = unknown> {
+    messages: CustomPlanningMessageConfig<TParsed>;
+    parseResponse(rawResponse: string, input: CustomPlanningInput): TParsed;
+    transformActions(parsed: TParsed, input: CustomPlanningInput): PlanningAction[];
+    shouldContinuePlanning(parsed: TParsed, actions: PlanningAction[]): boolean;
+    buildResponseLog(parsed: TParsed, rawResponse: string): string;
+}
+export interface CustomPlanningDefinition<TParsed = unknown> extends CustomPlanningLifecycle<TParsed> {
+    coordinates: LocateResultCoordinates;
+}
+export interface ResolvedCustomPlanningDefinition<TParsed = unknown> extends CustomPlanningLifecycle<TParsed> {
+    coordinateSystem: ResolvedLocateResultCoordinates;
+    coordinateNormalizer: LocateResultAdapter;
+}
+export interface CustomPlanningMessageConfig<TParsed = unknown> {
+    systemPromptPlacement: 'system-message' | 'user-message';
+    buildSystemPrompt(): string;
+    historyImageLimit?: number;
+    buildUserInstruction?: (userInstruction: string) => string;
+    buildAssistantContent?: (parsed: TParsed, rawResponse: string, input: CustomPlanningInput) => string | undefined;
+}
+export {};

package/dist/types/ai-model/model-adapter/index.d.ts

@@ -0,0 +1,2 @@
+export { ResolvedModelAdapter } from './resolve';
+export type { ModelRuntime } from './types';

package/dist/types/ai-model/model-adapter/locate.d.ts

@@ -0,0 +1,3 @@
+import type { ResolvedCustomPlanningDefinition } from './custom-planning-types';
+import type { LocateAdapter, ModelAdapterDefinition } from './types';
+export declare function resolveLocate(locate: ModelAdapterDefinition['locate'], resolvedCustomPlanner: ResolvedCustomPlanningDefinition | undefined): LocateAdapter;

package/dist/types/ai-model/model-adapter/planning.d.ts

@@ -0,0 +1,4 @@
+import type { CustomPlanningDefinition, ResolvedCustomPlanningDefinition } from './custom-planning-types';
+import type { ModelAdapterDefinition, PlanningAdapter } from './types';
+export declare function resolveCustomPlanningDefinition<TParsed>(config: CustomPlanningDefinition<TParsed>): ResolvedCustomPlanningDefinition<TParsed>;
+export declare function resolvePlanning(planning: ModelAdapterDefinition['planning'], resolvedCustomPlanner?: ResolvedCustomPlanningDefinition): PlanningAdapter;

package/dist/types/ai-model/{models → model-adapter}/types.d.ts

@@ -1,10 +1,12 @@
+import type { PixelBbox, PlanningAction } from '../../types';
 import type { IModelConfig, TIntent, TModelReasoningEnabled } from '@midscene/shared/env';
 import type OpenAI from 'openai';
 import type { JsonParser, JsonParserContext, JsonParserSource } from '../service-caller/json';
-import type { LocateResultAdapter, LocateResultAdapterDefinition } from '../shared/model-locate-result/types';
+import type { LocateResultAdapter, LocateResultAdapterDefinition, ResolvedLocateResultCoordinates } from '../shared/model-locate-result/types';
 import type { ImagePreprocessPolicy } from '../workflows/image-preprocess';
 import type { LocateFn } from '../workflows/inspect/types';
 import type { PlanFn } from '../workflows/planning/types';
+import type { CustomPlanningDefinition } from './custom-planning-types';
 export type { ImagePreprocessPolicy, JsonParser, JsonParserContext, JsonParserSource, };
 export type JsonParserPreset = 'lenient-json';
 export interface ReasoningInput {
@@ -60,6 +62,15 @@ export type ImagePreprocessDefinition = Partial<ImagePreprocessPolicy>;
 interface PlanningPolicy {
     cacheEnabled: boolean;
     defaultReplanningCycleLimit: number;
+    /**
+     * Whether aiAct can use planning action coordinates as the first-stage
+     * search area for deepLocate.
+     *
+     * Custom planning models may return only action coordinates without a target
+     * element description. Those results can be used as direct plan hits, but
+     * cannot drive deepLocate's second-stage locate call because that call also
+     * needs a query prompt describing the target element.
+     */
     supportsActionDeepLocate: boolean;
 }
 export type PlanningAdapter = (PlanningPolicy & {
@@ -67,14 +78,29 @@ export type PlanningAdapter = (PlanningPolicy & {
 }) | (PlanningPolicy & {
     kind: 'custom';
     planFn: PlanFn;
+    coordinateSystem?: ResolvedLocateResultCoordinates;
 });
 export type PlanningDefinition = (Partial<PlanningPolicy> & {
     kind?: 'standard';
-}) | (Partial<PlanningPolicy> & {
+}) | (Partial<PlanningPolicy> & ({
+    kind: 'custom';
+    planner: CustomPlanningDefinition<any>;
+    planFn?: never;
+} | {
     kind: 'custom';
     planFn: PlanFn;
-});
+    planner?: never;
+}));
 interface LocatePolicy {
+    /**
+     * Whether the locate adapter supports finding a coarse search area before the
+     * final element locate step.
+     *
+     * Some custom model families provide their own planning flow but do not
+     * support standalone locate/section-locate. They cannot behave like standard
+     * deepLocate, where a reference element is first located to build the search
+     * area for the final locate call.
+     */
     supportsSearchArea: boolean;
 }
 type StandardLocateAdapter = LocatePolicy & {
@@ -90,10 +116,19 @@ type StandardLocateDefinition = Partial<LocatePolicy> & {
     kind?: 'standard';
     resultAdapter?: LocateResultAdapterDefinition;
 };
+export interface PlanningTapLocatorDefinition {
+    buildSystemPrompt(): string;
+    getLocatedPixelBbox(actions: PlanningAction[]): PixelBbox | undefined;
+}
 type CustomLocateDefinition = Partial<LocatePolicy> & {
     kind: 'custom';
+} & ({
     locateFn: LocateFn;
-};
+    planningTapLocator?: never;
+} | {
+    planningTapLocator: PlanningTapLocatorDefinition;
+    locateFn?: never;
+});
 export type LocateDefinition = StandardLocateDefinition | CustomLocateDefinition;
 export interface ModelAdapter {
     jsonParser: JsonParser;

package/dist/types/ai-model/models/auto-glm/actions.d.ts

@@ -1,5 +1,7 @@
 import type { DeviceAction } from '../../../device';
 import type { PlanningAction } from '../../../types';
+import type { CoordinateDistanceAxis } from '../../shared/model-locate-result';
+type CoordinateDistanceToPixels = (delta: number, axis: CoordinateDistanceAxis) => number;
 export interface BaseAction {
     _metadata: string;
     think?: string;
@@ -71,8 +73,9 @@ export interface FinishAction extends BaseAction {
     _metadata: 'finish';
     message: string;
 }
-export type ParsedAction = TapAction | DoubleTapAction | TypeAction | SwipeAction | LongPressAction | LaunchAction | BackAction | HomeAction | WaitAction | InteractAction | CallAPIAction | TakeoverAction | NoteAction | FinishAction;
-export declare function transformAutoGLMAction(action: ParsedAction, size: {
-    width: number;
-    height: number;
-}, actionSpace?: DeviceAction[]): PlanningAction[];
+export type AutoGLMParsedAction = TapAction | DoubleTapAction | TypeAction | SwipeAction | LongPressAction | LaunchAction | BackAction | HomeAction | WaitAction | InteractAction | CallAPIAction | TakeoverAction | NoteAction | FinishAction;
+export declare function transformAutoGLMAction(action: AutoGLMParsedAction, { actionSpace, coordinateDistanceToPixels, }: {
+    actionSpace?: DeviceAction[];
+    coordinateDistanceToPixels: CoordinateDistanceToPixels;
+}): PlanningAction[];
+export {};

package/dist/types/ai-model/models/auto-glm/adapter.d.ts

@@ -1,4 +1,4 @@
-import type { ModelAdapterDefinition } from '../types';
+import type { ModelAdapterDefinition } from '../../model-adapter/types';
 export declare const autoGlmAdapters: {
     'auto-glm': ModelAdapterDefinition;
     'auto-glm-multilingual': ModelAdapterDefinition;

package/dist/types/ai-model/models/auto-glm/locate.d.ts

@@ -1,3 +1,2 @@
-import type { TUserPrompt } from '../../../common';
-import type { LocateOptions, LocateResult } from '../../workflows/inspect/types';
-export declare function autoGlmLocate(elementDescription: TUserPrompt, options: LocateOptions, getSystemPrompt: () => string): Promise<LocateResult>;
+import type { PlanningTapLocatorDefinition } from '../../model-adapter/types';
+export declare function createAutoGlmPlanningTapLocator(isMultilingual: boolean): PlanningTapLocatorDefinition;

package/dist/types/ai-model/models/auto-glm/parser.d.ts

@@ -1,18 +1,14 @@
-import type { ParsedAction } from './actions';
+import type { AutoGLMParsedAction } from './actions';
 export declare const extractValueAfter: (src: string, key: string) => string;
-export declare function parseAction(response: {
+export declare function parseAutoGLMPlanningAction(response: {
     think: string;
     content: string;
-}): ParsedAction;
+}): AutoGLMParsedAction;
 export declare function parseAutoGLMResponse(content: string): {
     think: string;
     content: string;
 };
-export declare function parseAutoGLMLocateResponse(rawResponse: string): {
-    think: string;
-    coordinates: {
-        x: number;
-        y: number;
-    } | null;
-    error?: string;
+export declare function parseAutoGLMPlanningResponse(content: string): {
+    response: ReturnType<typeof parseAutoGLMResponse>;
+    action: AutoGLMParsedAction;
 };