@midscene/core 0.30.5 → 1.0.1-beta-20251021060907.0

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["import {\n  ConversationHistory,\n  findAllMidsceneLocatorField,\n  parseActionParam,\n  plan,\n  uiTarsPlanning,\n} from '@/ai-model';\nimport { Executor } from '@/ai-model/action-executor';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type Insight from '@/insight';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DumpSubscriber,\n  ElementCacheFeature,\n  ExecutionRecorderItem,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskInsightLocateApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  ExecutorContext,\n  InsightDump,\n  InsightExtractOption,\n  InsightExtractParam,\n  InterfaceType,\n  LocateResultElement,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamError,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n  PlanningLocateParam,\n} from '@/types';\nimport { sleep } from '@/utils';\nimport {\n  type IModelConfig,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { taskTitleStr } from './ui-utils';\nimport {\n  matchElementFromCache,\n  matchElementFromPlan,\n  parsePrompt,\n} from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  executor: Executor;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    locate,\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  insight: Insight;\n\n  taskCache?: TaskCache;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    insight: Insight,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.insight = insight;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.conversationHistory = new ConversationHistory();\n  }\n\n  private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {\n    const base64 = await this.interface.screenshotBase64();\n    const item: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot: base64,\n      timing,\n    };\n    return item;\n  }\n\n  private prependExecutorWithScreenshot(\n    taskApply: ExecutionTaskApply,\n    appendAfterExecution = false,\n  ): ExecutionTaskApply {\n    const taskWithScreenshot: ExecutionTaskApply = {\n      ...taskApply,\n      executor: async (param, context, ...args) => {\n        const recorder: ExecutionRecorderItem[] = [];\n        const { task } = context;\n        // set the recorder before executor in case of error\n        task.recorder = recorder;\n        const shot = await this.recordScreenshot(`before ${task.type}`);\n        recorder.push(shot);\n\n        const result = await taskApply.executor(param, context, ...args);\n\n        if (appendAfterExecution) {\n          const shot2 = await this.recordScreenshot('after Action');\n          recorder.push(shot2);\n        }\n        return result;\n      },\n    };\n    return taskWithScreenshot;\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    cacheable?: boolean,\n  ) {\n    const tasks: ExecutionTaskApply[] = [];\n\n    const taskForLocatePlan = (\n      plan: PlanningAction<PlanningLocateParam>,\n      detailedLocateParam: DetailedLocateParam | string,\n      onResult?: (result: LocateResultElement) => void,\n    ): ExecutionTaskInsightLocateApply => {\n      if (typeof detailedLocateParam === 'string') {\n        detailedLocateParam = {\n          prompt: detailedLocateParam,\n        };\n      }\n      // Apply cacheable option from convertPlanToExecutable if it was explicitly set\n      if (cacheable !== undefined) {\n        detailedLocateParam = {\n          ...detailedLocateParam,\n          cacheable,\n        };\n      }\n      const taskFind: ExecutionTaskInsightLocateApply = {\n        type: 'Insight',\n        subType: 'Locate',\n        param: detailedLocateParam,\n        thought: plan.thought,\n        executor: async (param, taskContext) => {\n          const { task } = taskContext;\n          assert(\n            param?.prompt || param?.id || param?.bbox,\n            `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n              param,\n            )}`,\n          );\n          let insightDump: InsightDump | undefined;\n          let usage: AIUsageInfo | undefined;\n          const dumpCollector: DumpSubscriber = (dump) => {\n            insightDump = dump;\n            usage = dump?.taskInfo?.usage;\n\n            task.log = {\n              dump: insightDump,\n            };\n\n            task.usage = usage;\n\n            // Store searchAreaUsage in task metadata\n            if (dump?.taskInfo?.searchAreaUsage) {\n              task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n            }\n          };\n          this.insight.onceDumpUpdatedFn = dumpCollector;\n          const shotTime = Date.now();\n\n          // Get context through contextRetrieverFn which handles frozen context\n          const uiContext = await this.insight.contextRetrieverFn('locate');\n          task.uiContext = uiContext;\n\n          const recordItem: ExecutionRecorderItem = {\n            type: 'screenshot',\n            ts: shotTime,\n            screenshot: uiContext.screenshotBase64,\n            timing: 'before Insight',\n          };\n          task.recorder = [recordItem];\n\n          // try matching xpath\n          const elementFromXpath =\n            param.xpath && (this.interface as any).getElementInfoByXpath\n              ? await (this.interface as any).getElementInfoByXpath(param.xpath)\n              : undefined;\n          const userExpectedPathHitFlag = !!elementFromXpath;\n\n          // try matching cache\n          const cachePrompt = param.prompt;\n          const locateCacheRecord =\n            this.taskCache?.matchLocateCache(cachePrompt);\n          const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n          const elementFromCache = userExpectedPathHitFlag\n            ? null\n            : await matchElementFromCache(\n                this,\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n          const cacheHitFlag = !!elementFromCache;\n\n          // try matching plan\n          const elementFromPlan =\n            !userExpectedPathHitFlag && !cacheHitFlag\n              ? matchElementFromPlan(param, uiContext.tree)\n              : undefined;\n          const planHitFlag = !!elementFromPlan;\n\n          // try ai locate\n          const elementFromAiLocate =\n            !userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag\n              ? (\n                  await this.insight.locate(\n                    param,\n                    {\n                      // fallback to ai locate\n                      context: uiContext,\n                    },\n                    modelConfig,\n                  )\n                ).element\n              : undefined;\n          const aiLocateHitFlag = !!elementFromAiLocate;\n\n          const element =\n            elementFromXpath || // highest priority\n            elementFromCache || // second priority\n            elementFromPlan || // third priority\n            elementFromAiLocate;\n\n          // update cache\n          let currentCacheEntry: ElementCacheFeature | undefined;\n          if (\n            element &&\n            this.taskCache &&\n            !cacheHitFlag &&\n            param?.cacheable !== false\n          ) {\n            if (this.interface.cacheFeatureForRect) {\n              try {\n                const feature = await this.interface.cacheFeatureForRect(\n                  element.rect,\n                  element.isOrderSensitive !== undefined\n                    ? { _orderSensitive: element.isOrderSensitive }\n                    : undefined,\n                );\n                if (feature && Object.keys(feature).length > 0) {\n                  debug(\n                    'update cache, prompt: %s, cache: %o',\n                    cachePrompt,\n                    feature,\n                  );\n                  currentCacheEntry = feature;\n                  this.taskCache.updateOrAppendCacheRecord(\n                    {\n                      type: 'locate',\n                      prompt: cachePrompt,\n                      cache: feature,\n                    },\n                    locateCacheRecord,\n                  );\n                } else {\n                  debug(\n                    'no cache data returned, skip cache update, prompt: %s',\n                    cachePrompt,\n                  );\n                }\n              } catch (error) {\n                debug('cacheFeatureForRect failed: %s', error);\n              }\n            } else {\n              debug('cacheFeatureForRect is not supported, skip cache update');\n            }\n          }\n          if (!element) {\n            throw new Error(`Element not found: ${param.prompt}`);\n          }\n\n          let hitBy: ExecutionTaskHitBy | undefined;\n\n          if (userExpectedPathHitFlag) {\n            hitBy = {\n              from: 'User expected path',\n              context: {\n                xpath: param.xpath,\n              },\n            };\n          } else if (cacheHitFlag) {\n            hitBy = {\n              from: 'Cache',\n              context: {\n                cacheEntry,\n                cacheToSave: currentCacheEntry,\n              },\n            };\n          } else if (planHitFlag) {\n            hitBy = {\n              from: 'Planning',\n              context: {\n                id: elementFromPlan?.id,\n                bbox: elementFromPlan?.bbox,\n              },\n            };\n          } else if (aiLocateHitFlag) {\n            hitBy = {\n              from: 'AI model',\n              context: {\n                prompt: param.prompt,\n              },\n            };\n          }\n\n          onResult?.(element);\n\n          return {\n            output: {\n              element,\n            },\n            uiContext,\n            hitBy,\n          };\n        },\n      };\n      return taskFind;\n    };\n\n    for (const plan of plans) {\n      if (plan.type === 'Locate') {\n        if (\n          !plan.locate ||\n          plan.locate === null ||\n          plan.locate?.id === null ||\n          plan.locate?.id === 'null'\n        ) {\n          debug('Locate action with id is null, will be ignored', plan);\n          continue;\n        }\n        const taskLocate = taskForLocatePlan(plan, plan.locate);\n\n        tasks.push(taskLocate);\n      } else if (plan.type === 'Error') {\n        const taskActionError: ExecutionTaskActionApply<PlanningActionParamError> =\n          {\n            type: 'Action',\n            subType: 'Error',\n            param: plan.param,\n            thought: plan.thought || plan.param?.thought,\n            locate: plan.locate,\n            executor: async () => {\n              throw new Error(\n                plan?.thought || plan.param?.thought || 'error without thought',\n              );\n            },\n          };\n        tasks.push(taskActionError);\n      } else if (plan.type === 'Finished') {\n        const taskActionFinished: ExecutionTaskActionApply<null> = {\n          type: 'Action',\n          subType: 'Finished',\n          param: null,\n          thought: plan.thought,\n          locate: plan.locate,\n          executor: async (param) => {},\n        };\n        tasks.push(taskActionFinished);\n      } else if (plan.type === 'Sleep') {\n        const taskActionSleep: ExecutionTaskActionApply<PlanningActionParamSleep> =\n          {\n            type: 'Action',\n            subType: 'Sleep',\n            param: plan.param,\n            thought: plan.thought,\n            locate: plan.locate,\n            executor: async (taskParam) => {\n              await sleep(taskParam?.timeMs || 3000);\n            },\n          };\n        tasks.push(taskActionSleep);\n      } else {\n        // action in action space\n        const planType = plan.type;\n        const actionSpace = await this.interface.actionSpace();\n        const action = actionSpace.find((action) => action.name === planType);\n        const param = plan.param;\n\n        if (!action) {\n          throw new Error(`Action type '${planType}' not found`);\n        }\n\n        // find all params that needs location\n        const locateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema)\n          : [];\n\n        const requiredLocateFields = action\n          ? findAllMidsceneLocatorField(action.paramSchema, true)\n          : [];\n\n        locateFields.forEach((field) => {\n          if (param[field]) {\n            const locatePlan = locatePlanForLocate(param[field]);\n            debug(\n              'will prepend locate param for field',\n              `action.type=${planType}`,\n              `param=${JSON.stringify(param[field])}`,\n              `locatePlan=${JSON.stringify(locatePlan)}`,\n            );\n            const locateTask = taskForLocatePlan(\n              locatePlan,\n              param[field],\n              (result) => {\n                param[field] = result;\n              },\n            );\n            tasks.push(locateTask);\n          } else {\n            assert(\n              !requiredLocateFields.includes(field),\n              `Required locate field '${field}' is not provided for action ${planType}`,\n            );\n            debug(`field '${field}' is not provided for action ${planType}`);\n          }\n        });\n\n        const task: ExecutionTaskApply<\n          'Action',\n          any,\n          { success: boolean; action: string; param: any },\n          void\n        > = {\n          type: 'Action',\n          subType: planType,\n          thought: plan.thought,\n          param: plan.param,\n          executor: async (param, context) => {\n            debug(\n              'executing action',\n              planType,\n              param,\n              `context.element.center: ${context.element?.center}`,\n            );\n\n            // Get context for actionSpace operations to ensure size info is available\n            const uiContext = await this.insight.contextRetrieverFn('locate');\n            context.task.uiContext = uiContext;\n\n            requiredLocateFields.forEach((field) => {\n              assert(\n                param[field],\n                `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n              );\n            });\n\n            try {\n              await Promise.all([\n                (async () => {\n                  if (this.interface.beforeInvokeAction) {\n                    debug('will call \"beforeInvokeAction\" for interface');\n                    await this.interface.beforeInvokeAction(action.name, param);\n                    debug('called \"beforeInvokeAction\" for interface');\n                  }\n                })(),\n                sleep(200),\n              ]);\n            } catch (originalError: any) {\n              const originalMessage =\n                originalError?.message || String(originalError);\n              throw new Error(\n                `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n                { cause: originalError },\n              );\n            }\n\n            // Validate and parse parameters with defaults\n            if (action.paramSchema) {\n              try {\n                param = parseActionParam(param, action.paramSchema);\n              } catch (error: any) {\n                throw new Error(\n                  `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n                  { cause: error },\n                );\n              }\n            }\n\n            debug('calling action', action.name);\n            const actionFn = action.call.bind(this.interface);\n            await actionFn(param, context);\n            debug('called action', action.name);\n\n            try {\n              if (this.interface.afterInvokeAction) {\n                debug('will call \"afterInvokeAction\" for interface');\n                await this.interface.afterInvokeAction(action.name, param);\n                debug('called \"afterInvokeAction\" for interface');\n              }\n            } catch (originalError: any) {\n              const originalMessage =\n                originalError?.message || String(originalError);\n              throw new Error(\n                `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n                { cause: originalError },\n              );\n            }\n            // Return a proper result for report generation\n            return {\n              output: {\n                success: true,\n                action: planType,\n                param: param,\n              },\n            };\n          },\n        };\n        tasks.push(task);\n      }\n    }\n\n    const wrappedTasks = tasks.map(\n      (task: ExecutionTaskApply, index: number) => {\n        if (task.type === 'Action') {\n          return this.prependExecutorWithScreenshot(\n            task,\n            index === tasks.length - 1,\n          );\n        }\n        return task;\n      },\n    );\n\n    return {\n      tasks: wrappedTasks,\n    };\n  }\n\n  private async setupPlanningContext(executorContext: ExecutorContext) {\n    const shotTime = Date.now();\n    const uiContext = await this.insight.contextRetrieverFn('locate');\n    const recordItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: shotTime,\n      screenshot: uiContext.screenshotBase64,\n      timing: 'before Planning',\n    };\n\n    executorContext.task.recorder = [recordItem];\n    (executorContext.task as ExecutionTaskPlanning).uiContext = uiContext;\n\n    return {\n      uiContext,\n    };\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const taskExecutor = new Executor(taskTitleStr('Action', userInstruction), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        await this.setupPlanningContext(executorContext);\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n\n    await taskExecutor.append(task);\n    await taskExecutor.flush();\n\n    return {\n      executor: taskExecutor,\n    };\n  }\n\n  private createPlanningTask(\n    userInstruction: string,\n    actionContext: string | undefined,\n    modelConfig: IModelConfig,\n  ): ExecutionTaskPlanningApply {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { uiContext } = await this.setupPlanningContext(executorContext);\n        const { vlMode } = modelConfig;\n        const uiTarsModelVersion =\n          vlMode === 'vlm-ui-tars' ? modelConfig.uiTarsModelVersion : undefined;\n\n        assert(\n          this.interface.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.interface.actionSpace();\n        debug(\n          'actionSpace for this interface is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await (uiTarsModelVersion ? uiTarsPlanning : plan)(\n          param.userInstruction,\n          {\n            context: uiContext,\n            actionContext,\n            interfaceType: this.interface.interfaceType as InterfaceType,\n            actionSpace,\n            modelConfig,\n            conversationHistory: this.conversationHistory,\n          },\n        );\n        debug('planResult', JSON.stringify(planResult, null, 2));\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push({\n              type: 'Sleep',\n              param: {\n                timeMs: timeRemaining,\n              },\n              locate: null,\n            } as PlanningAction<PlanningActionParamSleep>);\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          uiContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const taskExecutor = new Executor(title, {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { tasks } = await this.convertPlanToExecutable(plans, modelConfig);\n    await taskExecutor.append(tasks);\n    const result = await taskExecutor.flush();\n    const { output } = result!;\n    return {\n      output,\n      executor: taskExecutor,\n    };\n  }\n\n  private getReplanningCycleLimit(isVlmUiTars: boolean) {\n    return (\n      this.replanningCycleLimit ||\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) ||\n      (isVlmUiTars\n        ? defaultVlmUiTarsReplanningCycleLimit\n        : defaultReplanningCycleLimit)\n    );\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfig: IModelConfig,\n    actionContext?: string,\n    cacheable?: boolean,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit = this.getReplanningCycleLimit(\n      modelConfig.vlMode === 'vlm-ui-tars',\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanning ${replanningCycleLimit} times, which is more than the limit, please split the task into multiple steps`;\n\n        return this.appendErrorPlan(taskExecutor, errorMsg, modelConfig);\n      }\n\n      // Create planning task (automatically includes execution history if available)\n      const planningTask = this.createPlanningTask(\n        userPrompt,\n        actionContext,\n        modelConfig,\n      );\n\n      await taskExecutor.append(planningTask);\n      const result = await taskExecutor.flush();\n      const planResult: PlanningAIResponse = result?.output;\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: planResult,\n          executor: taskExecutor,\n        };\n      }\n\n      // Execute planned actions\n      const plans = planResult.actions || [];\n      yamlFlow.push(...(planResult.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          modelConfig,\n          cacheable,\n        );\n        taskExecutor.append(executables.tasks);\n      } catch (error) {\n        return this.appendErrorPlan(\n          taskExecutor,\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n          modelConfig,\n        );\n      }\n\n      await taskExecutor.flush();\n      if (taskExecutor.isInErrorState()) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      // Check if task is complete\n      if (!planResult.more_actions_needed_by_instruction) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      replanCount++;\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      executor: taskExecutor,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let insightDump: InsightDump | undefined;\n        const dumpCollector: DumpSubscriber = (dump) => {\n          insightDump = dump;\n        };\n        this.insight.onceDumpUpdatedFn = dumpCollector;\n\n        // Get context for query operations\n        const shotTime = Date.now();\n        const uiContext = await this.insight.contextRetrieverFn('extract');\n        task.uiContext = uiContext;\n\n        const recordItem: ExecutionRecorderItem = {\n          type: 'screenshot',\n          ts: shotTime,\n          screenshot: uiContext.screenshotBase64,\n          timing: 'before Extract',\n        };\n        task.recorder = [recordItem];\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        const { data, usage, thought } = await this.insight.extract<any>(\n          demandInput,\n          modelConfig,\n          opt,\n          multimodalPrompt,\n        );\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else {\n            assert(\n              type !== 'WaitFor' ? data?.[keyOfResult] !== undefined : true,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        return {\n          output: outputResult,\n          log: insightDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const taskExecutor = new Executor(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      {\n        onTaskStart: this.onTaskStartCallback,\n      },\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n    const result = await taskExecutor.flush();\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      executor: taskExecutor,\n    };\n  }\n\n  private async appendErrorPlan(\n    taskExecutor: Executor,\n    errorMsg: string,\n    modelConfig: IModelConfig,\n  ) {\n    const errorPlan: PlanningAction<PlanningActionParamError> = {\n      type: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      locate: null,\n    };\n    const { tasks } = await this.convertPlanToExecutable(\n      [errorPlan],\n      modelConfig,\n    );\n    await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));\n    await taskExecutor.flush();\n\n    return {\n      output: undefined,\n      executor: taskExecutor,\n    };\n  }\n\n  async taskForSleep(timeMs: number, modelConfig: IModelConfig) {\n    const sleepPlan: PlanningAction<PlanningActionParamSleep> = {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n      locate: null,\n    };\n    // The convertPlanToExecutable requires modelConfig as a parameter but will not consume it when type is Sleep\n    const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n      [sleepPlan],\n      modelConfig,\n    );\n\n    return this.prependExecutorWithScreenshot(sleepTasks[0]);\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const taskExecutor = new Executor(taskTitleStr('WaitFor', description), {\n      onTaskStart: this.onTaskStartCallback,\n    });\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let startTime = Date.now();\n    let errorThought = '';\n    while (Date.now() - overallStartTime < timeoutMs) {\n      startTime = Date.now();\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        {\n          doNotThrowError: true,\n        },\n        multimodalPrompt,\n      );\n\n      await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));\n      const result = (await taskExecutor.flush()) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          executor: taskExecutor,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - startTime < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - startTime);\n        const sleepTask = await this.taskForSleep(timeRemaining, modelConfig);\n        await taskExecutor.append(sleepTask);\n      }\n    }\n\n    return this.appendErrorPlan(\n      taskExecutor,\n      `waitFor timeout: ${errorThought}`,\n      modelConfig,\n    );\n  }\n}\n"],"names":["debug","getDebug","defaultReplanningCycleLimit","defaultVlmUiTarsReplanningCycleLimit","locatePlanForLocate","param","locate","locatePlan","TaskExecutor","timing","base64","item","Date","taskApply","appendAfterExecution","taskWithScreenshot","context","args","recorder","task","shot","result","shot2","plans","modelConfig","cacheable","tasks","taskForLocatePlan","plan","detailedLocateParam","onResult","undefined","taskFind","taskContext","_this_taskCache","_locateCacheRecord_cacheContent","assert","JSON","insightDump","usage","dumpCollector","dump","_dump_taskInfo","_dump_taskInfo1","shotTime","uiContext","recordItem","elementFromXpath","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","cacheEntry","elementFromCache","matchElementFromCache","cacheHitFlag","elementFromPlan","matchElementFromPlan","planHitFlag","elementFromAiLocate","aiLocateHitFlag","element","currentCacheEntry","feature","Object","error","Error","hitBy","_plan_locate","_plan_locate1","taskLocate","_plan_param","taskActionError","taskActionFinished","taskActionSleep","taskParam","sleep","planType","actionSpace","action","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","locateTask","_context_element","Promise","originalError","originalMessage","String","parseActionParam","actionFn","wrappedTasks","index","executorContext","userInstruction","yamlString","taskExecutor","Executor","taskTitleStr","actionContext","startTime","vlMode","uiTarsModelVersion","Array","console","planResult","uiTarsPlanning","actions","log","more_actions_needed_by_instruction","rawResponse","finalActions","timeNow","timeRemaining","title","output","isVlmUiTars","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","userPrompt","replanCount","yamlFlow","replanningCycleLimit","errorMsg","planningTask","executables","type","demand","opt","multimodalPrompt","queryTask","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","data","thought","outputResult","errorPlan","timeMs","sleepPlan","sleepTasks","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","errorThought","now","sleepTask","interfaceInstance","insight","opts","ConversationHistory"],"mappings":";;;;;;;;;;;;;;;;;;AA6DA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,8BAA8B;AACpC,MAAMC,uCAAuC;AAEtC,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACND;QACA,OAAOA;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAEO,MAAMC;IAcX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAmBA,MAAc,iBAAiBC,MAAuC,EAAE;QACtE,MAAMC,SAAS,MAAM,IAAI,CAAC,SAAS,CAAC,gBAAgB;QACpD,MAAMC,OAA8B;YAClC,MAAM;YACN,IAAIC,KAAK,GAAG;YACZ,YAAYF;YACZD;QACF;QACA,OAAOE;IACT;IAEQ,8BACNE,SAA6B,EAC7BC,uBAAuB,KAAK,EACR;QACpB,MAAMC,qBAAyC;YAC7C,GAAGF,SAAS;YACZ,UAAU,OAAOR,OAAOW,SAAS,GAAGC;gBAClC,MAAMC,WAAoC,EAAE;gBAC5C,MAAM,EAAEC,IAAI,EAAE,GAAGH;gBAEjBG,KAAK,QAAQ,GAAGD;gBAChB,MAAME,OAAO,MAAM,IAAI,CAAC,gBAAgB,CAAC,CAAC,OAAO,EAAED,KAAK,IAAI,EAAE;gBAC9DD,SAAS,IAAI,CAACE;gBAEd,MAAMC,SAAS,MAAMR,UAAU,QAAQ,CAACR,OAAOW,YAAYC;gBAE3D,IAAIH,sBAAsB;oBACxB,MAAMQ,QAAQ,MAAM,IAAI,CAAC,gBAAgB,CAAC;oBAC1CJ,SAAS,IAAI,CAACI;gBAChB;gBACA,OAAOD;YACT;QACF;QACA,OAAON;IACT;IAEA,MAAa,wBACXQ,KAAuB,EACvBC,WAAyB,EACzBC,SAAmB,EACnB;QACA,MAAMC,QAA8B,EAAE;QAEtC,MAAMC,oBAAoB,CACxBC,MACAC,qBACAC;YAEA,IAAI,AAA+B,YAA/B,OAAOD,qBACTA,sBAAsB;gBACpB,QAAQA;YACV;YAGF,IAAIJ,AAAcM,WAAdN,WACFI,sBAAsB;gBACpB,GAAGA,mBAAmB;gBACtBJ;YACF;YAEF,MAAMO,WAA4C;gBAChD,MAAM;gBACN,SAAS;gBACT,OAAOH;gBACP,SAASD,KAAK,OAAO;gBACrB,UAAU,OAAOvB,OAAO4B;wBAkDpBC,iBACiBC;oBAlDnB,MAAM,EAAEhB,IAAI,EAAE,GAAGc;oBACjBG,OACE/B,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,EAAE,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GACxC,CAAC,qDAAqD,EAAEgC,KAAK,SAAS,CACpEhC,QACC;oBAEL,IAAIiC;oBACJ,IAAIC;oBACJ,MAAMC,gBAAgC,CAACC;4BAE7BC,gBASJC;wBAVJL,cAAcG;wBACdF,QAAQG,QAAAA,OAAAA,KAAAA,IAAAA,QAAAA,CAAAA,iBAAAA,KAAM,QAAQ,AAAD,IAAbA,KAAAA,IAAAA,eAAgB,KAAK;wBAE7BvB,KAAK,GAAG,GAAG;4BACT,MAAMmB;wBACR;wBAEAnB,KAAK,KAAK,GAAGoB;wBAGb,IAAII,QAAAA,OAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,KAAM,QAAQ,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,eAAe,EACjCxB,KAAK,eAAe,GAAGsB,KAAK,QAAQ,CAAC,eAAe;oBAExD;oBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGD;oBACjC,MAAMI,WAAWhC,KAAK,GAAG;oBAGzB,MAAMiC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACxD1B,KAAK,SAAS,GAAG0B;oBAEjB,MAAMC,aAAoC;wBACxC,MAAM;wBACN,IAAIF;wBACJ,YAAYC,UAAU,gBAAgB;wBACtC,QAAQ;oBACV;oBACA1B,KAAK,QAAQ,GAAG;wBAAC2B;qBAAW;oBAG5B,MAAMC,mBACJ1C,MAAM,KAAK,IAAK,IAAI,CAAC,SAAS,CAAS,qBAAqB,GACxD,MAAO,IAAI,CAAC,SAAS,CAAS,qBAAqB,CAACA,MAAM,KAAK,IAC/D0B;oBACN,MAAMiB,0BAA0B,CAAC,CAACD;oBAGlC,MAAME,cAAc5C,MAAM,MAAM;oBAChC,MAAM6C,oBAAAA,QACJhB,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CAACe;oBACnC,MAAME,aAAahB,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,KAAK;oBACzD,MAAMiB,mBAAmBJ,0BACrB,OACA,MAAMK,sBACJ,IAAI,EACJF,YACAF,aACA5C,MAAM,SAAS;oBAErB,MAAMiD,eAAe,CAAC,CAACF;oBAGvB,MAAMG,kBACJ,AAACP,2BAA4BM,eAEzBvB,SADAyB,qBAAqBnD,OAAOwC,UAAU,IAAI;oBAEhD,MAAMY,cAAc,CAAC,CAACF;oBAGtB,MAAMG,sBACJ,AAACV,2BAA4BM,gBAAiBG,cAW1C1B,SATE,OAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACvB1B,OACA;wBAEE,SAASwC;oBACX,GACArB,YAAW,EAEb,OAAO;oBAEf,MAAMmC,kBAAkB,CAAC,CAACD;oBAE1B,MAAME,UACJb,oBACAK,oBACAG,mBACAG;oBAGF,IAAIG;oBACJ,IACED,WACA,IAAI,CAAC,SAAS,IACd,CAACN,gBACDjD,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EACpC,IAAI;wBACF,MAAMyD,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CACtDF,QAAQ,IAAI,EACZA,AAA6B7B,WAA7B6B,QAAQ,gBAAgB,GACpB;4BAAE,iBAAiBA,QAAQ,gBAAgB;wBAAC,IAC5C7B;wBAEN,IAAI+B,WAAWC,OAAO,IAAI,CAACD,SAAS,MAAM,GAAG,GAAG;4BAC9C9D,MACE,uCACAiD,aACAa;4BAEFD,oBAAoBC;4BACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;gCACE,MAAM;gCACN,QAAQb;gCACR,OAAOa;4BACT,GACAZ;wBAEJ,OACElD,MACE,yDACAiD;oBAGN,EAAE,OAAOe,OAAO;wBACdhE,MAAM,kCAAkCgE;oBAC1C;yBAEAhE,MAAM;oBAGV,IAAI,CAAC4D,SACH,MAAM,IAAIK,MAAM,CAAC,mBAAmB,EAAE5D,MAAM,MAAM,EAAE;oBAGtD,IAAI6D;oBAEJ,IAAIlB,yBACFkB,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,OAAO7D,MAAM,KAAK;wBACpB;oBACF;yBACK,IAAIiD,cACTY,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACPf;4BACA,aAAaU;wBACf;oBACF;yBACK,IAAIJ,aACTS,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,IAAIX,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,EAAE;4BACvB,MAAMA,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,IAAI;wBAC7B;oBACF;yBACK,IAAII,iBACTO,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,QAAQ7D,MAAM,MAAM;wBACtB;oBACF;oBAGFyB,QAAAA,YAAAA,SAAW8B;oBAEX,OAAO;wBACL,QAAQ;4BACNA;wBACF;wBACAf;wBACAqB;oBACF;gBACF;YACF;YACA,OAAOlC;QACT;QAEA,KAAK,MAAMJ,QAAQL,MACjB,IAAIK,AAAc,aAAdA,KAAK,IAAI,EAAe;gBAIxBuC,cACAC;YAJF,IACE,CAACxC,KAAK,MAAM,IACZA,AAAgB,SAAhBA,KAAK,MAAM,IACXuC,AAAAA,SAAAA,CAAAA,eAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,aAAa,EAAE,AAAD,MAAM,QACpBC,AAAAA,SAAAA,CAAAA,gBAAAA,KAAK,MAAM,AAAD,IAAVA,KAAAA,IAAAA,cAAa,EAAE,AAAD,MAAM,QACpB;gBACApE,MAAM,kDAAkD4B;gBACxD;YACF;YACA,MAAMyC,aAAa1C,kBAAkBC,MAAMA,KAAK,MAAM;YAEtDF,MAAM,IAAI,CAAC2C;QACb,OAAO,IAAIzC,AAAc,YAAdA,KAAK,IAAI,EAAc;gBAMH0C;YAL7B,MAAMC,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO3C,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO,aAAI0C,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD;gBAC3C,QAAQ1C,KAAK,MAAM;gBACnB,UAAU;wBAEW0C;oBADnB,MAAM,IAAIL,MACRrC,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO,AAAD,KAAC,SAAI0C,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD,KAAK;gBAE5C;YACF;YACF5C,MAAM,IAAI,CAAC6C;QACb,OAAO,IAAI3C,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACnC,MAAM4C,qBAAqD;gBACzD,MAAM;gBACN,SAAS;gBACT,OAAO;gBACP,SAAS5C,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAOvB,SAAW;YAC9B;YACAqB,MAAM,IAAI,CAAC8C;QACb,OAAO,IAAI5C,AAAc,YAAdA,KAAK,IAAI,EAAc;YAChC,MAAM6C,kBACJ;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO7C,KAAK,KAAK;gBACjB,SAASA,KAAK,OAAO;gBACrB,QAAQA,KAAK,MAAM;gBACnB,UAAU,OAAO8C;oBACf,MAAMC,yBAAMD,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;gBACnC;YACF;YACFhD,MAAM,IAAI,CAAC+C;QACb,OAAO;YAEL,MAAMG,WAAWhD,KAAK,IAAI;YAC1B,MAAMiD,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;YACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACC,SAAWA,OAAO,IAAI,KAAKF;YAC5D,MAAMvE,QAAQuB,KAAK,KAAK;YAExB,IAAI,CAACkD,QACH,MAAM,IAAIb,MAAM,CAAC,aAAa,EAAEW,SAAS,WAAW,CAAC;YAIvD,MAAMG,eAAeD,SACjBE,4BAA4BF,OAAO,WAAW,IAC9C,EAAE;YAEN,MAAMG,uBAAuBH,SACzBE,4BAA4BF,OAAO,WAAW,EAAE,QAChD,EAAE;YAENC,aAAa,OAAO,CAAC,CAACG;gBACpB,IAAI7E,KAAK,CAAC6E,MAAM,EAAE;oBAChB,MAAM3E,aAAaH,oBAAoBC,KAAK,CAAC6E,MAAM;oBACnDlF,MACE,uCACA,CAAC,YAAY,EAAE4E,UAAU,EACzB,CAAC,MAAM,EAAEvC,KAAK,SAAS,CAAChC,KAAK,CAAC6E,MAAM,GAAG,EACvC,CAAC,WAAW,EAAE7C,KAAK,SAAS,CAAC9B,aAAa;oBAE5C,MAAM4E,aAAaxD,kBACjBpB,YACAF,KAAK,CAAC6E,MAAM,EACZ,CAAC7D;wBACChB,KAAK,CAAC6E,MAAM,GAAG7D;oBACjB;oBAEFK,MAAM,IAAI,CAACyD;gBACb,OAAO;oBACL/C,OACE,CAAC6C,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEN,UAAU;oBAE3E5E,MAAM,CAAC,OAAO,EAAEkF,MAAM,6BAA6B,EAAEN,UAAU;gBACjE;YACF;YAEA,MAAMzD,OAKF;gBACF,MAAM;gBACN,SAASyD;gBACT,SAAShD,KAAK,OAAO;gBACrB,OAAOA,KAAK,KAAK;gBACjB,UAAU,OAAOvB,OAAOW;wBAKOoE;oBAJ7BpF,MACE,oBACA4E,UACAvE,OACA,CAAC,wBAAwB,EAAE,QAAA+E,CAAAA,mBAAAA,QAAQ,OAAO,AAAD,IAAdA,KAAAA,IAAAA,iBAAiB,MAAM,EAAE;oBAItD,MAAMvC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;oBACxD7B,QAAQ,IAAI,CAAC,SAAS,GAAG6B;oBAEzBoC,qBAAqB,OAAO,CAAC,CAACC;wBAC5B9C,OACE/B,KAAK,CAAC6E,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEN,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;oBAE9G;oBAEA,IAAI;wBACF,MAAMS,QAAQ,GAAG,CAAC;4BACf;gCACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;oCACrCrF,MAAM;oCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC8E,OAAO,IAAI,EAAEzE;oCACrDL,MAAM;gCACR;4BACF;4BACA2E,yBAAM;yBACP;oBACH,EAAE,OAAOW,eAAoB;wBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;wBACnC,MAAM,IAAIrB,MACR,CAAC,wCAAwC,EAAEa,OAAO,IAAI,CAAC,EAAE,EAAES,iBAAiB,EAC5E;4BAAE,OAAOD;wBAAc;oBAE3B;oBAGA,IAAIR,OAAO,WAAW,EACpB,IAAI;wBACFzE,QAAQoF,iBAAiBpF,OAAOyE,OAAO,WAAW;oBACpD,EAAE,OAAOd,OAAY;wBACnB,MAAM,IAAIC,MACR,CAAC,8BAA8B,EAAEa,OAAO,IAAI,CAAC,EAAE,EAAEd,MAAM,OAAO,CAAC,cAAc,EAAE3B,KAAK,SAAS,CAAChC,QAAQ,EACtG;4BAAE,OAAO2D;wBAAM;oBAEnB;oBAGFhE,MAAM,kBAAkB8E,OAAO,IAAI;oBACnC,MAAMY,WAAWZ,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;oBAChD,MAAMY,SAASrF,OAAOW;oBACtBhB,MAAM,iBAAiB8E,OAAO,IAAI;oBAElC,IAAI;wBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;4BACpC9E,MAAM;4BACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC8E,OAAO,IAAI,EAAEzE;4BACpDL,MAAM;wBACR;oBACF,EAAE,OAAOsF,eAAoB;wBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;wBACnC,MAAM,IAAIrB,MACR,CAAC,uCAAuC,EAAEa,OAAO,IAAI,CAAC,EAAE,EAAES,iBAAiB,EAC3E;4BAAE,OAAOD;wBAAc;oBAE3B;oBAEA,OAAO;wBACL,QAAQ;4BACN,SAAS;4BACT,QAAQV;4BACR,OAAOvE;wBACT;oBACF;gBACF;YACF;YACAqB,MAAM,IAAI,CAACP;QACb;QAGF,MAAMwE,eAAejE,MAAM,GAAG,CAC5B,CAACP,MAA0ByE;YACzB,IAAIzE,AAAc,aAAdA,KAAK,IAAI,EACX,OAAO,IAAI,CAAC,6BAA6B,CACvCA,MACAyE,UAAUlE,MAAM,MAAM,GAAG;YAG7B,OAAOP;QACT;QAGF,OAAO;YACL,OAAOwE;QACT;IACF;IAEA,MAAc,qBAAqBE,eAAgC,EAAE;QACnE,MAAMjD,WAAWhC,KAAK,GAAG;QACzB,MAAMiC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;QACxD,MAAMC,aAAoC;YACxC,MAAM;YACN,IAAIF;YACJ,YAAYC,UAAU,gBAAgB;YACtC,QAAQ;QACV;QAEAgD,gBAAgB,IAAI,CAAC,QAAQ,GAAG;YAAC/C;SAAW;QAC3C+C,gBAAgB,IAAI,CAA2B,SAAS,GAAGhD;QAE5D,OAAO;YACLA;QACF;IACF;IAEA,MAAM,uBAAuBiD,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,eAAe,IAAIC,SAASC,aAAa,UAAUJ,kBAAkB;YACzE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,MAAM3E,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL2E;YACF;YACA,UAAU,OAAOzF,OAAOwF;gBACtB,MAAM,IAAI,CAAC,oBAAoB,CAACA;gBAChC,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLE;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QAEA,MAAMC,aAAa,MAAM,CAAC7E;QAC1B,MAAM6E,aAAa,KAAK;QAExB,OAAO;YACL,UAAUA;QACZ;IACF;IAEQ,mBACNF,eAAuB,EACvBK,aAAiC,EACjC3E,WAAyB,EACG;QAC5B,MAAML,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACL2E;YACF;YACA,UAAU,OAAOzF,OAAOwF;gBACtB,MAAMO,YAAYxF,KAAK,GAAG;gBAC1B,MAAM,EAAEiC,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,oBAAoB,CAACgD;gBACtD,MAAM,EAAEQ,MAAM,EAAE,GAAG7E;gBACnB,MAAM8E,qBACJD,AAAW,kBAAXA,SAA2B7E,YAAY,kBAAkB,GAAGO;gBAE9DK,OACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;gBAEF,MAAMyC,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;gBACpD7E,MACE,sCACA6E,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhD1C,OAAOmE,MAAM,OAAO,CAAC1B,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpB2B,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;gBAIrG,MAAMC,aAAa,MAAOH,AAAAA,CAAAA,qBAAqBI,iBAAiB9E,cAAAA,EAC9DvB,MAAM,eAAe,EACrB;oBACE,SAASwC;oBACTsD;oBACA,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;oBAC3CtB;oBACArD;oBACA,qBAAqB,IAAI,CAAC,mBAAmB;gBAC/C;gBAEFxB,MAAM,cAAcqC,KAAK,SAAS,CAACoE,YAAY,MAAM;gBAErD,MAAM,EACJE,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClC7C,KAAK,EACLzB,KAAK,EACLuE,WAAW,EACXnC,KAAK,EACN,GAAG8B;gBAEJZ,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCiB;gBACF;gBACAjB,gBAAgB,IAAI,CAAC,KAAK,GAAGtD;gBAE7B,MAAMwE,eAAeJ,WAAW,EAAE;gBAElC,IAAIhC,OAAO;oBACT,MAAMqC,UAAUpG,KAAK,GAAG;oBACxB,MAAMqG,gBAAgBtC,QAASqC,CAAAA,UAAUZ,SAAQ;oBACjD,IAAIa,gBAAgB,GAClBF,aAAa,IAAI,CAAC;wBAChB,MAAM;wBACN,OAAO;4BACL,QAAQE;wBACV;wBACA,QAAQ;oBACV;gBAEJ;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrB3E,OACE,CAACyE,sCAAsClC,OACvCX,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAAS+C;wBACTF;wBACAD;wBACA,UAAUH,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA5D;gBACF;YACF;QACF;QAEA,OAAO1B;IACT;IAEA,MAAM,SACJ+F,KAAa,EACb3F,KAAuB,EACvBC,WAAyB,EACC;QAC1B,MAAMwE,eAAe,IAAIC,SAASiB,OAAO;YACvC,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAExF,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACH,OAAOC;QAC5D,MAAMwE,aAAa,MAAM,CAACtE;QAC1B,MAAML,SAAS,MAAM2E,aAAa,KAAK;QACvC,MAAM,EAAEmB,MAAM,EAAE,GAAG9F;QACnB,OAAO;YACL8F;YACA,UAAUnB;QACZ;IACF;IAEQ,wBAAwBoB,WAAoB,EAAE;QACpD,OACE,IAAI,CAAC,oBAAoB,IACzBC,oBAAoB,oBAAoB,CACtCC,oCAEDF,CAAAA,cACGjH,uCACAD,2BAA0B;IAElC;IAEA,MAAM,OACJqH,UAAkB,EAClB/F,WAAyB,EACzB2E,aAAsB,EACtB1E,SAAmB,EAQnB;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMuE,eAAe,IAAIC,SAASC,aAAa,UAAUqB,aAAa;YACpE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAEA,IAAIC,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBAAuB,IAAI,CAAC,uBAAuB,CACvDlG,AAAuB,kBAAvBA,YAAY,MAAM;QAIpB,MAAO,KAAM;YACX,IAAIgG,cAAcE,sBAAsB;gBACtC,MAAMC,WAAW,CAAC,WAAW,EAAED,qBAAqB,+EAA+E,CAAC;gBAEpI,OAAO,IAAI,CAAC,eAAe,CAAC1B,cAAc2B,UAAUnG;YACtD;YAGA,MAAMoG,eAAe,IAAI,CAAC,kBAAkB,CAC1CL,YACApB,eACA3E;YAGF,MAAMwE,aAAa,MAAM,CAAC4B;YAC1B,MAAMvG,SAAS,MAAM2E,aAAa,KAAK;YACvC,MAAMS,aAAiCpF,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YACrD,IAAI2E,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQS;gBACR,UAAUT;YACZ;YAIF,MAAMzE,QAAQkF,WAAW,OAAO,IAAI,EAAE;YACtCgB,SAAS,IAAI,IAAKhB,WAAW,QAAQ,IAAI,EAAE;YAE3C,IAAIoB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CtG,OACAC,aACAC;gBAEFuE,aAAa,MAAM,CAAC6B,YAAY,KAAK;YACvC,EAAE,OAAO7D,OAAO;gBACd,OAAO,IAAI,CAAC,eAAe,CACzBgC,cACA,CAAC,4CAA4C,EAAEhC,MAAM,SAAS,EAAE3B,KAAK,SAAS,CAC5Ed,QACC,EACHC;YAEJ;YAEA,MAAMwE,aAAa,KAAK;YACxB,IAAIA,aAAa,cAAc,IAC7B,OAAO;gBACL,QAAQjE;gBACR,UAAUiE;YACZ;YAIF,IAAI,CAACS,WAAW,kCAAkC,EAChD;YAIFe;QACF;QAEA,OAAO;YACL,QAAQ;gBACNC;YACF;YACA,UAAUzB;QACZ;IACF;IAEQ,oBACN8B,IAAsE,EACtEC,MAA2B,EAC3BvG,WAAyB,EACzBwG,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBACL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO1H,OAAO4B;gBACtB,MAAM,EAAEd,IAAI,EAAE,GAAGc;gBACjB,IAAIK;gBACJ,MAAME,gBAAgC,CAACC;oBACrCH,cAAcG;gBAChB;gBACA,IAAI,CAAC,OAAO,CAAC,iBAAiB,GAAGD;gBAGjC,MAAMI,WAAWhC,KAAK,GAAG;gBACzB,MAAMiC,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC;gBACxD1B,KAAK,SAAS,GAAG0B;gBAEjB,MAAMC,aAAoC;oBACxC,MAAM;oBACN,IAAIF;oBACJ,YAAYC,UAAU,gBAAgB;oBACtC,QAAQ;gBACV;gBACA1B,KAAK,QAAQ,GAAG;oBAAC2B;iBAAW;gBAE5B,MAAMqF,mBAAmBL,AAAS,YAATA;gBACzB,IAAIM,cAAcL;gBAClB,IAAIM,cAAc;gBAClB,IAAIF,oBAAqBL,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEO,cAAc;oBACd,MAAMC,gBACJR,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIK,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGP,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,MAAM,EAAEQ,IAAI,EAAEhG,KAAK,EAAEiG,OAAO,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACzDJ,aACA5G,aACAwG,KACAC;gBAGF,IAAIQ,eAAeF;gBACnB,IAAIJ,kBAEF,IAAI,AAAgB,YAAhB,OAAOI,MACTE,eAAeF;qBACV;oBACLnG,OACE0F,AAAS,cAATA,OAAqBS,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,IAAM,CAACF,YAAY,AAAD,MAAMtG,SAAY,MACzD;oBAEF0G,eAAgBF,IAAY,CAACF,YAAY;gBAC3C;gBAGF,OAAO;oBACL,QAAQI;oBACR,KAAKnG;oBACLC;oBACAiG;gBACF;YACF;QACF;QAEA,OAAON;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BvG,WAAyB,EACzBwG,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMjC,eAAe,IAAIC,SACvBC,aACE4B,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAAS1F,KAAK,SAAS,CAAC0F,UAEvD;YACE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QAGF,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACAvG,aACAwG,KACAC;QAGF,MAAMjC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACkC;QAC7D,MAAM7G,SAAS,MAAM2E,aAAa,KAAK;QAEvC,IAAI,CAAC3E,QACH,MAAM,IAAI4C,MACR;QAIJ,MAAM,EAAEkD,MAAM,EAAEqB,OAAO,EAAE,GAAGnH;QAE5B,OAAO;YACL8F;YACAqB;YACA,UAAUxC;QACZ;IACF;IAEA,MAAc,gBACZA,YAAsB,EACtB2B,QAAgB,EAChBnG,WAAyB,EACzB;QACA,MAAMkH,YAAsD;YAC1D,MAAM;YACN,OAAO;gBACL,SAASf;YACX;YACA,QAAQ;QACV;QACA,MAAM,EAAEjG,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClD;YAACgH;SAAU,EACXlH;QAEF,MAAMwE,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACtE,KAAK,CAAC,EAAE;QACrE,MAAMsE,aAAa,KAAK;QAExB,OAAO;YACL,QAAQjE;YACR,UAAUiE;QACZ;IACF;IAEA,MAAM,aAAa2C,MAAc,EAAEnH,WAAyB,EAAE;QAC5D,MAAMoH,YAAsD;YAC1D,MAAM;YACN,OAAO;gBACLD;YACF;YACA,QAAQ;QACV;QAEA,MAAM,EAAE,OAAOE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;YAACD;SAAU,EACXpH;QAGF,OAAO,IAAI,CAAC,6BAA6B,CAACqH,UAAU,CAAC,EAAE;IACzD;IAEA,MAAM,QACJC,SAAsB,EACtBd,GAA+B,EAC/BxG,WAAyB,EACO;QAChC,MAAM,EAAEuH,UAAU,EAAEd,gBAAgB,EAAE,GAAGe,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM/C,eAAe,IAAIC,SAASC,aAAa,WAAW+C,cAAc;YACtE,aAAa,IAAI,CAAC,mBAAmB;QACvC;QACA,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE,GAAGnB;QAEvC5F,OAAO0G,WAAW;QAClB1G,OAAO8G,WAAW;QAClB9G,OAAO+G,iBAAiB;QAExB/G,OACE+G,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBxI,KAAK,GAAG;QACjC,IAAIwF,YAAYxF,KAAK,GAAG;QACxB,IAAIyI,eAAe;QACnB,MAAOzI,KAAK,GAAG,KAAKwI,mBAAmBF,UAAW;YAChD9C,YAAYxF,KAAK,GAAG;YACpB,MAAMsH,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAa,YACAvH,aACA;gBACE,iBAAiB;YACnB,GACAyG;YAGF,MAAMjC,aAAa,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAACkC;YAC7D,MAAM7G,SAAU,MAAM2E,aAAa,KAAK;YAOxC,IAAI3E,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQU;gBACR,UAAUiE;YACZ;YAGFqD,eACEhI,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACb,CAACA,UAAU,CAAC,0BAA0B,EAAE0H,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMO,MAAM1I,KAAK,GAAG;YACpB,IAAI0I,MAAMlD,YAAY+C,iBAAiB;gBACrC,MAAMlC,gBAAgBkC,kBAAmBG,CAAAA,MAAMlD,SAAQ;gBACvD,MAAMmD,YAAY,MAAM,IAAI,CAAC,YAAY,CAACtC,eAAezF;gBACzD,MAAMwE,aAAa,MAAM,CAACuD;YAC5B;QACF;QAEA,OAAO,IAAI,CAAC,eAAe,CACzBvD,cACA,CAAC,iBAAiB,EAAEqD,cAAc,EAClC7H;IAEJ;IAv/BA,YACEgI,iBAAoC,EACpCC,OAAgB,EAChBC,IAIC,CACD;QAzBF;QAEA;QAEA;QAEA,uBAAQ,uBAAR;QAEA;QAEA;QAgBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;QAC5C,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,mBAAmB,GAAG,IAAIC;IACjC;AAy+BF"}
+{"version":3,"file":"agent/tasks.mjs","sources":["webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["import { ConversationHistory, plan, uiTarsPlanning } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type { TaskRunner } from '@/task-runner';\nimport type Insight from '@/insight';\nimport type {\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InsightDump,\n  InsightExtractOption,\n  InsightExtractParam,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n} from '@/types';\nimport { InsightError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nexport { locatePlanForLocate } from './task-builder';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  insight: Insight;\n\n  taskCache?: TaskCache;\n\n  private readonly taskBuilder: TaskBuilder;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    insight: Insight,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.insight = insight;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.conversationHistory = new ConversationHistory();\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      insight,\n      taskCache: opts.taskCache,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.insight.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n      },\n    );\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    cacheable?: boolean,\n  ) {\n    return this.taskBuilder.build(plans, modelConfig, {\n      cacheable,\n    });\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    await session.appendAndRun(task);\n\n    return {\n      runner: session.getRunner(),\n    };\n  }\n\n  private createPlanningTask(\n    userInstruction: string,\n    actionContext: string | undefined,\n    modelConfig: IModelConfig,\n  ): ExecutionTaskPlanningApply {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        const { vlMode } = modelConfig;\n        const uiTarsModelVersion =\n          vlMode === 'vlm-ui-tars' ? modelConfig.uiTarsModelVersion : undefined;\n\n        assert(\n          this.interface.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.interface.actionSpace();\n        debug(\n          'actionSpace for this interface is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await (uiTarsModelVersion ? uiTarsPlanning : plan)(\n          param.userInstruction,\n          {\n            context: uiContext,\n            actionContext,\n            interfaceType: this.interface.interfaceType as InterfaceType,\n            actionSpace,\n            modelConfig,\n            conversationHistory: this.conversationHistory,\n          },\n        );\n        debug('planResult', JSON.stringify(planResult, null, 2));\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push({\n              type: 'Sleep',\n              param: {\n                timeMs: timeRemaining,\n              },\n              locate: null,\n            } as PlanningAction<PlanningActionParamSleep>);\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          uiContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(plans, modelConfig);\n    const result = await session.appendAndRun(tasks);\n    const { output } = result!;\n    return {\n      output,\n      runner: session.getRunner(),\n    };\n  }\n\n  private getReplanningCycleLimit(isVlmUiTars: boolean) {\n    return (\n      this.replanningCycleLimit ||\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) ||\n      (isVlmUiTars\n        ? defaultVlmUiTarsReplanningCycleLimit\n        : defaultReplanningCycleLimit)\n    );\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfig: IModelConfig,\n    actionContext?: string,\n    cacheable?: boolean,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit = this.getReplanningCycleLimit(\n      modelConfig.vlMode === 'vlm-ui-tars',\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanning ${replanningCycleLimit} times, which is more than the limit, please split the task into multiple steps`;\n\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      // Create planning task (automatically includes execution history if available)\n      const planningTask = this.createPlanningTask(\n        userPrompt,\n        actionContext,\n        modelConfig,\n      );\n\n      const result = await session.appendAndRun(planningTask);\n      const planResult: PlanningAIResponse = result?.output;\n      if (session.isInErrorState()) {\n        return {\n          output: planResult,\n          runner,\n        };\n      }\n\n      // Execute planned actions\n      const plans = planResult.actions || [];\n      yamlFlow.push(...(planResult.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          modelConfig,\n          cacheable,\n        );\n        await session.appendAndRun(executables.tasks);\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (session.isInErrorState()) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      // Check if task is complete\n      if (!planResult.more_actions_needed_by_instruction) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      replanCount++;\n    }\n\n    return {\n      output: {\n        yamlFlow,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: InsightDump | undefined;\n        const applyDump = (dump: InsightDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n          };\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n        try {\n          extractResult = await this.insight.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            multimodalPrompt,\n          );\n        } catch (error) {\n          if (error instanceof InsightError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, usage, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else {\n            assert(\n              type !== 'WaitFor' ? data?.[keyOfResult] !== undefined : true,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner: session.getRunner(),\n    };\n  }\n\n  async taskForSleep(timeMs: number, modelConfig: IModelConfig) {\n    const sleepPlan: PlanningAction<PlanningActionParamSleep> = {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n      locate: null,\n    };\n    // The convertPlanToExecutable requires modelConfig as a parameter but will not consume it when type is Sleep\n    const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n      [sleepPlan],\n      modelConfig,\n    );\n\n    return sleepTasks[0];\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let startTime = Date.now();\n    let errorThought = '';\n    while (Date.now() - overallStartTime < timeoutMs) {\n      startTime = Date.now();\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        {\n          doNotThrowError: true,\n        },\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - startTime < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - startTime);\n        const sleepTask = await this.taskForSleep(timeRemaining, modelConfig);\n        await session.append(sleepTask);\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n"],"names":["debug","getDebug","defaultReplanningCycleLimit","defaultVlmUiTarsReplanningCycleLimit","TaskExecutor","title","options","ExecutionSession","Promise","plans","modelConfig","cacheable","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","actionContext","startTime","Date","vlMode","uiTarsModelVersion","undefined","actionSpace","action","Array","console","planResult","uiTarsPlanning","plan","JSON","actions","log","more_actions_needed_by_instruction","error","usage","rawResponse","sleep","finalActions","timeNow","timeRemaining","tasks","result","output","isVlmUiTars","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","userPrompt","runner","replanCount","yamlFlow","replanningCycleLimit","errorMsg","planningTask","executables","type","demand","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","InsightError","data","thought","outputResult","Error","timeMs","sleepPlan","sleepTasks","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","errorThought","now","sleepTask","interfaceInstance","insight","opts","ConversationHistory","TaskBuilder"],"mappings":";;;;;;;;;;;;;;;;;;;AAyCA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,8BAA8B;AACpC,MAAMC,uCAAuC;AAEtC,MAAMC;IAgBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAwBQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK;QACvB;IAEJ;IAEA,MAAa,wBACXG,KAAuB,EACvBC,WAAyB,EACzBC,SAAmB,EACnB;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACF,OAAOC,aAAa;YAChDC;QACF;IACF;IAEA,MAAM,uBAAuBC,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUH;QAGzB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMC,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACL,QAAQF,QAAQ,SAAS;QAC3B;IACF;IAEQ,mBACNF,eAAuB,EACvBS,aAAiC,EACjCX,WAAyB,EACG;QAC5B,MAAMM,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAMI,YAAYC,KAAK,GAAG;gBAC1B,MAAM,EAAEJ,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,MAAM,EAAEK,MAAM,EAAE,GAAGd;gBACnB,MAAMe,qBACJD,AAAW,kBAAXA,SAA2Bd,YAAY,kBAAkB,GAAGgB;gBAE9DN,OACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;gBAEF,MAAMO,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;gBACpD3B,MACE,sCACA2B,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDR,OAAOS,MAAM,OAAO,CAACF,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;gBAIrG,MAAMC,aAAa,MAAON,AAAAA,CAAAA,qBAAqBO,iBAAiBC,IAAG,EACjEhB,MAAM,eAAe,EACrB;oBACE,SAASE;oBACTE;oBACA,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;oBAC3CM;oBACAjB;oBACA,qBAAqB,IAAI,CAAC,mBAAmB;gBAC/C;gBAEFV,MAAM,cAAckC,KAAK,SAAS,CAACH,YAAY,MAAM;gBAErD,MAAM,EACJI,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClCC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,KAAK,EACN,GAAGV;gBAEJb,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCsB;gBACF;gBACAtB,gBAAgB,IAAI,CAAC,KAAK,GAAGqB;gBAE7B,MAAMG,eAAeP,WAAW,EAAE;gBAElC,IAAIM,OAAO;oBACT,MAAME,UAAUpB,KAAK,GAAG;oBACxB,MAAMqB,gBAAgBH,QAASE,CAAAA,UAAUrB,SAAQ;oBACjD,IAAIsB,gBAAgB,GAClBF,aAAa,IAAI,CAAC;wBAChB,MAAM;wBACN,OAAO;4BACL,QAAQE;wBACV;wBACA,QAAQ;oBACV;gBAEJ;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrBtB,OACE,CAACiB,sCAAsCI,OACvCH,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAASI;wBACTL;wBACAD;wBACA,UAAUL,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACAZ;gBACF;YACF;QACF;QAEA,OAAOH;IACT;IAEA,MAAM,SACJX,KAAa,EACbI,KAAuB,EACvBC,WAAyB,EACC;QAC1B,MAAMI,UAAU,IAAI,CAAC,sBAAsB,CAACT;QAC5C,MAAM,EAAEwC,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACpC,OAAOC;QAC5D,MAAMoC,SAAS,MAAMhC,QAAQ,YAAY,CAAC+B;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD;QACnB,OAAO;YACLC;YACA,QAAQjC,QAAQ,SAAS;QAC3B;IACF;IAEQ,wBAAwBkC,WAAoB,EAAE;QACpD,OACE,IAAI,CAAC,oBAAoB,IACzBC,oBAAoB,oBAAoB,CACtCC,oCAEDF,CAAAA,cACG7C,uCACAD,2BAA0B;IAElC;IAEA,MAAM,OACJiD,UAAkB,EAClBzC,WAAyB,EACzBW,aAAsB,EACtBV,SAAmB,EAQnB;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUoC;QAEzB,MAAMC,SAAStC,QAAQ,SAAS;QAEhC,IAAIuC,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBAAuB,IAAI,CAAC,uBAAuB,CACvD7C,AAAuB,kBAAvBA,YAAY,MAAM;QAIpB,MAAO,KAAM;YACX,IAAI2C,cAAcE,sBAAsB;gBACtC,MAAMC,WAAW,CAAC,WAAW,EAAED,qBAAqB,+EAA+E,CAAC;gBAEpI,OAAOzC,QAAQ,eAAe,CAAC0C;YACjC;YAGA,MAAMC,eAAe,IAAI,CAAC,kBAAkB,CAC1CN,YACA9B,eACAX;YAGF,MAAMoC,SAAS,MAAMhC,QAAQ,YAAY,CAAC2C;YAC1C,MAAM1B,aAAiCe,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YACrD,IAAIhC,QAAQ,cAAc,IACxB,OAAO;gBACL,QAAQiB;gBACRqB;YACF;YAIF,MAAM3C,QAAQsB,WAAW,OAAO,IAAI,EAAE;YACtCuB,SAAS,IAAI,IAAKvB,WAAW,QAAQ,IAAI,EAAE;YAE3C,IAAI2B;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CjD,OACAC,aACAC;gBAEF,MAAMG,QAAQ,YAAY,CAAC4C,YAAY,KAAK;YAC9C,EAAE,OAAOpB,OAAO;gBACd,OAAOxB,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEwB,MAAM,SAAS,EAAEJ,KAAK,SAAS,CAC5EzB,QACC;YAEP;YACA,IAAIK,QAAQ,cAAc,IACxB,OAAO;gBACL,QAAQY;gBACR0B;YACF;YAIF,IAAI,CAACrB,WAAW,kCAAkC,EAChD;YAIFsB;QACF;QAEA,OAAO;YACL,QAAQ;gBACNC;YACF;YACAF;QACF;IACF;IAEQ,oBACNO,IAAsE,EACtEC,MAA2B,EAC3BlD,WAAyB,EACzBmD,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBACL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO3C,OAAO+C;gBACtB,MAAM,EAAEhD,IAAI,EAAE,GAAGgD;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZnD,KAAK,GAAG,GAAG;wBACTmD;oBACF;gBACF;gBAGA,MAAMhD,YAAY6C,YAAY,SAAS;gBACvC5C,OAAOD,WAAW;gBAElB,MAAMiD,mBAAmBT,AAAS,YAATA;gBACzB,IAAIU,cAAcT;gBAClB,IAAIU,cAAc;gBAClB,IAAIF,oBAAqBT,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEW,cAAc;oBACd,MAAMC,gBACJZ,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIS,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGX,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,IAAIY;gBACJ,IAAI;oBACFA,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACA3D,aACAmD,KACAC;gBAEJ,EAAE,OAAOxB,OAAO;oBACd,IAAIA,iBAAiBmC,cACnBP,UAAU5B,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEoC,IAAI,EAAEnC,KAAK,EAAEoC,OAAO,EAAER,IAAI,EAAE,GAAGK;gBACvCN,UAAUC;gBAEV,IAAIS,eAAeF;gBACnB,IAAIN,kBAEF,IAAI,AAAgB,YAAhB,OAAOM,MACTE,eAAeF;qBACV;oBACLtD,OACEuC,AAAS,cAATA,OAAqBe,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,IAAM,CAACJ,YAAY,AAAD,MAAM5C,SAAY,MACzD;oBAEFkD,eAAgBF,IAAY,CAACJ,YAAY;gBAC3C;gBAGF,OAAO;oBACL,QAAQM;oBACR,KAAKX;oBACL1B;oBACAoC;gBACF;YACF;QACF;QAEA,OAAOZ;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BlD,WAAyB,EACzBmD,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMhD,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACE4C,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAAS1B,KAAK,SAAS,CAAC0B;QAIzD,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACAlD,aACAmD,KACAC;QAGF,MAAMhB,SAAS,MAAMhC,QAAQ,YAAY,CAACiD;QAE1C,IAAI,CAACjB,QACH,MAAM,IAAI+B,MACR;QAIJ,MAAM,EAAE9B,MAAM,EAAE4B,OAAO,EAAE,GAAG7B;QAE5B,OAAO;YACLC;YACA4B;YACA,QAAQ7D,QAAQ,SAAS;QAC3B;IACF;IAEA,MAAM,aAAagE,MAAc,EAAEpE,WAAyB,EAAE;QAC5D,MAAMqE,YAAsD;YAC1D,MAAM;YACN,OAAO;gBACLD;YACF;YACA,QAAQ;QACV;QAEA,MAAM,EAAE,OAAOE,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;YAACD;SAAU,EACXrE;QAGF,OAAOsE,UAAU,CAAC,EAAE;IACtB;IAEA,MAAM,QACJC,SAAsB,EACtBpB,GAA+B,EAC/BnD,WAAyB,EACO;QAChC,MAAM,EAAEwE,UAAU,EAAEpB,gBAAgB,EAAE,GAAGqB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMpE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAWqE;QAE1B,MAAMhC,SAAStC,QAAQ,SAAS;QAChC,MAAM,EAAEuE,SAAS,EAAEC,eAAe,EAAE,GAAGzB;QAEvCzC,OAAO6D,WAAW;QAClB7D,OAAOiE,WAAW;QAClBjE,OAAOkE,iBAAiB;QAExBlE,OACEkE,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBhE,KAAK,GAAG;QACjC,IAAID,YAAYC,KAAK,GAAG;QACxB,IAAIiE,eAAe;QACnB,MAAOjE,KAAK,GAAG,KAAKgE,mBAAmBF,UAAW;YAChD/D,YAAYC,KAAK,GAAG;YACpB,MAAMwC,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAmB,YACAxE,aACA;gBACE,iBAAiB;YACnB,GACAoD;YAGF,MAAMhB,SAAU,MAAMhC,QAAQ,YAAY,CAACiD;YAO3C,IAAIjB,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQpB;gBACR0B;YACF;YAGFoC,eACE1C,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACb,CAACA,UAAU,CAAC,0BAA0B,EAAEoC,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMO,MAAMlE,KAAK,GAAG;YACpB,IAAIkE,MAAMnE,YAAYgE,iBAAiB;gBACrC,MAAM1C,gBAAgB0C,kBAAmBG,CAAAA,MAAMnE,SAAQ;gBACvD,MAAMoE,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC9C,eAAelC;gBACzD,MAAMI,QAAQ,MAAM,CAAC4E;YACvB;QACF;QAEA,OAAO5E,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAE0E,cAAc;IACnE;IAhhBA,YACEG,iBAAoC,EACpCC,OAAgB,EAChBC,IAIC,CACD;QA3BF;QAEA;QAEA;QAEA,uBAAiB,eAAjB;QAEA,uBAAQ,uBAAR;QAEA;QAEA;QAgBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;QAC5C,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,mBAAmB,GAAG,IAAIC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCJ;YACAC;YACA,WAAWC,KAAK,SAAS;QAC3B;IACF;AA6fF"}

package/dist/es/agent/ui-utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.mjs","sources":["webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  AndroidPullParam,\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightLocate,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  return task.subType && task.subType !== 'Plan'\n    ? `${task.type} / ${task.subType || ''}`\n    : task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n      const prompt = locate.prompt.prompt;\n      const images = locate.prompt.images || [];\n\n      if (images.length === 0) return prompt;\n\n      const imagesStr = images\n        .map((image) => {\n          let url = image.url;\n          if (\n            url.startsWith('data:image/') ||\n            (url.startsWith('data:') && url.includes('base64'))\n          ) {\n            url = `${url.substring(0, 15)}...`;\n          }\n          return `[${image.name}](${url})`;\n        })\n        .join(', ');\n\n      return `${prompt}, ${imagesStr}`;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: AndroidPullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      locateParamStr((task as ExecutionTaskInsightLocate)?.param) ||\n      (task as ExecutionTaskInsightLocate)?.param?.id ||\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object' && locateParamStr(value as any)) {\n    return locateParamStr(value as any);\n  }\n\n  return JSON.stringify(value, undefined, 2);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","images","imagesStr","image","url","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","_task_param7","locateStr","_task_param8","_task_param9","JSON","undefined"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IACzC,OAAOA,KAAK,OAAO,IAAIA,AAAiB,WAAjBA,KAAK,OAAO,GAC/B,GAAGA,KAAK,IAAI,CAAC,GAAG,EAAEA,KAAK,OAAO,IAAI,IAAI,GACtCA,KAAK,IAAI;AACf;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAAE;YAC7D,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,MAAME,SAASF,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE;YAEzC,IAAIE,AAAkB,MAAlBA,OAAO,MAAM,EAAQ,OAAOD;YAEhC,MAAME,YAAYD,OACf,GAAG,CAAC,CAACE;gBACJ,IAAIC,MAAMD,MAAM,GAAG;gBACnB,IACEC,IAAI,UAAU,CAAC,kBACdA,IAAI,UAAU,CAAC,YAAYA,IAAI,QAAQ,CAAC,WAEzCA,MAAM,GAAGA,IAAI,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC;gBAEpC,OAAO,CAAC,CAAC,EAAED,MAAM,IAAI,CAAC,EAAE,EAAEC,IAAI,CAAC,CAAC;YAClC,GACC,IAAI,CAAC;YAER,OAAO,GAAGJ,OAAO,EAAE,EAAEE,WAAW;QAClC;IACF;IAEA,OAAO;AACT;AAEO,SAASG,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAA4B;IACvD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZX,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGW,KAAK,GAAG,EAAEX,QAAQ;IAE9B,OAAOW;AACT;AAEO,SAASC,SAASf,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACpBiB;QAARD,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,cAACjB,KAAgC,KAAK,AAAD,IAArCiB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAEA,IAAIjB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAGzBkB,cACAC,cACAC;QAJFJ,QACEf,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAqC,KAAK,KACzDA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAAqC,KAAK,AAAD,IAA1CkB,KAAAA,IAAAA,aAA6C,EAAE,AAAD,KAC7ClB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAAoC,KAAK,AAAD,IAAzCmB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDnB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAAwC,KAAK,AAAD,IAA7CoB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAIpB,AAAc,aAAdA,KAAK,IAAI,EAAe;YAKfqB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMtB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAMyB,YAAYvB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXV,QAAQ,GAAIhB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQ1B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAA8B,KAAK,AAAD,IAAnCsB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDN,QAAQR,eAAgBR,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACvB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CgB,QAAQN,aAAcV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADwB,CAAAA,eAACxB,KAA8B,KAAK,AAAD,IAAnCwB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARX,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD2B,CAAAA,eAAC3B,KAA8B,KAAK,AAAD,IAAnC2B,KAAAA,IAAAA,aAAsC,KAAK;QACrD;QAEA,IAAIF,WAEAT,QADEA,QACM,GAAGS,UAAU,GAAG,EAAET,OAAO,GAEzBS;IAGd;IAEA,IAAI,AAAiB,WAAVT,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,SAAsBf,eAAee,QAC9C,OAAOf,eAAee;IAGxB,OAAOY,KAAK,SAAS,CAACZ,OAAOa,QAAW;AAC1C"}
+{"version":3,"file":"agent/ui-utils.mjs","sources":["webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightLocate,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  return task.subType && task.subType !== 'Plan'\n    ? `${task.type} / ${task.subType || ''}`\n    : task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n      const prompt = locate.prompt.prompt;\n      const images = locate.prompt.images || [];\n\n      if (images.length === 0) return prompt;\n\n      const imagesStr = images\n        .map((image) => {\n          let url = image.url;\n          if (\n            url.startsWith('data:image/') ||\n            (url.startsWith('data:') && url.includes('base64'))\n          ) {\n            url = `${url.substring(0, 15)}...`;\n          }\n          return `[${image.name}](${url})`;\n        })\n        .join(', ');\n\n      return `${prompt}, ${imagesStr}`;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      locateParamStr((task as ExecutionTaskInsightLocate)?.param) ||\n      (task as ExecutionTaskInsightLocate)?.param?.id ||\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object' && locateParamStr(value as any)) {\n    return locateParamStr(value as any);\n  }\n\n  return JSON.stringify(value, undefined, 2);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","images","imagesStr","image","url","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","_task_param7","locateStr","_task_param8","_task_param9","JSON","undefined"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IACzC,OAAOA,KAAK,OAAO,IAAIA,AAAiB,WAAjBA,KAAK,OAAO,GAC/B,GAAGA,KAAK,IAAI,CAAC,GAAG,EAAEA,KAAK,OAAO,IAAI,IAAI,GACtCA,KAAK,IAAI;AACf;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAAE;YAC7D,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,MAAME,SAASF,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE;YAEzC,IAAIE,AAAkB,MAAlBA,OAAO,MAAM,EAAQ,OAAOD;YAEhC,MAAME,YAAYD,OACf,GAAG,CAAC,CAACE;gBACJ,IAAIC,MAAMD,MAAM,GAAG;gBACnB,IACEC,IAAI,UAAU,CAAC,kBACdA,IAAI,UAAU,CAAC,YAAYA,IAAI,QAAQ,CAAC,WAEzCA,MAAM,GAAGA,IAAI,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC;gBAEpC,OAAO,CAAC,CAAC,EAAED,MAAM,IAAI,CAAC,EAAE,EAAEC,IAAI,CAAC,CAAC;YAClC,GACC,IAAI,CAAC;YAER,OAAO,GAAGJ,OAAO,EAAE,EAAEE,WAAW;QAClC;IACF;IAEA,OAAO;AACT;AAEO,SAASG,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZX,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGW,KAAK,GAAG,EAAEX,QAAQ;IAE9B,OAAOW;AACT;AAEO,SAASC,SAASf,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EAAiB;YACpBiB;QAARD,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,cAACjB,KAAgC,KAAK,AAAD,IAArCiB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAEA,IAAIjB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAGzBkB,cACAC,cACAC;QAJFJ,QACEf,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAqC,KAAK,KACzDA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAAqC,KAAK,AAAD,IAA1CkB,KAAAA,IAAAA,aAA6C,EAAE,AAAD,KAC7ClB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAAoC,KAAK,AAAD,IAAzCmB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDnB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAAwC,KAAK,AAAD,IAA7CoB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAIpB,AAAc,aAAdA,KAAK,IAAI,EAAe;YAKfqB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMtB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAMyB,YAAYvB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXV,QAAQ,GAAIhB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQ1B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAA8B,KAAK,AAAD,IAAnCsB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDN,QAAQR,eAAgBR,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACvB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CgB,QAAQN,aAAcV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADwB,CAAAA,eAACxB,KAA8B,KAAK,AAAD,IAAnCwB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARX,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD2B,CAAAA,eAAC3B,KAA8B,KAAK,AAAD,IAAnC2B,KAAAA,IAAAA,aAAsC,KAAK;QACrD;QAEA,IAAIF,WAEAT,QADEA,QACM,GAAGS,UAAU,GAAG,EAAET,OAAO,GAEzBS;IAGd;IAEA,IAAI,AAAiB,WAAVT,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,SAAsBf,eAAee,QAC9C,OAAOf,eAAee;IAGxB,OAAOY,KAAK,SAAS,CAACZ,OAAOa,QAAW;AAC1C"}

package/dist/es/agent/utils.mjs

@@ -84,14 +84,14 @@ function matchElementFromPlan(planLocateParam, tree) {
         return element;
     }
 }
-async function matchElementFromCache(taskExecutor, cacheEntry, cachePrompt, cacheable) {
-    var _taskExecutor_taskCache;
+async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable) {
+    var _context_taskCache;
     if (!cacheEntry) return;
     if (false === cacheable) return void debug('cache disabled for prompt: %s', cachePrompt);
-    if (!(null == (_taskExecutor_taskCache = taskExecutor.taskCache) ? void 0 : _taskExecutor_taskCache.isCacheResultUsed)) return;
-    if (!taskExecutor.interface.rectMatchesCacheFeature) return void debug('interface does not implement rectMatchesCacheFeature, skip cache');
+    if (!(null == (_context_taskCache = context.taskCache) ? void 0 : _context_taskCache.isCacheResultUsed)) return;
+    if (!context.interfaceInstance.rectMatchesCacheFeature) return void debug('interface does not implement rectMatchesCacheFeature, skip cache');
     try {
-        const rect = await taskExecutor.interface.rectMatchesCacheFeature(cacheEntry);
+        const rect = await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);
         const element = {
             id: uuid(),
             center: [
@@ -143,7 +143,7 @@ function trimContextByViewport(execution) {
         }) : execution.tasks
     };
 }
-const getMidsceneVersion = ()=>"0.30.5";
+const getMidsceneVersion = ()=>"1.0.1-beta-20251021060907.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/agent/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.mjs","sources":["webpack://@midscene/core/./src/agent/utils.ts"],"sourcesContent":["import { elementByPositionWithElementInfo } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type {\n  BaseElement,\n  ElementCacheFeature,\n  ElementTreeNode,\n  ExecutionDump,\n  ExecutionTask,\n  ExecutorContext,\n  LocateResultElement,\n  PlanningLocateParam,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport {\n  generateElementByPosition,\n  getNodeFromCacheList,\n} from '@midscene/shared/extractor';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport { debug as cacheDebug } from './task-cache';\nimport type { TaskExecutor } from './tasks';\n\nconst debugProfile = getDebug('web:tool:profile');\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: { uploadServerUrl?: string },\n): Promise<UIContext> {\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debugProfile('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debugProfile('Interface description end');\n\n  debugProfile('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debugProfile('UploadTestInfoToServer end');\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await interfaceInstance.size();\n  debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);\n\n  return {\n    tree: {\n      node: null,\n      children: [],\n    },\n    size,\n    screenshotBase64: screenshotBase64!,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n  tree: ElementTreeNode<BaseElement>,\n) {\n  if (!planLocateParam) {\n    return undefined;\n  }\n  if (planLocateParam.id) {\n    return getNodeFromCacheList(planLocateParam.id);\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n    let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n    if (!element) {\n      element = generateElementByPosition(centerPosition) as BaseElement;\n    }\n\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  taskExecutor: TaskExecutor,\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!taskExecutor.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!taskExecutor.interface.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await taskExecutor.interface.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      id: uuid(),\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      xpaths: [],\n      attributes: {\n        nodeType: NodeType.POSITION,\n      },\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\nexport function trimContextByViewport(execution: ExecutionDump) {\n  function filterVisibleTree(\n    node: ElementTreeNode<BaseElement>,\n  ): ElementTreeNode<BaseElement> | null {\n    if (!node) return null;\n\n    // recursively process all children\n    const filteredChildren = Array.isArray(node.children)\n      ? (node.children\n          .map(filterVisibleTree)\n          .filter((child) => child !== null) as ElementTreeNode<BaseElement>[])\n      : [];\n\n    // if the current node is visible, keep it and the filtered children\n    if (node.node && node.node.isVisible === true) {\n      return {\n        ...node,\n        children: filteredChildren,\n      };\n    }\n\n    // if the current node is invisible, but has visible children, create an empty node to include these children\n    if (filteredChildren.length > 0) {\n      return {\n        node: null,\n        children: filteredChildren,\n      };\n    }\n\n    // if the current node is invisible and has no visible children, return null\n    return null;\n  }\n\n  return {\n    ...execution,\n    tasks: Array.isArray(execution.tasks)\n      ? execution.tasks.map((task: ExecutionTask) => {\n          const newTask = { ...task };\n          if (task.uiContext?.tree) {\n            newTask.uiContext = {\n              ...task.uiContext,\n              tree: filterVisibleTree(task.uiContext.tree) || {\n                node: null,\n                children: [],\n              },\n            };\n          }\n          return newTask;\n        })\n      : execution.tasks,\n  };\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n"],"names":["debugProfile","getDebug","commonContextParser","interfaceInstance","_opt","assert","description","uploadTestInfoToServer","screenshotBase64","size","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","logMsg","getCurrentExecutionFile","trace","error","Error","stackTrace","pkgDir","process","stackLines","line","match","targetFileName","testFileIndex","Map","generateCacheId","fileName","taskFile","console","currentIndex","undefined","matchElementFromPlan","planLocateParam","tree","getNodeFromCacheList","centerPosition","Math","element","elementByPositionWithElementInfo","generateElementByPosition","matchElementFromCache","taskExecutor","cacheEntry","cachePrompt","cacheable","_taskExecutor_taskCache","cacheDebug","rect","NodeType","trimContextByViewport","execution","filterVisibleTree","node","filteredChildren","Array","child","task","_task_uiContext","newTask","getMidsceneVersion","__VERSION__","parsePrompt","prompt"],"mappings":";;;;;;;;;AAgCA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,oBACpBC,iBAAoC,EACpCC,IAAkC;QAKdD;IAHpBE,OAAOF,mBAAmB;IAE1BH,aAAa;IACb,MAAMM,cAAcH,AAAAA,SAAAA,CAAAA,8BAAAA,kBAAkB,QAAQ,AAAD,IAAzBA,KAAAA,IAAAA,4BAAAA,IAAAA,CAAAA,kBAAiB,KAAiB;IACtDH,aAAa;IAEbA,aAAa;IACbO,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAJ,aAAa;IAEb,MAAMQ,mBAAmB,MAAML,kBAAkB,gBAAgB;IACjEE,OAAOG,kBAAmB;IAE1B,MAAMC,OAAO,MAAMN,kBAAkB,IAAI;IACzCH,aAAa,CAAC,MAAM,EAAES,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,MAAM,EAAEA,KAAK,GAAG,EAAE;IAElE,OAAO;QACL,MAAM;YACJ,MAAM;YACN,UAAU,EAAE;QACd;QACAA;QACA,kBAAkBD;IACpB;AACF;AAEO,SAASE,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7CC,OAAO,CAAC,gCAAgC,EAAED,UAAU;AACtD;AAMO,SAASE,wBAAwBC,KAAc;IACpD,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,aAAaH,SAASC,MAAM,KAAK;IACvC,MAAMG,SAASC,QAAQ,GAAG,MAAM;IAChC,IAAIF,YAAY;QACd,MAAMG,aAAaH,WAAW,KAAK,CAAC;QACpC,KAAK,MAAMI,QAAQD,WACjB,IACEC,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,UACdA,KAAK,QAAQ,CAAC,QACd;YACA,MAAMC,QAAQD,KAAK,KAAK,CAAC;YACzB,IAAIC,QAAAA,QAAAA,KAAAA,IAAAA,KAAO,CAAC,EAAE,EAAE;gBACd,MAAMC,iBAAiBD,KAAK,CAAC,EAAE,CAC5B,OAAO,CAACJ,QAAQ,IAChB,IAAI,GACJ,OAAO,CAAC,OAAO;gBAClB,OAAOK;YACT;QACF;IAEJ;IACA,OAAO;AACT;AAEA,MAAMC,gBAAgB,IAAIC;AAEnB,SAASC,gBAAgBC,QAAiB;IAC/C,IAAIC,WAAWD,YAAYd;IAC3B,IAAI,CAACe,UAAU;QACbA,WAAWnB;QACXoB,QAAQ,IAAI,CACV;IAEJ;IAEA,IAAIL,cAAc,GAAG,CAACI,WAAW;QAC/B,MAAME,eAAeN,cAAc,GAAG,CAACI;QACvC,IAAIE,AAAiBC,WAAjBD,cACFN,cAAc,GAAG,CAACI,UAAUE,eAAe;IAE/C,OACEN,cAAc,GAAG,CAACI,UAAU;IAE9B,OAAO,GAAGA,SAAS,CAAC,EAAEJ,cAAc,GAAG,CAACI,WAAW;AACrD;AAEO,SAASI,qBACdC,eAAoC,EACpCC,IAAkC;IAElC,IAAI,CAACD,iBACH;IAEF,IAAIA,gBAAgB,EAAE,EACpB,OAAOE,qBAAqBF,gBAAgB,EAAE;IAGhD,IAAIA,gBAAgB,IAAI,EAAE;QACxB,MAAMG,iBAAiB;YACrB,GAAGC,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;YACpE,GAAGI,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;QACtE;QACA,IAAIK,UAAUC,iCAAiCL,MAAME;QAErD,IAAI,CAACE,SACHA,UAAUE,0BAA0BJ;QAGtC,OAAOE;IACT;AAGF;AAEO,eAAeG,sBACpBC,YAA0B,EAC1BC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;QAWzBC;IATL,IAAI,CAACH,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBE,MAAW,iCAAiCH;IAI9C,IAAI,UAACE,CAAAA,0BAAAA,aAAa,SAAS,AAAD,IAArBA,KAAAA,IAAAA,wBAAwB,iBAAiB,AAAD,GAC3C;IAGF,IAAI,CAACJ,aAAa,SAAS,CAAC,uBAAuB,EAAE,YACnDK,MACE;IAKJ,IAAI;QACF,MAAMC,OACJ,MAAMN,aAAa,SAAS,CAAC,uBAAuB,CAACC;QACvD,MAAML,UAA+B;YACnC,IAAI7B;YACJ,QAAQ;gBACN4B,KAAK,KAAK,CAACW,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCX,KAAK,KAAK,CAACW,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,QAAQ,EAAE;YACV,YAAY;gBACV,UAAUC,SAAS,QAAQ;YAC7B;QACF;QAEAF,MAAW,yBAAyBH;QACpC,OAAON;IACT,EAAE,OAAOvB,OAAO;QACdgC,MAAW,qCAAqChC;QAChD;IACF;AACF;AAEO,SAASmC,sBAAsBC,SAAwB;IAC5D,SAASC,kBACPC,IAAkC;QAElC,IAAI,CAACA,MAAM,OAAO;QAGlB,MAAMC,mBAAmBC,MAAM,OAAO,CAACF,KAAK,QAAQ,IAC/CA,KAAK,QAAQ,CACX,GAAG,CAACD,mBACJ,MAAM,CAAC,CAACI,QAAUA,AAAU,SAAVA,SACrB,EAAE;QAGN,IAAIH,KAAK,IAAI,IAAIA,AAAwB,SAAxBA,KAAK,IAAI,CAAC,SAAS,EAClC,OAAO;YACL,GAAGA,IAAI;YACP,UAAUC;QACZ;QAIF,IAAIA,iBAAiB,MAAM,GAAG,GAC5B,OAAO;YACL,MAAM;YACN,UAAUA;QACZ;QAIF,OAAO;IACT;IAEA,OAAO;QACL,GAAGH,SAAS;QACZ,OAAOI,MAAM,OAAO,CAACJ,UAAU,KAAK,IAChCA,UAAU,KAAK,CAAC,GAAG,CAAC,CAACM;gBAEfC;YADJ,MAAMC,UAAU;gBAAE,GAAGF,IAAI;YAAC;YAC1B,IAAI,QAAAC,CAAAA,kBAAAA,KAAK,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,IAAI,EACtBC,QAAQ,SAAS,GAAG;gBAClB,GAAGF,KAAK,SAAS;gBACjB,MAAML,kBAAkBK,KAAK,SAAS,CAAC,IAAI,KAAK;oBAC9C,MAAM;oBACN,UAAU,EAAE;gBACd;YACF;YAEF,OAAOE;QACT,KACAR,UAAU,KAAK;IACrB;AACF;AAIO,MAAMS,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBhC;IACpB;IAEF,OAAO;QACL,YAAYgC,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAhC;IACN;AACF"}
+{"version":3,"file":"agent/utils.mjs","sources":["webpack://@midscene/core/./src/agent/utils.ts"],"sourcesContent":["import { elementByPositionWithElementInfo } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type {\n  BaseElement,\n  ElementCacheFeature,\n  ElementTreeNode,\n  ExecutionDump,\n  ExecutionTask,\n  ExecutorContext,\n  LocateResultElement,\n  PlanningLocateParam,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport { NodeType } from '@midscene/shared/constants';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport {\n  generateElementByPosition,\n  getNodeFromCacheList,\n} from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst debugProfile = getDebug('web:tool:profile');\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: { uploadServerUrl?: string },\n): Promise<UIContext> {\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debugProfile('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debugProfile('Interface description end');\n\n  debugProfile('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debugProfile('UploadTestInfoToServer end');\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await interfaceInstance.size();\n  debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);\n\n  return {\n    tree: {\n      node: null,\n      children: [],\n    },\n    size,\n    screenshotBase64: screenshotBase64!,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n  tree: ElementTreeNode<BaseElement>,\n) {\n  if (!planLocateParam) {\n    return undefined;\n  }\n  if (planLocateParam.id) {\n    return getNodeFromCacheList(planLocateParam.id);\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n    let element = elementByPositionWithElementInfo(tree, centerPosition);\n\n    if (!element) {\n      element = generateElementByPosition(centerPosition) as BaseElement;\n    }\n\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      id: uuid(),\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      xpaths: [],\n      attributes: {\n        nodeType: NodeType.POSITION,\n      },\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\nexport function trimContextByViewport(execution: ExecutionDump) {\n  function filterVisibleTree(\n    node: ElementTreeNode<BaseElement>,\n  ): ElementTreeNode<BaseElement> | null {\n    if (!node) return null;\n\n    // recursively process all children\n    const filteredChildren = Array.isArray(node.children)\n      ? (node.children\n          .map(filterVisibleTree)\n          .filter((child) => child !== null) as ElementTreeNode<BaseElement>[])\n      : [];\n\n    // if the current node is visible, keep it and the filtered children\n    if (node.node && node.node.isVisible === true) {\n      return {\n        ...node,\n        children: filteredChildren,\n      };\n    }\n\n    // if the current node is invisible, but has visible children, create an empty node to include these children\n    if (filteredChildren.length > 0) {\n      return {\n        node: null,\n        children: filteredChildren,\n      };\n    }\n\n    // if the current node is invisible and has no visible children, return null\n    return null;\n  }\n\n  return {\n    ...execution,\n    tasks: Array.isArray(execution.tasks)\n      ? execution.tasks.map((task: ExecutionTask) => {\n          const newTask = { ...task };\n          if (task.uiContext?.tree) {\n            newTask.uiContext = {\n              ...task.uiContext,\n              tree: filterVisibleTree(task.uiContext.tree) || {\n                node: null,\n                children: [],\n              },\n            };\n          }\n          return newTask;\n        })\n      : execution.tasks,\n  };\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n"],"names":["debugProfile","getDebug","commonContextParser","interfaceInstance","_opt","assert","description","uploadTestInfoToServer","screenshotBase64","size","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","logMsg","getCurrentExecutionFile","trace","error","Error","stackTrace","pkgDir","process","stackLines","line","match","targetFileName","testFileIndex","Map","generateCacheId","fileName","taskFile","console","currentIndex","undefined","matchElementFromPlan","planLocateParam","tree","getNodeFromCacheList","centerPosition","Math","element","elementByPositionWithElementInfo","generateElementByPosition","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","_context_taskCache","cacheDebug","rect","NodeType","trimContextByViewport","execution","filterVisibleTree","node","filteredChildren","Array","child","task","_task_uiContext","newTask","getMidsceneVersion","__VERSION__","parsePrompt","prompt"],"mappings":";;;;;;;;;AA+BA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,oBACpBC,iBAAoC,EACpCC,IAAkC;QAKdD;IAHpBE,OAAOF,mBAAmB;IAE1BH,aAAa;IACb,MAAMM,cAAcH,AAAAA,SAAAA,CAAAA,8BAAAA,kBAAkB,QAAQ,AAAD,IAAzBA,KAAAA,IAAAA,4BAAAA,IAAAA,CAAAA,kBAAiB,KAAiB;IACtDH,aAAa;IAEbA,aAAa;IACbO,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAJ,aAAa;IAEb,MAAMQ,mBAAmB,MAAML,kBAAkB,gBAAgB;IACjEE,OAAOG,kBAAmB;IAE1B,MAAMC,OAAO,MAAMN,kBAAkB,IAAI;IACzCH,aAAa,CAAC,MAAM,EAAES,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,MAAM,EAAEA,KAAK,GAAG,EAAE;IAElE,OAAO;QACL,MAAM;YACJ,MAAM;YACN,UAAU,EAAE;QACd;QACAA;QACA,kBAAkBD;IACpB;AACF;AAEO,SAASE,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7CC,OAAO,CAAC,gCAAgC,EAAED,UAAU;AACtD;AAMO,SAASE,wBAAwBC,KAAc;IACpD,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,aAAaH,SAASC,MAAM,KAAK;IACvC,MAAMG,SAASC,QAAQ,GAAG,MAAM;IAChC,IAAIF,YAAY;QACd,MAAMG,aAAaH,WAAW,KAAK,CAAC;QACpC,KAAK,MAAMI,QAAQD,WACjB,IACEC,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,UACdA,KAAK,QAAQ,CAAC,QACd;YACA,MAAMC,QAAQD,KAAK,KAAK,CAAC;YACzB,IAAIC,QAAAA,QAAAA,KAAAA,IAAAA,KAAO,CAAC,EAAE,EAAE;gBACd,MAAMC,iBAAiBD,KAAK,CAAC,EAAE,CAC5B,OAAO,CAACJ,QAAQ,IAChB,IAAI,GACJ,OAAO,CAAC,OAAO;gBAClB,OAAOK;YACT;QACF;IAEJ;IACA,OAAO;AACT;AAEA,MAAMC,gBAAgB,IAAIC;AAEnB,SAASC,gBAAgBC,QAAiB;IAC/C,IAAIC,WAAWD,YAAYd;IAC3B,IAAI,CAACe,UAAU;QACbA,WAAWnB;QACXoB,QAAQ,IAAI,CACV;IAEJ;IAEA,IAAIL,cAAc,GAAG,CAACI,WAAW;QAC/B,MAAME,eAAeN,cAAc,GAAG,CAACI;QACvC,IAAIE,AAAiBC,WAAjBD,cACFN,cAAc,GAAG,CAACI,UAAUE,eAAe;IAE/C,OACEN,cAAc,GAAG,CAACI,UAAU;IAE9B,OAAO,GAAGA,SAAS,CAAC,EAAEJ,cAAc,GAAG,CAACI,WAAW;AACrD;AAEO,SAASI,qBACdC,eAAoC,EACpCC,IAAkC;IAElC,IAAI,CAACD,iBACH;IAEF,IAAIA,gBAAgB,EAAE,EACpB,OAAOE,qBAAqBF,gBAAgB,EAAE;IAGhD,IAAIA,gBAAgB,IAAI,EAAE;QACxB,MAAMG,iBAAiB;YACrB,GAAGC,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;YACpE,GAAGI,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;QACtE;QACA,IAAIK,UAAUC,iCAAiCL,MAAME;QAErD,IAAI,CAACE,SACHA,UAAUE,0BAA0BJ;QAGtC,OAAOE;IACT;AAGF;AAEO,eAAeG,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;QAWzBC;IATL,IAAI,CAACH,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBE,MAAW,iCAAiCH;IAI9C,IAAI,UAACE,CAAAA,qBAAAA,QAAQ,SAAS,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,iBAAiB,AAAD,GACtC;IAGF,IAAI,CAACJ,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDK,MACE;IAKJ,IAAI;QACF,MAAMC,OACJ,MAAMN,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAML,UAA+B;YACnC,IAAI7B;YACJ,QAAQ;gBACN4B,KAAK,KAAK,CAACW,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCX,KAAK,KAAK,CAACW,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,QAAQ,EAAE;YACV,YAAY;gBACV,UAAUC,SAAS,QAAQ;YAC7B;QACF;QAEAF,MAAW,yBAAyBH;QACpC,OAAON;IACT,EAAE,OAAOvB,OAAO;QACdgC,MAAW,qCAAqChC;QAChD;IACF;AACF;AAEO,SAASmC,sBAAsBC,SAAwB;IAC5D,SAASC,kBACPC,IAAkC;QAElC,IAAI,CAACA,MAAM,OAAO;QAGlB,MAAMC,mBAAmBC,MAAM,OAAO,CAACF,KAAK,QAAQ,IAC/CA,KAAK,QAAQ,CACX,GAAG,CAACD,mBACJ,MAAM,CAAC,CAACI,QAAUA,AAAU,SAAVA,SACrB,EAAE;QAGN,IAAIH,KAAK,IAAI,IAAIA,AAAwB,SAAxBA,KAAK,IAAI,CAAC,SAAS,EAClC,OAAO;YACL,GAAGA,IAAI;YACP,UAAUC;QACZ;QAIF,IAAIA,iBAAiB,MAAM,GAAG,GAC5B,OAAO;YACL,MAAM;YACN,UAAUA;QACZ;QAIF,OAAO;IACT;IAEA,OAAO;QACL,GAAGH,SAAS;QACZ,OAAOI,MAAM,OAAO,CAACJ,UAAU,KAAK,IAChCA,UAAU,KAAK,CAAC,GAAG,CAAC,CAACM;gBAEfC;YADJ,MAAMC,UAAU;gBAAE,GAAGF,IAAI;YAAC;YAC1B,IAAI,QAAAC,CAAAA,kBAAAA,KAAK,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,IAAI,EACtBC,QAAQ,SAAS,GAAG;gBAClB,GAAGF,KAAK,SAAS;gBACjB,MAAML,kBAAkBK,KAAK,SAAS,CAAC,IAAI,KAAK;oBAC9C,MAAM;oBACN,UAAU,EAAE;gBACd;YACF;YAEF,OAAOE;QACT,KACAR,UAAU,KAAK;IACrB;AACF;AAIO,MAAMS,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBhC;IACpB;IAEF,OAAO;QACL,YAAYgC,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAhC;IACN;AACF"}

package/dist/es/ai-model/common.mjs

@@ -134,20 +134,6 @@ function adaptBboxToRect(bbox, width, height, offsetX = 0, offsetY = 0, rightLim
     debugInspectUtils('adaptBboxToRect, result=', rect);
     return rect;
 }
-let warned = false;
-function warnGPT4oSizeLimit(size, modelName) {
-    if (warned) return;
-    if (modelName.toLowerCase().includes('gpt-4o')) {
-        const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your interface to a smaller resolution. Otherwise, the result may be inaccurate.`;
-        if (Math.max(size.width, size.height) > 2000 || Math.min(size.width, size.height) > 768) {
-            console.warn(warningMsg);
-            warned = true;
-        }
-    } else if (size.width > 1800 || size.height > 1800) {
-        console.warn(`The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`);
-        warned = true;
-    }
-}
 function mergeRects(rects) {
     const minLeft = Math.min(...rects.map((r)=>r.left));
     const minTop = Math.min(...rects.map((r)=>r.top));
@@ -350,6 +336,6 @@ const parseActionParam = (rawParam, zodSchema)=>{
     for(const fieldName in locateFieldValues)validated[fieldName] = locateFieldValues[fieldName];
     return validated;
 };
-export { common_AIActionType as AIActionType, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptQwenBbox, buildYamlFlowFromPlans, dumpActionParam, dumpMidsceneLocatorField, expandSearchArea, fillBboxParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, ifMidsceneLocatorField, loadActionParam, markupImageForLLM, mergeRects, normalized01000, parseActionParam, warnGPT4oSizeLimit };
+export { common_AIActionType as AIActionType, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptQwenBbox, buildYamlFlowFromPlans, dumpActionParam, dumpMidsceneLocatorField, expandSearchArea, fillBboxParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, ifMidsceneLocatorField, loadActionParam, markupImageForLLM, mergeRects, normalized01000, parseActionParam };
 
 //# sourceMappingURL=common.mjs.map

package/dist/es/ai-model/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n  TEXT = 5,\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(\n      locate.bbox,\n      width,\n      height,\n      rightLimit,\n      bottomLimit,\n      vlMode,\n    );\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n): [number, number, number, number] {\n  let result: [number, number, number, number] = [0, 0, 0, 0];\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    result = adaptDoubaoBbox(bbox, width, height);\n  } else if (vlMode === 'gemini') {\n    result = adaptGeminiBbox(bbox, width, height);\n  } else if (vlMode === 'qwen3-vl') {\n    result = normalized01000(bbox, width, height);\n  } else {\n    result = adaptQwenBbox(bbox);\n  }\n\n  result[2] = Math.min(result[2], rightLimit);\n  result[3] = Math.min(result[3], bottomLimit);\n\n  return result;\n}\n\n// x1, y1, x2, y2 -> 0-1000\nexport function normalized01000(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  return [\n    Math.round((bbox[0] * width) / 1000),\n    Math.round((bbox[1] * height) / 1000),\n    Math.round((bbox[2] * width) / 1000),\n    Math.round((bbox[3] * height) / 1000),\n  ];\n}\n\n// y1, x1, y2, x2 -> 0-1000\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n  rightLimit = width,\n  bottomLimit = height,\n  vlMode?: TVlModeTypes | undefined,\n): Rect {\n  debugInspectUtils(\n    'adaptBboxToRect',\n    bbox,\n    width,\n    height,\n    'offset',\n    offsetX,\n    offsetY,\n    'limit',\n    rightLimit,\n    bottomLimit,\n    'vlMode',\n    vlMode,\n  );\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    rightLimit,\n    bottomLimit,\n    vlMode,\n  );\n\n  // Calculate initial rect dimensions\n  const rectLeft = left;\n  const rectTop = top;\n  let rectWidth = right - left;\n  let rectHeight = bottom - top;\n\n  // Ensure the rect doesn't exceed image boundaries\n  // If right edge exceeds width, adjust the width\n  if (rectLeft + rectWidth > width) {\n    rectWidth = width - rectLeft;\n  }\n\n  // If bottom edge exceeds height, adjust the height\n  if (rectTop + rectHeight > height) {\n    rectHeight = height - rectTop;\n  }\n\n  // Ensure minimum dimensions (width and height should be at least 1)\n  rectWidth = Math.max(1, rectWidth);\n  rectHeight = Math.max(1, rectHeight);\n\n  const rect = {\n    left: rectLeft + offsetX,\n    top: rectTop + offsetY,\n    width: rectWidth,\n    height: rectHeight,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(size: Size, modelName: string) {\n  if (warned) return;\n  if (modelName.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your interface to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  vlMode: TVlModeTypes | undefined,\n) {\n  const minEdgeSize = vlMode === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  // Calculate padding needed to reach minimum edge size\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n\n  // Calculate new dimensions (ensure minimum edge size)\n  let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n  let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n  // Calculate initial position with padding\n  let newLeft = rect.left - paddingSizeHorizontal;\n  let newTop = rect.top - paddingSizeVertical;\n\n  // Ensure the rect doesn't exceed screen boundaries by adjusting position\n  // If the rect goes beyond the right edge, shift it left\n  if (newLeft + newWidth > screenSize.width) {\n    newLeft = screenSize.width - newWidth;\n  }\n\n  // If the rect goes beyond the bottom edge, shift it up\n  if (newTop + newHeight > screenSize.height) {\n    newTop = screenSize.height - newHeight;\n  }\n\n  // Ensure the rect doesn't go beyond the left/top edges\n  newLeft = Math.max(0, newLeft);\n  newTop = Math.max(0, newTop);\n\n  // If after position adjustment, the rect still exceeds screen boundaries,\n  // clamp the dimensions to fit within screen\n  if (newLeft + newWidth > screenSize.width) {\n    newWidth = screenSize.width - newLeft;\n  }\n  if (newTop + newHeight > screenSize.height) {\n    newHeight = screenSize.height - newTop;\n  }\n\n  rect.left = newLeft;\n  rect.top = newTop;\n  rect.width = newWidth;\n  rect.height = newHeight;\n\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepThink: z.boolean().optional(),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n// Schema for locator field result (when AI returns locate results)\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\n// Export the result type - this is used for runtime results that include center and rect\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\n\n// Export the input type - this is the inferred type from getMidsceneLocationSchema()\nexport type MidsceneLocationInputType = z.infer<typeof MidsceneLocationInput>;\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Method 1: Check for the location field flag (for result schema)\n    if (locateFieldFlagName in shape) {\n      return true;\n    }\n\n    // Method 2: Check if it's the input schema by checking for 'prompt' field\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement)\n  for (const fieldName in locateFieldValues) {\n    validated[fieldName] = locateFieldValues[fieldName];\n  }\n\n  return validated;\n};\n"],"names":["AIActionType","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","rightLimit","bottomLimit","vlMode","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","adaptGeminiBbox","normalized01000","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","warned","warnGPT4oSizeLimit","size","modelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","_actualField__def","_actualField__def1","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","_zodObject__def","zodObject","keys","Object","key","_field__def","jsonObject","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam","parseActionParam","rawParam","param","locateFields","locateFieldValues","paramsForValidation","validated"],"mappings":";;;;;;AAwBO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;;WAAZA;;AASZ,MAAMC,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAGhC,IAAKL,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGM,UACZN,OAAO,IAAI,EACXC,OACAC,QACAC,YACAC,aACAC;IAIJ,OAAOL;AACT;AAEO,SAASO,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;QACN,YAAnB,OAAOY,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;KAC1B;IACD,OAAOgB;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCP,KAAa,EACbC,MAAc;IAEda,OACEd,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOM,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;YAC5CW,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;SAC7C;QAEH,MAAM,IAAIS,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;QACpCW,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;KACrC;IAIH,IACEiB,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU,QAAQN,kBAAkB;QAEhEiB,KAAK,GAAG,CACNZ,OACAY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACNX,QACAW,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIY,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;QACpCW,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;KACrC;IAGH,MAAMO,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdP,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAEhC,IAAIO,SAA2C;QAAC;QAAG;QAAG;QAAG;KAAE;IAEzDA,SADEP,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,SACvBS,gBAAgBN,MAAMP,OAAOC,UAC7BG,AAAW,aAAXA,SACAkB,gBAAgBf,MAAMP,OAAOC,UAC7BG,AAAW,eAAXA,SACAmB,gBAAgBhB,MAAMP,OAAOC,UAE7BK,cAAcC;IAGzBI,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAET;IAChCS,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAER;IAEhC,OAAOQ;AACT;AAGO,SAASY,gBACdhB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,OAAO;QACLW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;QAChCW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;KACjC;AACH;AAGO,SAASqB,gBACdf,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,MAAMuB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC5C,MAAMyB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC5C,MAAMyB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC7C,MAAM2B,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC/C,OAAO;QAACuB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdP,KAAa,EACbC,MAAc,EACd4B,UAAU,CAAC,EACXC,UAAU,CAAC,EACX5B,aAAaF,KAAK,EAClBG,cAAcF,MAAM,EACpBG,MAAiC;IAEjCR,kBACE,mBACAW,MACAP,OACAC,QACA,UACA4B,SACAC,SACA,SACA5B,YACAC,aACA,UACAC;IAEF,MAAM,CAACoB,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UACjCE,MACAP,OACAC,QACAC,YACAC,aACAC;IAIF,MAAM2B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAYjC,OACzBiC,YAAYjC,QAAQ+B;IAItB,IAAIC,UAAUE,aAAajC,QACzBiC,aAAajC,SAAS+B;IAIxBC,YAAYrB,KAAK,GAAG,CAAC,GAAGqB;IACxBC,aAAatB,KAAK,GAAG,CAAC,GAAGsB;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACAtC,kBAAkB,4BAA4BuC;IAE9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBAAmBC,IAAU,EAAEC,SAAiB;IAC9D,IAAIH,QAAQ;IACZ,IAAIG,UAAU,WAAW,GAAG,QAAQ,CAAC,WAAW;QAC9C,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,6FAA6F,CAAC;QAErN,IACE1B,KAAK,GAAG,CAAC0B,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpC1B,KAAK,GAAG,CAAC0B,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAUhC,KAAK,GAAG,IAAI+B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAASlC,KAAK,GAAG,IAAI+B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAWnC,KAAK,GAAG,IAAI+B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYpC,KAAK,GAAG,IAAI+B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdd,IAAU,EACVe,UAAgB,EAChB9C,MAAgC;IAEhC,MAAM+C,cAAc/C,AAAW,oBAAXA,SAA6B,MAAM;IACvD,MAAMgD,iBAAiB;IAGvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTvC,KAAK,IAAI,CAAEuC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVvC,KAAK,IAAI,CAAEuC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IAGN,IAAIG,WAAW3C,KAAK,GAAG,CAACuC,aAAahB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA;IAClD,IAAIG,YAAY5C,KAAK,GAAG,CAACuC,aAAahB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUtB,KAAK,IAAI,GAAGkB;IAC1B,IAAIK,SAASvB,KAAK,GAAG,GAAGmB;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAU7C,KAAK,GAAG,CAAC,GAAG6C;IACtBC,SAAS9C,KAAK,GAAG,CAAC,GAAG8C;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCvB,KAAK,IAAI,GAAGsB;IACZtB,KAAK,GAAG,GAAGuB;IACXvB,KAAK,KAAK,GAAGoB;IACbpB,KAAK,MAAM,GAAGqB;IAEd,OAAOrB;AACT;AAEO,eAAewB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCvB,IAAU;IAEV,MAAMwB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtB1B;IACF;IACA,OAAO6B;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXnC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEkC,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAME,UAAUD,OAAO,cAAc,IAAID;QACzC,MAAMG,YAAYF,OAAO,WAAW,GAChCG,gBAAgBL,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMI,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAL,KAAK,IAAI,CAACO;IACZ;IAEA,IAAIR,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMQ,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAGiBA,EAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,EAAE,OAAO,CAAC;IACjC,QAAQI;IAGR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,OAAO,GAAG,QAAQ;IAG3B,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAYP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;QAGjCC,mBAKAC;IANJ,IAAIC,cAAcH;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,oBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,QAAQ,AAAD,MAAM,eACjCE,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAID,AAAAA,SAAAA,CAAAA,qBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,mBAAkB,QAAQ,AAAD,MAAM,aAAa;QAC9C,MAAME,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIP,uBAAuBQ,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACL;IACvC7E,OACE4E,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOM,OAAON;AAChB;AAEO,MAAMO,8BAA8B,CACzCC,SACAC;QAQIC;IANJ,IAAI,CAACF,SACH,OAAO,EAAE;IAIX,MAAMG,YAAYH;IAClB,IAAIE,AAAAA,SAAAA,CAAAA,kBAAAA,UAAU,IAAI,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,QAAQ,AAAD,MAAM,eAAeC,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE;YAClB,MAAMd,QAAQW,UAAU,KAAK,CAACG,IAAI;YAClC,IAAI,CAACf,uBAAuBC,QAC1B,OAAO;YAIT,IAAIS,cAAc;oBACTM;gBAAP,OAAOA,AAAAA,SAAAA,CAAAA,cAAAA,MAAM,IAAI,AAAD,IAATA,KAAAA,IAAAA,YAAY,QAAQ,AAAD,MAAM;YAClC;YAEA,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAM3B,kBAAkB,CAC7B4B,YACAC;IAEA,MAAMC,gBAAgBX,4BAA4BU;IAClD,MAAMjG,SAAS;QAAE,GAAGgG,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAapG,MAAM,CAACmG,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTpG,MAAM,CAACmG,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BpG,MAAM,CAACmG,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBpG,MAAM,CAACmG,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAOpG;AACT;AAEO,MAAMqG,kBAAkB,CAC7BL,YACAC;IAEA,MAAMC,gBAAgBX,4BAA4BU;IAClD,MAAMjG,SAAS;QAAE,GAAGgG,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAapG,MAAM,CAACmG,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvBpG,MAAM,CAACmG,UAAU,GAAG;YAClB,CAACvB,oBAAoB,EAAE;YACvB,QAAQwB;QACV;IAEJ;IAEA,OAAOpG;AACT;AAUO,MAAMsG,mBAAmB,CAC9BC,UACAN;IAGA,MAAMO,QAAQD,YAAY,CAAC;IAG3B,MAAME,eAAelB,4BAA4BU;IAGjD,IAAIQ,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOR,UAAU,KAAK,CAACO;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMb,OAAOU,MAChB,IAAIC,aAAa,QAAQ,CAACX,MAExBa,mBAAmB,CAACb,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/Ca,mBAAmB,CAACb,IAAI,GAAGU,KAAK,CAACV,IAAI;IAKzC,MAAMc,YAAYX,UAAU,KAAK,CAACU;IAGlC,IAAK,MAAMR,aAAaO,kBACtBE,SAAS,CAACT,UAAU,GAAGO,iBAAiB,CAACP,UAAU;IAGrD,OAAOS;AACT"}
+{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n  TEXT = 5,\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(\n      locate.bbox,\n      width,\n      height,\n      rightLimit,\n      bottomLimit,\n      vlMode,\n    );\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n): [number, number, number, number] {\n  let result: [number, number, number, number] = [0, 0, 0, 0];\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    result = adaptDoubaoBbox(bbox, width, height);\n  } else if (vlMode === 'gemini') {\n    result = adaptGeminiBbox(bbox, width, height);\n  } else if (vlMode === 'qwen3-vl') {\n    result = normalized01000(bbox, width, height);\n  } else {\n    result = adaptQwenBbox(bbox);\n  }\n\n  result[2] = Math.min(result[2], rightLimit);\n  result[3] = Math.min(result[3], bottomLimit);\n\n  return result;\n}\n\n// x1, y1, x2, y2 -> 0-1000\nexport function normalized01000(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  return [\n    Math.round((bbox[0] * width) / 1000),\n    Math.round((bbox[1] * height) / 1000),\n    Math.round((bbox[2] * width) / 1000),\n    Math.round((bbox[3] * height) / 1000),\n  ];\n}\n\n// y1, x1, y2, x2 -> 0-1000\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n  rightLimit = width,\n  bottomLimit = height,\n  vlMode?: TVlModeTypes | undefined,\n): Rect {\n  debugInspectUtils(\n    'adaptBboxToRect',\n    bbox,\n    width,\n    height,\n    'offset',\n    offsetX,\n    offsetY,\n    'limit',\n    rightLimit,\n    bottomLimit,\n    'vlMode',\n    vlMode,\n  );\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    rightLimit,\n    bottomLimit,\n    vlMode,\n  );\n\n  // Calculate initial rect dimensions\n  const rectLeft = left;\n  const rectTop = top;\n  let rectWidth = right - left;\n  let rectHeight = bottom - top;\n\n  // Ensure the rect doesn't exceed image boundaries\n  // If right edge exceeds width, adjust the width\n  if (rectLeft + rectWidth > width) {\n    rectWidth = width - rectLeft;\n  }\n\n  // If bottom edge exceeds height, adjust the height\n  if (rectTop + rectHeight > height) {\n    rectHeight = height - rectTop;\n  }\n\n  // Ensure minimum dimensions (width and height should be at least 1)\n  rectWidth = Math.max(1, rectWidth);\n  rectHeight = Math.max(1, rectHeight);\n\n  const rect = {\n    left: rectLeft + offsetX,\n    top: rectTop + offsetY,\n    width: rectWidth,\n    height: rectHeight,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n\n  return rect;\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  vlMode: TVlModeTypes | undefined,\n) {\n  const minEdgeSize = vlMode === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  // Calculate padding needed to reach minimum edge size\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n\n  // Calculate new dimensions (ensure minimum edge size)\n  let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n  let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n  // Calculate initial position with padding\n  let newLeft = rect.left - paddingSizeHorizontal;\n  let newTop = rect.top - paddingSizeVertical;\n\n  // Ensure the rect doesn't exceed screen boundaries by adjusting position\n  // If the rect goes beyond the right edge, shift it left\n  if (newLeft + newWidth > screenSize.width) {\n    newLeft = screenSize.width - newWidth;\n  }\n\n  // If the rect goes beyond the bottom edge, shift it up\n  if (newTop + newHeight > screenSize.height) {\n    newTop = screenSize.height - newHeight;\n  }\n\n  // Ensure the rect doesn't go beyond the left/top edges\n  newLeft = Math.max(0, newLeft);\n  newTop = Math.max(0, newTop);\n\n  // If after position adjustment, the rect still exceeds screen boundaries,\n  // clamp the dimensions to fit within screen\n  if (newLeft + newWidth > screenSize.width) {\n    newWidth = screenSize.width - newLeft;\n  }\n  if (newTop + newHeight > screenSize.height) {\n    newHeight = screenSize.height - newTop;\n  }\n\n  rect.left = newLeft;\n  rect.top = newTop;\n  rect.width = newWidth;\n  rect.height = newHeight;\n\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepThink: z.boolean().optional(),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n// Schema for locator field result (when AI returns locate results)\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\n// Export the result type - this is used for runtime results that include center and rect\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\n\n// Export the input type - this is the inferred type from getMidsceneLocationSchema()\nexport type MidsceneLocationInputType = z.infer<typeof MidsceneLocationInput>;\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Method 1: Check for the location field flag (for result schema)\n    if (locateFieldFlagName in shape) {\n      return true;\n    }\n\n    // Method 2: Check if it's the input schema by checking for 'prompt' field\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement)\n  for (const fieldName in locateFieldValues) {\n    validated[fieldName] = locateFieldValues[fieldName];\n  }\n\n  return validated;\n};\n"],"names":["AIActionType","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","rightLimit","bottomLimit","vlMode","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","adaptGeminiBbox","normalized01000","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","console","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","_actualField__def","_actualField__def1","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","_zodObject__def","zodObject","keys","Object","key","_field__def","jsonObject","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam","parseActionParam","rawParam","param","locateFields","locateFieldValues","paramsForValidation","validated"],"mappings":";;;;;;AAwBO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;;WAAZA;;AASZ,MAAMC,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAGhC,IAAKL,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGM,UACZN,OAAO,IAAI,EACXC,OACAC,QACAC,YACAC,aACAC;IAIJ,OAAOL;AACT;AAEO,SAASO,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;QACN,YAAnB,OAAOY,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;KAC1B;IACD,OAAOgB;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCP,KAAa,EACbC,MAAc;IAEda,OACEd,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOM,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;YAC5CW,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;SAC7C;QAEH,MAAM,IAAIS,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;QACpCW,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;KACrC;IAIH,IACEiB,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU,QAAQN,kBAAkB;QAEhEiB,KAAK,GAAG,CACNZ,OACAY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACNX,QACAW,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIY,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;QACpCW,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;KACrC;IAGH,MAAMO,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdP,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAEhC,IAAIO,SAA2C;QAAC;QAAG;QAAG;QAAG;KAAE;IAEzDA,SADEP,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,SACvBS,gBAAgBN,MAAMP,OAAOC,UAC7BG,AAAW,aAAXA,SACAkB,gBAAgBf,MAAMP,OAAOC,UAC7BG,AAAW,eAAXA,SACAmB,gBAAgBhB,MAAMP,OAAOC,UAE7BK,cAAcC;IAGzBI,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAET;IAChCS,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAER;IAEhC,OAAOQ;AACT;AAGO,SAASY,gBACdhB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,OAAO;QACLW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;QAChCW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;KACjC;AACH;AAGO,SAASqB,gBACdf,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,MAAMuB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC5C,MAAMyB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC5C,MAAMyB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC7C,MAAM2B,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC/C,OAAO;QAACuB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdP,KAAa,EACbC,MAAc,EACd4B,UAAU,CAAC,EACXC,UAAU,CAAC,EACX5B,aAAaF,KAAK,EAClBG,cAAcF,MAAM,EACpBG,MAAiC;IAEjCR,kBACE,mBACAW,MACAP,OACAC,QACA,UACA4B,SACAC,SACA,SACA5B,YACAC,aACA,UACAC;IAEF,MAAM,CAACoB,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UACjCE,MACAP,OACAC,QACAC,YACAC,aACAC;IAIF,MAAM2B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAYjC,OACzBiC,YAAYjC,QAAQ+B;IAItB,IAAIC,UAAUE,aAAajC,QACzBiC,aAAajC,SAAS+B;IAIxBC,YAAYrB,KAAK,GAAG,CAAC,GAAGqB;IACxBC,aAAatB,KAAK,GAAG,CAAC,GAAGsB;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACAtC,kBAAkB,4BAA4BuC;IAE9C,OAAOA;AACT;AAEO,SAASC,WAAWC,KAAa;IACtC,MAAMC,UAAU1B,KAAK,GAAG,IAAIyB,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS5B,KAAK,GAAG,IAAIyB,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW7B,KAAK,GAAG,IAAIyB,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAY9B,KAAK,GAAG,IAAIyB,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdR,IAAU,EACVS,UAAgB,EAChBxC,MAAgC;IAEhC,MAAMyC,cAAczC,AAAW,oBAAXA,SAA6B,MAAM;IACvD,MAAM0C,iBAAiB;IAGvB,MAAMC,wBACJZ,KAAK,KAAK,GAAGU,cACTjC,KAAK,IAAI,CAAEiC,AAAAA,CAAAA,cAAcV,KAAK,KAAI,IAAK,KACvCW;IACN,MAAME,sBACJb,KAAK,MAAM,GAAGU,cACVjC,KAAK,IAAI,CAAEiC,AAAAA,CAAAA,cAAcV,KAAK,MAAK,IAAK,KACxCW;IAGN,IAAIG,WAAWrC,KAAK,GAAG,CAACiC,aAAaV,KAAK,KAAK,GAAGY,AAAwB,IAAxBA;IAClD,IAAIG,YAAYtC,KAAK,GAAG,CAACiC,aAAaV,KAAK,MAAM,GAAGa,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUhB,KAAK,IAAI,GAAGY;IAC1B,IAAIK,SAASjB,KAAK,GAAG,GAAGa;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAUvC,KAAK,GAAG,CAAC,GAAGuC;IACtBC,SAASxC,KAAK,GAAG,CAAC,GAAGwC;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCjB,KAAK,IAAI,GAAGgB;IACZhB,KAAK,GAAG,GAAGiB;IACXjB,KAAK,KAAK,GAAGc;IACbd,KAAK,MAAM,GAAGe;IAEd,OAAOf;AACT;AAEO,eAAekB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,WAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEF,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMG,UAAUF,OAAO,cAAc,IAAID;QACzC,MAAMI,YAAYH,OAAO,WAAW,GAChCI,gBAAgBN,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMK,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAN,KAAK,IAAI,CAACQ;IACZ;IAEA,IAAIT,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMS,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAGiBA,EAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,EAAE,OAAO,CAAC;IACjC,QAAQI;IAGR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,OAAO,GAAG,QAAQ;IAG3B,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAYP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;QAGjCC,mBAKAC;IANJ,IAAIC,cAAcH;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,oBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,QAAQ,AAAD,MAAM,eACjCE,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAID,AAAAA,SAAAA,CAAAA,qBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,mBAAkB,QAAQ,AAAD,MAAM,aAAa;QAC9C,MAAME,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIP,uBAAuBQ,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACL;IACvCzE,OACEwE,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOM,OAAON;AAChB;AAEO,MAAMO,8BAA8B,CACzCC,SACAC;QAQIC;IANJ,IAAI,CAACF,SACH,OAAO,EAAE;IAIX,MAAMG,YAAYH;IAClB,IAAIE,AAAAA,SAAAA,CAAAA,kBAAAA,UAAU,IAAI,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,QAAQ,AAAD,MAAM,eAAeC,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE;YAClB,MAAMd,QAAQW,UAAU,KAAK,CAACG,IAAI;YAClC,IAAI,CAACf,uBAAuBC,QAC1B,OAAO;YAIT,IAAIS,cAAc;oBACTM;gBAAP,OAAOA,AAAAA,SAAAA,CAAAA,cAAAA,MAAM,IAAI,AAAD,IAATA,KAAAA,IAAAA,YAAY,QAAQ,AAAD,MAAM;YAClC;YAEA,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAM3B,kBAAkB,CAC7B4B,YACAC;IAEA,MAAMC,gBAAgBX,4BAA4BU;IAClD,MAAM7F,SAAS;QAAE,GAAG4F,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAahG,MAAM,CAAC+F,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACThG,MAAM,CAAC+F,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BhG,MAAM,CAAC+F,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBhG,MAAM,CAAC+F,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAOhG;AACT;AAEO,MAAMiG,kBAAkB,CAC7BL,YACAC;IAEA,MAAMC,gBAAgBX,4BAA4BU;IAClD,MAAM7F,SAAS;QAAE,GAAG4F,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAahG,MAAM,CAAC+F,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvBhG,MAAM,CAAC+F,UAAU,GAAG;YAClB,CAACvB,oBAAoB,EAAE;YACvB,QAAQwB;QACV;IAEJ;IAEA,OAAOhG;AACT;AAUO,MAAMkG,mBAAmB,CAC9BC,UACAN;IAGA,MAAMO,QAAQD,YAAY,CAAC;IAG3B,MAAME,eAAelB,4BAA4BU;IAGjD,IAAIQ,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOR,UAAU,KAAK,CAACO;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMb,OAAOU,MAChB,IAAIC,aAAa,QAAQ,CAACX,MAExBa,mBAAmB,CAACb,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/Ca,mBAAmB,CAACb,IAAI,GAAGU,KAAK,CAACV,IAAI;IAKzC,MAAMc,YAAYX,UAAU,KAAK,CAACU;IAGlC,IAAK,MAAMR,aAAaO,kBACtBE,SAAS,CAACT,UAAU,GAAGO,iBAAiB,CAACP,UAAU;IAGrD,OAAOS;AACT"}

package/dist/es/ai-model/inspect.mjs

@@ -1,7 +1,7 @@
 import { cropByRect, paddingToMatchBlockByBase64, preProcessImageUrl } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
-import { AIActionType, adaptBboxToRect, expandSearchArea, markupImageForLLM, mergeRects } from "./common.mjs";
+import { AIActionType, adaptBboxToRect, expandSearchArea, mergeRects } from "./common.mjs";
 import { extractDataQueryPrompt, systemPromptToExtract } from "./prompt/extraction.mjs";
 import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
@@ -85,8 +85,7 @@ async function AiLocateElement(options) {
         imageWidth = paddedResult.width;
         imageHeight = paddedResult.height;
         imagePayload = paddedResult.imageBase64;
-    } else if ('qwen3-vl' === vlMode) ;
-    else if (!vlMode) imagePayload = await markupImageForLLM(screenshotBase64, context.tree, context.size);
+    }
     const msgs = [
         {
             role: 'system',