@midscene/core 1.9.6 → 1.9.7

package/dist/lib/agent/tasks.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/tasks.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeviceAction,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  PlanningLocateParam,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nexport type ActionReportOptions = {\n  type?: TaskTitleType;\n  prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap each task's planning feedback so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. This is the\n// single place that truncates feedback before it is sent to the model; action\n// implementations should hand over the untruncated value.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(feedback: string): string {\n  if (feedback.length <= maxPlanningFeedbackLength) {\n    return feedback;\n  }\n\n  return `${feedback.slice(0, maxPlanningFeedbackLength)}\n...[truncated, ${feedback.length - maxPlanningFeedbackLength} more characters]`;\n}\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Set the pending feedback message consumed by the next planning round.\n   * The message is always prefixed with the current time. When a body is\n   * provided it is appended after the timestamp; otherwise only the time\n   * context is recorded. This is the single entry point for writing\n   * `pendingFeedbackMessage` so the time prefix stays consistent.\n   */\n  private setPendingFeedbackMessage(\n    conversationHistory: ConversationHistory,\n    timeString: string,\n    body?: string,\n  ) {\n    conversationHistory.pendingFeedbackMessage = body\n      ? `Time: ${timeString}, ${body}`\n      : `Current time: ${timeString}`;\n  }\n\n  /**\n   * Collect feedback produced by executed tasks for the next planning round.\n   * Returns undefined when no task reported feedback.\n   */\n  private collectPlanningFeedback(tasks: ExecutionTask[]): string | undefined {\n    const feedbackMessages = tasks.flatMap(({ planningFeedback }) =>\n      planningFeedback ? [truncatePlanningFeedback(planningFeedback)] : [],\n    );\n    return feedbackMessages.length > 0\n      ? feedbackMessages.join('\\n\\n')\n      : undefined;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(\n    userInstruction: TUserPrompt,\n    yamlString: string,\n    reportOptions?: ActionReportOptions,\n  ) {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userInstruction),\n      ),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n        ...(reportOptions?.prompt\n          ? { userInstructionDisplay: reportOptions.prompt }\n          : {}),\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      planningModel,\n      defaultModel,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        includeLocateInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n        reportOptions,\n      );\n    });\n  }\n\n  /**\n   * Called when the task is about to replan. Marks every cache-hit locate task\n   * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n   * did not finish the task, so the element each cache hit produced is suspect.\n   * The upcoming re-locate of the same prompt then replaces the bad entry in\n   * place instead of appending a duplicate that would re-poison the cache on the\n   * next run (#2529).\n   *\n   * Marking a locate that was actually fine is harmless: the step is only ever\n   * replaced if the same prompt is located again (i.e. the step is redone),\n   * which does not happen for a locate that already succeeded.\n   */\n  private invalidateFailedCacheHitLocates(\n    runner: TaskRunner,\n    fromIndex: number,\n  ) {\n    if (!this.taskCache) {\n      return;\n    }\n    for (let i = fromIndex; i < runner.tasks.length; i++) {\n      const task = runner.tasks[i];\n      if (\n        task.type === 'Planning' &&\n        task.subType === 'Locate' &&\n        task.hitBy?.from === 'Cache'\n      ) {\n        const prompt = (task.param as PlanningLocateParam | undefined)?.prompt;\n        if (prompt) {\n          this.taskCache.markLocateCacheStale(prompt);\n        }\n      }\n    }\n  }\n\n  private async runAction(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    if (\n      deepLocate &&\n      !planningModel.adapter.planning.supportsActionDeepLocate\n    ) {\n      warnLog(\n        `The \"deepLocate\" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`,\n      );\n      deepLocate = false;\n    }\n\n    const conversationHistory = new ConversationHistory();\n\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userPrompt),\n      ),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n\n    if (abortSignal?.aborted) {\n      return session.appendErrorPlan(\n        `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n      );\n    }\n    const referenceImageMessages = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(userPrompt),\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            ...(reportOptions?.prompt\n              ? { userInstructionDisplay: reportOptions.prompt }\n              : {}),\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const timing = executorContext.task.timing;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : genericXmlPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: uiContext,\n                actionContext: param.aiActContext,\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                referenceImageMessages,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                  rawChoiceMessage: planError.rawChoiceMessage,\n                };\n              }\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              rawChoiceMessage,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n              rawChoiceMessage,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = uiContext;\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              assert(\n                false,\n                `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`,\n              );\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Capture the time context for the next planning call before running.\n      const initialTimeString = await this.getTimeString();\n\n      const taskCountBeforeRun = runner.tasks.length;\n      try {\n        await session.appendAndRun(executables.tasks);\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          initialTimeString,\n          this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n        );\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          timeString,\n          `Error executing running tasks: ${error?.message || String(error)}`,\n        );\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // We are about to replan, which means the batch we just ran did not finish\n      // the task. Any locate task in that batch that was served from cache\n      // produced an element that failed to complete the step (the action threw,\n      // or it clicked the wrong element and the goal was not reached). Mark those\n      // cache entries stale so the re-locate of the same prompt replaces them in\n      // place instead of appending a poisoning duplicate that would be matched\n      // first on the next run (#2529).\n      this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelRuntime,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = await getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","TaskExecutor","title","options","ExecutionSession","Promise","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","undefined","format","error","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","task","param","executorContext","uiContext","assert","runner","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","fromIndex","i","prompt","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","errorCountInOnePlanningLoop","outputString","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","subGoalStatus","memoriesStatus","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","rawChoiceMessage","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","taskCountBeforeRun","String","Error","errorMsg","type","demand","modelRuntime","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACyDA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAM9C,MAAMC,4BAA4B;AAElC,SAASC,yBAAyBC,QAAgB;IAChD,IAAIA,SAAS,MAAM,IAAIF,2BACrB,OAAOE;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGF,2BAA2B;eAC1C,EAAEE,SAAS,MAAM,GAAGF,0BAA0B,iBAAiB,CAAC;AAC/E;AAIO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,8CAAAA,gBAAgBA,CACzBF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASQ,0BACNG,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CAAC,CAAC,EAAEE,gBAAgB,EAAE,GAC1DA,mBAAmB;gBAACZ,yBAAyBY;aAAkB,GAAG,EAAE;QAEtE,OAAOD,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBE;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdlB,QACE,CAAC,gEAAgE,EAAEkB,OAAO;QAE9E;aAEAlB,QACE;QAKN,OAAOmB,AAAAA,IAAAA,mCAAAA,qBAAAA,AAAAA,EAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACa,OAAOC,eAAeC,cAAcf;IACpE;IAEA,MAAM,uBACJgB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBL;QAIhD,MAAMM,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLN;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMU,SAASR,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACG;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJ5B,KAAa,EACbc,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EACA;QAC1B,MAAMI,UAAU,IAAI,CAAC,sBAAsB,CAACpB;QAC5C,MAAM,EAAEO,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDO,OACAC,eACAC;QAEF,MAAMY,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACb;QAC1C,MAAM,EAAEuB,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAF;QACF;IACF;IAEA,MAAM,OACJG,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,OAAOsB,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAhB,eACAC,cACAgB,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC,aACArB;IAGN;IAcQ,gCACNS,MAAkB,EAClBc,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAIf,OAAO,KAAK,CAAC,MAAM,EAAEe,IAAK;YACpD,MAAMpB,OAAOK,OAAO,KAAK,CAACe,EAAE;YAC5B,IACEpB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMqB,SAAUrB,KAAK,KAAK,EAAsC;gBAChE,IAAIqB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZb,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,IACEoB,cACA,CAACxB,cAAc,OAAO,CAAC,QAAQ,CAAC,wBAAwB,EACxD;YACArB,QACE,CAAC,mGAAmG,EAAEqB,cAAc,MAAM,CAAC,WAAW,IAAI,UAAU,sBAAsB,CAAC;YAE7KwB,aAAa;QACf;QAEA,MAAMnC,sBAAsB,IAAIyC,yBAAAA,mBAAmBA;QAEnD,MAAMzB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBS;QAGhD,MAAMH,SAASR,QAAQ,SAAS;QAEhC,IAAI0B,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJb,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DR,IAAAA,sBAAAA,MAAAA,AAAAA,EACEqB,AAAyBtC,WAAzBsC,sBACA;QAGF,IAAIC,8BAA8B;QAClC,IAAIC;QAEJ,IAAIV,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMW,yBAAyB,MAAMC,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EACnCC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6BtB;QAI/B,MAAO,KAAM;YAEX,IAAIS,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMc,gBAAgBlD,oBAAoB,cAAc,MAAMM;YAG9D,MAAM6C,iBAAiBnD,oBAAoB,cAAc,MAAMM;YAE/D,MAAMmB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjB,GAAIZ,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACNc;oBACAG;oBACAC;oBACA,GAAIiB,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO/B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;oBAClB,MAAM8B,SAAS/B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAMgC,cAAc,IAAI,CAAC,cAAc;oBACvCjE,MACE,sCACAiE,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD/B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOgC,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJ9C,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrC+C,kCAAAA,cAAcA;oBAEpB,IAAIC;oBACJ,IAAI;wBACFC,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAASrC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjCiC;4BACA,cAAc1C;4BACdX;4BACA4B;4BACAI;4BACAC;4BACAc;4BACAX;wBACF;oBACF,EAAE,OAAOyB,WAAW;wBAClB,IAAIA,qBAAqBC,yBAAAA,oBAAoBA,EAAE;4BAE7CzC,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAC3BF,UAAU,KAAK,EACf;4BAEFxC,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAawC,UAAU,WAAW;gCAClC,kBAAkBA,UAAU,gBAAgB;4BAC9C;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBR,QAAQ;oBAC7B;oBACAhE,MAAM,cAAc4E,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN5D,KAAK,EACL6D,KAAK,EACLC,WAAW,EACXC,gBAAgB,EAChBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGjB;oBACJb,eAAe4B;oBAEfrD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCiD;wBACAC;oBACF;oBACAlD,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgBM,OAAO;oBACpDhD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGmD;oBACzCnD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS4C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQe;wBACR,wBAAwBf,WAAW,sBAAsB;wBACzDgB;wBACAC;oBACF;oBACAvD,gBAAgB,SAAS,GAAGC;oBAE5BC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACf,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAE2D,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBACFlD,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,OACA,CAAC,aAAa,EAAEmD,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAalC,QAAQ;YAG3B,MAAMf,QAAQiD,YAAY,WAAW,EAAE;YACvChB,SAAS,IAAI,IAAKgB,YAAY,YAAY,EAAE;YAE5C,IAAIkB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CnE,OACAC,eACAC,cACA;oBACEkB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO5B,OAAO;gBACd,OAAOQ,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAER,MAAM,SAAS,EAAEwD,KAAK,SAAS,CAC5EtD,QACC;YAEP;YACA,IAAIV,oBAAoB,sBAAsB,EAC5CwD,QAAQ,IAAI,CACV,8FACAxD,oBAAoB,sBAAsB;YAK9C,MAAM8E,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqBvD,OAAO,KAAK,CAAC,MAAM;YAC9C,IAAI;gBACF,MAAMR,QAAQ,YAAY,CAAC6D,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5B7E,qBACA8E,mBACA,IAAI,CAAC,uBAAuB,CAACtD,OAAO,KAAK,CAAC,KAAK,CAACuD;YAEpD,EAAE,OAAOvE,OAAY;gBAEnBqC;gBACA,MAAM5C,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEO,OAAO,WAAWwE,OAAOxE,QAAQ;gBAErEpB,MACE,yFACAoB,iBAAiByE,QAAQzE,MAAM,OAAO,GAAGwE,OAAOxE,QAChD,6CACAqC;YAEJ;YAEA,IAAIA,8BAA8BtD,uCAChC,OAAOyB,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACuB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAACnC,QAAQuD;YAG7C,EAAErC;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMsC,WAAW,CAAC,UAAU,EAAEtC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO5B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAClF,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACN0C;gBACA,QAAQG;YACV;YACAtB;QACF;IACF;IAEQ,oBACN2D,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOhE,OAAOqE;gBACtB,MAAM,EAAEtE,IAAI,EAAE,GAAGsE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZzE,KAAK,GAAG,GAAG;wBACTyE;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACAzE,KAAK,KAAK,GAAG4C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgB6B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjBzE,KAAK,iBAAiB,GAAGyE,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMtE,YAAYmE,YAAY,SAAS;gBACvClE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElB,MAAMuE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,AAAAA,IAAAA,8BAAAA,yBAAAA,AAAAA,EAA0Bb,MAAMC;oBACjD;gBACF,OAAO,IAAIS,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAEC,AAAAA,IAAAA,8BAAAA,yBAAAA,AAAAA,EAA0Bb,MAAMC;oBACjD;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DlG,MAAM;oBACN,MAAM+G,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,AAAAA,IAAAA,0BAAAA,iBAAAA,AAAAA,EAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,cACAC,KACAY,sBACAX,kBACAjE;gBAEJ,EAAE,OAAOd,OAAO;oBACd,IAAIA,iBAAiB6F,kCAAAA,YAAYA,EAC/BV,UAAUnF,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE8F,IAAI,EAAEpC,OAAO,EAAE0B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAKzF,QAC1BiG,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAWhG,QAC1BiG,eAAgBD,KAAa,MAAM;qBAEnC/E,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,OAAO;gBAKpB,IAAI4D,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCpF,KAAK,OAAO,GAAG+C;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQqC;oBACR,KAAKb;oBACLxB;gBACF;YACF;QACF;QAEA,OAAOsB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,cACAC,KACAC;QAGF,MAAM/D,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACwE;QAE1C,IAAI,CAAC/D,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEwC,OAAO,EAAE,GAAGzC;QAE5B,OAAO;YACLC;YACAwC;YACA1C;QACF;IACF;IAEA,MAAM,QACJgF,SAAsB,EACtBlB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,AAAAA,IAAAA,kCAAAA,WAAAA,AAAAA,EAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMzF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,WAAW0F;QAE1B,MAAMnF,SAASR,QAAQ,SAAS;QAChC,MAAM,EACJ4F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEAzF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOiF,WAAW;QAClBjF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqF,WAAW;QAClBrF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsF,iBAAiB;QAExBtF,IAAAA,sBAAAA,MAAAA,AAAAA,EACEsF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,cACA4B,mBACA1B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACwE;YAO3C,IAAI/D,QAAQ,QACV,OAAO;gBACL,QAAQnB;gBACRkB;YACF;YAGF6F,eACE5F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEgF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMtD,UAAU,CAAC,kBAAkB,EAAE2C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGvD;oBAAQ;iBAAE,EAC9DmB,cACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAM1G,QAAQ,YAAY,CAAC0G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO1G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEqG,cAAc;IACnE;IAp0BA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,yCAAAA,WAAWA,CAAC;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAwyBF;AAEO,eAAexF,gBACpBsF,iBAAoC,EACpCzF,iBAAuC,EACvCoB,MAAwB;IAExB,IAAI,CAACpB,mBAAmB,QACtB,OAAOoB;IAGT,IAAI,CAACqE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC9F;IACvB;IAEA,MAAM,EAAE+F,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMtG,SAAS,MAAM6B;QAErB,MAAM9C,QAAQ,MAAM0H;QACpB,IAAI1H,OACF,MAAMA;QAER,OAAOiB;IACT,SAAU;QACRwG;IACF;AACF"}
+{"version":3,"file":"agent/tasks.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/tasks.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeviceAction,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  PlanningLocateParam,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nexport type ActionReportOptions = {\n  type?: TaskTitleType;\n  prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap each task's planning feedback so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. This is the\n// single place that truncates feedback before it is sent to the model; action\n// implementations should hand over the untruncated value.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(feedback: string): string {\n  if (feedback.length <= maxPlanningFeedbackLength) {\n    return feedback;\n  }\n\n  return `${feedback.slice(0, maxPlanningFeedbackLength)}\n...[truncated, ${feedback.length - maxPlanningFeedbackLength} more characters]`;\n}\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[]; uiContext?: UIContext },\n  ) {\n    return new ExecutionSession(\n      title,\n      () =>\n        options?.uiContext\n          ? Promise.resolve(options.uiContext)\n          : Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Set the pending feedback message consumed by the next planning round.\n   * The message is always prefixed with the current time. When a body is\n   * provided it is appended after the timestamp; otherwise only the time\n   * context is recorded. This is the single entry point for writing\n   * `pendingFeedbackMessage` so the time prefix stays consistent.\n   */\n  private setPendingFeedbackMessage(\n    conversationHistory: ConversationHistory,\n    timeString: string,\n    body?: string,\n  ) {\n    conversationHistory.pendingFeedbackMessage = body\n      ? `Time: ${timeString}, ${body}`\n      : `Current time: ${timeString}`;\n  }\n\n  /**\n   * Collect feedback produced by executed tasks for the next planning round.\n   * Returns undefined when no task reported feedback.\n   */\n  private collectPlanningFeedback(tasks: ExecutionTask[]): string | undefined {\n    const feedbackMessages = tasks.flatMap(({ planningFeedback }) =>\n      planningFeedback ? [truncatePlanningFeedback(planningFeedback)] : [],\n    );\n    return feedbackMessages.length > 0\n      ? feedbackMessages.join('\\n\\n')\n      : undefined;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(\n    userInstruction: TUserPrompt,\n    yamlString: string,\n    reportOptions?: ActionReportOptions,\n  ) {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userInstruction),\n      ),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n        ...(reportOptions?.prompt\n          ? { userInstructionDisplay: reportOptions.prompt }\n          : {}),\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: { uiContext?: UIContext },\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title, options);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      planningModel,\n      defaultModel,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        includeLocateInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n        reportOptions,\n      );\n    });\n  }\n\n  /**\n   * Called when the task is about to replan. Marks every cache-hit locate task\n   * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n   * did not finish the task, so the element each cache hit produced is suspect.\n   * The upcoming re-locate of the same prompt then replaces the bad entry in\n   * place instead of appending a duplicate that would re-poison the cache on the\n   * next run (#2529).\n   *\n   * Marking a locate that was actually fine is harmless: the step is only ever\n   * replaced if the same prompt is located again (i.e. the step is redone),\n   * which does not happen for a locate that already succeeded.\n   */\n  private invalidateFailedCacheHitLocates(\n    runner: TaskRunner,\n    fromIndex: number,\n  ) {\n    if (!this.taskCache) {\n      return;\n    }\n    for (let i = fromIndex; i < runner.tasks.length; i++) {\n      const task = runner.tasks[i];\n      if (\n        task.type === 'Planning' &&\n        task.subType === 'Locate' &&\n        task.hitBy?.from === 'Cache'\n      ) {\n        const prompt = (task.param as PlanningLocateParam | undefined)?.prompt;\n        if (prompt) {\n          this.taskCache.markLocateCacheStale(prompt);\n        }\n      }\n    }\n  }\n\n  private async runAction(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    if (\n      deepLocate &&\n      !planningModel.adapter.planning.supportsActionDeepLocate\n    ) {\n      warnLog(\n        `The \"deepLocate\" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`,\n      );\n      deepLocate = false;\n    }\n\n    const conversationHistory = new ConversationHistory();\n\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userPrompt),\n      ),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n\n    if (abortSignal?.aborted) {\n      return session.appendErrorPlan(\n        `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n      );\n    }\n    const referenceImageMessages = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(userPrompt),\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            ...(reportOptions?.prompt\n              ? { userInstructionDisplay: reportOptions.prompt }\n              : {}),\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const timing = executorContext.task.timing;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : genericXmlPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: uiContext,\n                actionContext: param.aiActContext,\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                referenceImageMessages,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                  rawChoiceMessage: planError.rawChoiceMessage,\n                };\n              }\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              rawChoiceMessage,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n              rawChoiceMessage,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = uiContext;\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              assert(\n                false,\n                `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`,\n              );\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Capture the time context for the next planning call before running.\n      const initialTimeString = await this.getTimeString();\n\n      const taskCountBeforeRun = runner.tasks.length;\n      try {\n        await session.appendAndRun(executables.tasks);\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          initialTimeString,\n          this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n        );\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          timeString,\n          `Error executing running tasks: ${error?.message || String(error)}`,\n        );\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // We are about to replan, which means the batch we just ran did not finish\n      // the task. Any locate task in that batch that was served from cache\n      // produced an element that failed to complete the step (the action threw,\n      // or it clicked the wrong element and the goal was not reached). Mark those\n      // cache entries stale so the re-locate of the same prompt replaces them in\n      // place instead of appending a poisoning duplicate that would be matched\n      // first on the next run (#2529).\n      this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelRuntime,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = await getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","TaskExecutor","title","options","ExecutionSession","Promise","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","undefined","format","error","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","task","param","executorContext","uiContext","assert","runner","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","fromIndex","i","prompt","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","errorCountInOnePlanningLoop","outputString","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","subGoalStatus","memoriesStatus","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","rawChoiceMessage","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","taskCountBeforeRun","String","Error","errorMsg","type","demand","modelRuntime","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC0DA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,UAAUD,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAM9C,MAAMC,4BAA4B;AAElC,SAASC,yBAAyBC,QAAgB;IAChD,IAAIA,SAAS,MAAM,IAAIF,2BACrB,OAAOE;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGF,2BAA2B;eAC1C,EAAEE,SAAS,MAAM,GAAGF,0BAA0B,iBAAiB,CAAC;AAC/E;AAIO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAAiE,EACjE;QACA,OAAO,IAAIC,8CAAAA,gBAAgBA,CACzBF,OACA,IACEC,SAAS,YACLE,QAAQ,OAAO,CAACF,QAAQ,SAAS,IACjCE,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASQ,0BACNG,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CAAC,CAAC,EAAEE,gBAAgB,EAAE,GAC1DA,mBAAmB;gBAACZ,yBAAyBY;aAAkB,GAAG,EAAE;QAEtE,OAAOD,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBE;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdlB,QACE,CAAC,gEAAgE,EAAEkB,OAAO;QAE9E;aAEAlB,QACE;QAKN,OAAOmB,AAAAA,IAAAA,mCAAAA,qBAAAA,AAAAA,EAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACa,OAAOC,eAAeC,cAAcf;IACpE;IAEA,MAAM,uBACJgB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBL;QAIhD,MAAMM,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLN;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMU,SAASR,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACG;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJ5B,KAAa,EACbc,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1Bf,OAAmC,EACT;QAC1B,MAAMmB,UAAU,IAAI,CAAC,sBAAsB,CAACpB,OAAOC;QACnD,MAAM,EAAEM,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDO,OACAC,eACAC;QAEF,MAAMY,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACb;QAC1C,MAAM,EAAEuB,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAF;QACF;IACF;IAEA,MAAM,OACJG,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,OAAOsB,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAhB,eACAC,cACAgB,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC,aACArB;IAGN;IAcQ,gCACNS,MAAkB,EAClBc,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAIf,OAAO,KAAK,CAAC,MAAM,EAAEe,IAAK;YACpD,MAAMpB,OAAOK,OAAO,KAAK,CAACe,EAAE;YAC5B,IACEpB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMqB,SAAUrB,KAAK,KAAK,EAAsC;gBAChE,IAAIqB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZb,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EACzBrB,aAAmC,EASnC;QACA,IACEoB,cACA,CAACxB,cAAc,OAAO,CAAC,QAAQ,CAAC,wBAAwB,EACxD;YACArB,QACE,CAAC,mGAAmG,EAAEqB,cAAc,MAAM,CAAC,WAAW,IAAI,UAAU,sBAAsB,CAAC;YAE7KwB,aAAa;QACf;QAEA,MAAMnC,sBAAsB,IAAIyC,yBAAAA,mBAAmBA;QAEnD,MAAMzB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,AAAAA,IAAAA,mCAAAA,kBAAAA,AAAAA,EAAmBS;QAGhD,MAAMH,SAASR,QAAQ,SAAS;QAEhC,IAAI0B,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJb,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DR,IAAAA,sBAAAA,MAAAA,AAAAA,EACEqB,AAAyBtC,WAAzBsC,sBACA;QAGF,IAAIC,8BAA8B;QAClC,IAAIC;QAEJ,IAAIV,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMW,yBAAyB,MAAMC,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EACnCC,AAAAA,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6BtB;QAI/B,MAAO,KAAM;YAEX,IAAIS,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMc,gBAAgBlD,oBAAoB,cAAc,MAAMM;YAG9D,MAAM6C,iBAAiBnD,oBAAoB,cAAc,MAAMM;YAE/D,MAAMmB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjB,GAAIZ,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACNc;oBACAG;oBACAC;oBACA,GAAIiB,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO/B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;oBAClB,MAAM8B,SAAS/B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAMgC,cAAc,IAAI,CAAC,cAAc;oBACvCjE,MACE,sCACAiE,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD/B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOgC,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJ9C,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrC+C,kCAAAA,cAAcA;oBAEpB,IAAIC;oBACJ,IAAI;wBACFC,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAASrC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjCiC;4BACA,cAAc1C;4BACdX;4BACA4B;4BACAI;4BACAC;4BACAc;4BACAX;wBACF;oBACF,EAAE,OAAOyB,WAAW;wBAClB,IAAIA,qBAAqBC,yBAAAA,oBAAoBA,EAAE;4BAE7CzC,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAC3BF,UAAU,KAAK,EACf;4BAEFxC,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAawC,UAAU,WAAW;gCAClC,kBAAkBA,UAAU,gBAAgB;4BAC9C;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBR,QAAQ;oBAC7B;oBACAhE,MAAM,cAAc4E,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN5D,KAAK,EACL6D,KAAK,EACLC,WAAW,EACXC,gBAAgB,EAChBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGjB;oBACJb,eAAe4B;oBAEfrD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCiD;wBACAC;oBACF;oBACAlD,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgBM,OAAO;oBACpDhD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGmD;oBACzCnD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS4C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQe;wBACR,wBAAwBf,WAAW,sBAAsB;wBACzDgB;wBACAC;oBACF;oBACAvD,gBAAgB,SAAS,GAAGC;oBAE5BC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACf,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAE2D,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBACFlD,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,OACA,CAAC,aAAa,EAAEmD,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAalC,QAAQ;YAG3B,MAAMf,QAAQiD,YAAY,WAAW,EAAE;YACvChB,SAAS,IAAI,IAAKgB,YAAY,YAAY,EAAE;YAE5C,IAAIkB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CnE,OACAC,eACAC,cACA;oBACEkB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO5B,OAAO;gBACd,OAAOQ,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAER,MAAM,SAAS,EAAEwD,KAAK,SAAS,CAC5EtD,QACC;YAEP;YACA,IAAIV,oBAAoB,sBAAsB,EAC5CwD,QAAQ,IAAI,CACV,8FACAxD,oBAAoB,sBAAsB;YAK9C,MAAM8E,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqBvD,OAAO,KAAK,CAAC,MAAM;YAC9C,IAAI;gBACF,MAAMR,QAAQ,YAAY,CAAC6D,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5B7E,qBACA8E,mBACA,IAAI,CAAC,uBAAuB,CAACtD,OAAO,KAAK,CAAC,KAAK,CAACuD;YAEpD,EAAE,OAAOvE,OAAY;gBAEnBqC;gBACA,MAAM5C,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEO,OAAO,WAAWwE,OAAOxE,QAAQ;gBAErEpB,MACE,yFACAoB,iBAAiByE,QAAQzE,MAAM,OAAO,GAAGwE,OAAOxE,QAChD,6CACAqC;YAEJ;YAEA,IAAIA,8BAA8BtD,uCAChC,OAAOyB,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACuB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAACnC,QAAQuD;YAG7C,EAAErC;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMsC,WAAW,CAAC,UAAU,EAAEtC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO5B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAClF,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACN0C;gBACA,QAAQG;YACV;YACAtB;QACF;IACF;IAEQ,oBACN2D,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOhE,OAAOqE;gBACtB,MAAM,EAAEtE,IAAI,EAAE,GAAGsE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZzE,KAAK,GAAG,GAAG;wBACTyE;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACAzE,KAAK,KAAK,GAAG4C,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgB6B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjBzE,KAAK,iBAAiB,GAAGyE,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMtE,YAAYmE,YAAY,SAAS;gBACvClE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElB,MAAMuE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,AAAAA,IAAAA,8BAAAA,yBAAAA,AAAAA,EAA0Bb,MAAMC;oBACjD;gBACF,OAAO,IAAIS,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAEC,AAAAA,IAAAA,8BAAAA,yBAAAA,AAAAA,EAA0Bb,MAAMC;oBACjD;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DlG,MAAM;oBACN,MAAM+G,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,AAAAA,IAAAA,0BAAAA,iBAAAA,AAAAA,EAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,cACAC,KACAY,sBACAX,kBACAjE;gBAEJ,EAAE,OAAOd,OAAO;oBACd,IAAIA,iBAAiB6F,kCAAAA,YAAYA,EAC/BV,UAAUnF,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE8F,IAAI,EAAEpC,OAAO,EAAE0B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAKzF,QAC1BiG,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAWhG,QAC1BiG,eAAgBD,KAAa,MAAM;qBAEnC/E,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,OAAO;gBAKpB,IAAI4D,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCpF,KAAK,OAAO,GAAG+C;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQqC;oBACR,KAAKb;oBACLxB;gBACF;YACF;QACF;QAEA,OAAOsB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,cACAC,KACAC;QAGF,MAAM/D,SAASR,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACwE;QAE1C,IAAI,CAAC/D,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEwC,OAAO,EAAE,GAAGzC;QAE5B,OAAO;YACLC;YACAwC;YACA1C;QACF;IACF;IAEA,MAAM,QACJgF,SAAsB,EACtBlB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,AAAAA,IAAAA,kCAAAA,WAAAA,AAAAA,EAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMzF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,WAAW0F;QAE1B,MAAMnF,SAASR,QAAQ,SAAS;QAChC,MAAM,EACJ4F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEAzF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOiF,WAAW;QAClBjF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqF,WAAW;QAClBrF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsF,iBAAiB;QAExBtF,IAAAA,sBAAAA,MAAAA,AAAAA,EACEsF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,cACA4B,mBACA1B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACwE;YAO3C,IAAI/D,QAAQ,QACV,OAAO;gBACL,QAAQnB;gBACRkB;YACF;YAGF6F,eACE5F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEgF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMtD,UAAU,CAAC,kBAAkB,EAAE2C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGvD;oBAAQ;iBAAE,EAC9DmB,cACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAM1G,QAAQ,YAAY,CAAC0G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO1G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEqG,cAAc;IACnE;IAx0BA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,yCAAAA,WAAWA,CAAC;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AA4yBF;AAEO,eAAexF,gBACpBsF,iBAAoC,EACpCzF,iBAAuC,EACvCoB,MAAwB;IAExB,IAAI,CAACpB,mBAAmB,QACtB,OAAOoB;IAGT,IAAI,CAACqE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC9F;IACvB;IAEA,MAAM,EAAE+F,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMtG,SAAS,MAAM6B;QAErB,MAAM9C,QAAQ,MAAM0H;QACpB,IAAI1H,OACF,MAAMA;QAER,OAAOiB;IACT,SAAU;QACRwG;IACF;AACF"}

package/dist/lib/agent/utils.js

@@ -33,10 +33,11 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
+    createScreenshotBoundUIContext: ()=>createScreenshotBoundUIContext,
     getMidsceneVersion: ()=>getMidsceneVersion,
     getReportFileName: ()=>getReportFileName,
-    matchElementFromPlan: ()=>matchElementFromPlan,
     matchElementFromCache: ()=>matchElementFromCache,
+    matchElementFromPlan: ()=>matchElementFromPlan,
     ifPlanLocateParamHasLocatedPixelBbox: ()=>ifPlanLocateParamHasLocatedPixelBbox,
     commonContextParser: ()=>commonContextParser,
     parsePrompt: ()=>parsePrompt,
@@ -56,6 +57,7 @@ const utils_namespaceObject = require("@midscene/shared/utils");
 const external_dayjs_namespaceObject = require("dayjs");
 var external_dayjs_default = /*#__PURE__*/ __webpack_require__.n(external_dayjs_namespaceObject);
 const external_task_cache_js_namespaceObject = require("./task-cache.js");
+const agentDebug = (0, logger_namespaceObject.getDebug)('agent');
 async function commonContextParser(interfaceInstance, _opt) {
     const debug = (0, logger_namespaceObject.getDebug)('commonContextParser');
     (0, utils_namespaceObject.assert)(interfaceInstance, 'interfaceInstance is required');
@@ -124,6 +126,20 @@ async function commonContextParser(interfaceInstance, _opt) {
         shrunkShotToLogicalRatio
     };
 }
+async function createScreenshotBoundUIContext(screenshotBase64, opt) {
+    const normalizedScreenshotBase64 = (0, img_namespaceObject.normalizeBase64Image)(screenshotBase64);
+    const actualScreenshotSize = await (0, img_namespaceObject.imageInfoOfBase64)(normalizedScreenshotBase64);
+    if (opt.screenshotSize && (opt.screenshotSize.width !== actualScreenshotSize.width || opt.screenshotSize.height !== actualScreenshotSize.height)) agentDebug('describeElementAtPoint screenshotSize mismatch, use actual size', {
+        provided: opt.screenshotSize,
+        actual: actualScreenshotSize
+    });
+    return {
+        screenshot: external_screenshot_item_js_namespaceObject.ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),
+        shotSize: actualScreenshotSize,
+        shrunkShotToLogicalRatio: 1,
+        _isFrozen: true
+    };
+}
 function getReportFileName(tag = 'web') {
     const reportTagName = env_namespaceObject.globalConfigManager.getEnvConfigValue(env_namespaceObject.MIDSCENE_REPORT_TAG_NAME);
     const dateTimeInFileName = external_dayjs_default()().format('YYYY-MM-DD_HH-mm-ss');
@@ -168,7 +184,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.9.6";
+const getMidsceneVersion = ()=>"1.9.7";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,
@@ -210,6 +226,7 @@ const transformLogicalRectToScreenshotRect = (rect, shrunkShotToLogicalRatio)=>{
     };
 };
 exports.commonContextParser = __webpack_exports__.commonContextParser;
+exports.createScreenshotBoundUIContext = __webpack_exports__.createScreenshotBoundUIContext;
 exports.getMidsceneVersion = __webpack_exports__.getMidsceneVersion;
 exports.getReportFileName = __webpack_exports__.getReportFileName;
 exports.ifPlanLocateParamHasLocatedPixelBbox = __webpack_exports__.ifPlanLocateParamHasLocatedPixelBbox;
@@ -222,6 +239,7 @@ exports.transformLogicalElementToScreenshot = __webpack_exports__.transformLogic
 exports.transformLogicalRectToScreenshotRect = __webpack_exports__.transformLogicalRectToScreenshotRect;
 for(var __rspack_i in __webpack_exports__)if (-1 === [
     "commonContextParser",
+    "createScreenshotBoundUIContext",
     "getMidsceneVersion",
     "getReportFileName",
     "ifPlanLocateParamHasLocatedPixelBbox",

package/dist/lib/agent/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { pixelBboxToRect } from '@/ai-model/workflows/inspect/locate-result-rect';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PixelBbox,\n  PlanningLocateParam,\n  PlanningLocateParamWithLocatedPixelBbox,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { imageInfoOfBase64, resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\nexport function isPixelBbox(value: unknown): value is PixelBbox {\n  return (\n    Array.isArray(value) &&\n    value.length === 4 &&\n    value.every((item) => typeof item === 'number' && Number.isFinite(item))\n  );\n}\n\ntype PlanningLocateParamWithMaybeLocatedPixelBbox = PlanningLocateParam & {\n  locatedPixelBbox?: unknown;\n};\n\nexport function ifPlanLocateParamHasLocatedPixelBbox(\n  planLocateParam: PlanningLocateParamWithMaybeLocatedPixelBbox,\n): planLocateParam is PlanningLocateParamWithLocatedPixelBbox {\n  return isPixelBbox(planLocateParam.locatedPixelBbox);\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParamWithLocatedPixelBbox,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);\n\n  const element = generateElementByRect(\n    rect,\n    typeof planLocateParam.prompt === 'string'\n      ? planLocateParam.prompt\n      : planLocateParam.prompt?.prompt || '',\n  );\n  return element;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","commonContextParser","interfaceInstance","_opt","debug","getDebug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotBase64","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","isPixelBbox","value","Array","item","ifPlanLocateParamHasLocatedPixelBbox","planLocateParam","matchElementFromPlan","rect","pixelBboxToRect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACsBO,eAAeI,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;IAEvBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,mBAAmB;IAE1BE,MAAM;IACN,MAAMG,cAAcL,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNI,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuB;QACrB,SAASD;QACT,WAAWJ,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMK,gBAAgB,MAAMP,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOQ,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIP,MAAM,CAAC,MAAM,EAAEM,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMI,mBAAmB,MAAMb,kBAAkB,gBAAgB;IACjE,MAAMc,uBAAuBC,KAAK,GAAG;IACrCX,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOS,kBAAmB;IAG1BX,MAAM;IACN,MAAM,EAAE,OAAOc,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EAAkBL;IAE1B,IAAI,CAACF,OAAO,QAAQ,CAACK,aAAa,CAACL,OAAO,QAAQ,CAACM,YACjD,MAAM,IAAIL,MACR,CAAC,wFAAwF,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIL,MACR,CAAC,0FAA0F,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAGjIf,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAME,oBAAoBX,eAAeC;IACzC,MAAMW,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBb;IAExB,IAAIW,sBAAsBC,sBAAsB;QAC9ClB,MACE,CAAC,4CAA4C,EAAEM,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEU,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBZ;IAEtB;IAEA,MAAMa,mBAAmBrB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACU,OAAO,QAAQ,CAACW,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIV,MACR,CAAC,wEAAwE,EAAEU,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBnB,MAAM,mBAAmBqB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCpB,MAAM,4BAA4BsB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CpB,MACE,CAAC,mCAAmC,EAAEoB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBhB,kBAAkB;YAC5D,OAAOY;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACjB,kBAAkBC;QACpDU;IACF;AACF;AAEO,SAASO,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAAA,mBAAAA,CAAAA,iBAAqC,CACzDC,oBAAAA,wBAAwBA;IAE1B,MAAMC,qBAAqBC,2BAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAAA,mBAAAA,CAAAA,qBAAyC,CAACQ,oBAAAA,qBAAqBA,GACjE;IAEFC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAEO,SAASG,YAAYC,KAAc;IACxC,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,OAAS,AAAgB,YAAhB,OAAOA,QAAqBpC,OAAO,QAAQ,CAACoC;AAEtE;AAMO,SAASC,qCACdC,eAA6D;IAE7D,OAAOL,YAAYK,gBAAgB,gBAAgB;AACrD;AAEO,SAASC,qBACdD,eAAwD;IAExD,IAAI,CAACA,iBACH;IAGF,MAAME,OAAOC,AAAAA,IAAAA,sCAAAA,eAAAA,AAAAA,EAAgBH,gBAAgB,gBAAgB;IAE7D,MAAMI,UAAUC,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACdH,MACA,AAAkC,YAAlC,OAAOF,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;IAExC,OAAOI;AACT;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EACE;IAKJ,IAAI;QACF,MAAMT,OACJ,MAAMK,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACN3B,KAAK,KAAK,CAACyB,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCzB,KAAK,KAAK,CAACyB,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOO,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACA7B;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO6B;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACN3B,KAAK,KAAK,CAAC2B,QAAQ,MAAM,CAAC,EAAE,GAAG7B;YAC/BE,KAAK,KAAK,CAAC2B,QAAQ,MAAM,CAAC,EAAE,GAAG7B;SAChC;QACD,MAAM;YACJ,GAAG6B,QAAQ,IAAI;YACf,MAAM3B,KAAK,KAAK,CAAC2B,QAAQ,IAAI,CAAC,IAAI,GAAG7B;YACrC,KAAKE,KAAK,KAAK,CAAC2B,QAAQ,IAAI,CAAC,GAAG,GAAG7B;YACnC,OAAOE,KAAK,KAAK,CAAC2B,QAAQ,IAAI,CAAC,KAAK,GAAG7B;YACvC,QAAQE,KAAK,KAAK,CAAC2B,QAAQ,IAAI,CAAC,MAAM,GAAG7B;QAC3C;IACF;AACF;AAEO,MAAM4C,uCAAuC,CAClDjB,MACA3B;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO2B;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMzB,KAAK,KAAK,CAACyB,KAAK,IAAI,GAAG3B;QAC7B,KAAKE,KAAK,KAAK,CAACyB,KAAK,GAAG,GAAG3B;QAC3B,OAAOE,KAAK,KAAK,CAACyB,KAAK,KAAK,GAAG3B;QAC/B,QAAQE,KAAK,KAAK,CAACyB,KAAK,MAAM,GAAG3B;IACnC;AACF"}
+{"version":3,"file":"agent/utils.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { pixelBboxToRect } from '@/ai-model/workflows/inspect/locate-result-rect';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PixelBbox,\n  PlanningLocateParam,\n  PlanningLocateParamWithLocatedPixelBbox,\n  Rect,\n  Size,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport {\n  imageInfoOfBase64,\n  normalizeBase64Image,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst agentDebug = getDebug('agent');\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport async function createScreenshotBoundUIContext(\n  screenshotBase64: string,\n  opt: {\n    screenshotSize?: Size;\n  },\n): Promise<UIContext> {\n  const normalizedScreenshotBase64 = normalizeBase64Image(screenshotBase64);\n  const actualScreenshotSize = await imageInfoOfBase64(\n    normalizedScreenshotBase64,\n  );\n  if (\n    opt.screenshotSize &&\n    (opt.screenshotSize.width !== actualScreenshotSize.width ||\n      opt.screenshotSize.height !== actualScreenshotSize.height)\n  ) {\n    agentDebug(\n      'describeElementAtPoint screenshotSize mismatch, use actual size',\n      {\n        provided: opt.screenshotSize,\n        actual: actualScreenshotSize,\n      },\n    );\n  }\n\n  return {\n    screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n    shotSize: actualScreenshotSize,\n    shrunkShotToLogicalRatio: 1,\n    _isFrozen: true,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\nexport function isPixelBbox(value: unknown): value is PixelBbox {\n  return (\n    Array.isArray(value) &&\n    value.length === 4 &&\n    value.every((item) => typeof item === 'number' && Number.isFinite(item))\n  );\n}\n\ntype PlanningLocateParamWithMaybeLocatedPixelBbox = PlanningLocateParam & {\n  locatedPixelBbox?: unknown;\n};\n\nexport function ifPlanLocateParamHasLocatedPixelBbox(\n  planLocateParam: PlanningLocateParamWithMaybeLocatedPixelBbox,\n): planLocateParam is PlanningLocateParamWithLocatedPixelBbox {\n  return isPixelBbox(planLocateParam.locatedPixelBbox);\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParamWithLocatedPixelBbox,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);\n\n  const element = generateElementByRect(\n    rect,\n    typeof planLocateParam.prompt === 'string'\n      ? planLocateParam.prompt\n      : planLocateParam.prompt?.prompt || '',\n  );\n  return element;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","agentDebug","getDebug","commonContextParser","interfaceInstance","_opt","debug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotBase64","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","normalizeBase64Image","actualScreenshotSize","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","isPixelBbox","value","Array","item","ifPlanLocateParamHasLocatedPixelBbox","planLocateParam","matchElementFromPlan","rect","pixelBboxToRect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC2BA,MAAMI,aAAaC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAErB,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQJ,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;IAEvBK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOH,mBAAmB;IAE1BE,MAAM;IACN,MAAME,cAAcJ,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNG,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuB;QACrB,SAASD;QACT,WAAWH,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMI,gBAAgB,MAAMN,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOO,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIN,MAAM,CAAC,MAAM,EAAEK,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMI,mBAAmB,MAAMZ,kBAAkB,gBAAgB;IACjE,MAAMa,uBAAuBC,KAAK,GAAG;IACrCX,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOS,kBAAmB;IAG1BV,MAAM;IACN,MAAM,EAAE,OAAOa,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EAAkBL;IAE1B,IAAI,CAACF,OAAO,QAAQ,CAACK,aAAa,CAACL,OAAO,QAAQ,CAACM,YACjD,MAAM,IAAIL,MACR,CAAC,wFAAwF,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIL,MACR,CAAC,0FAA0F,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAGjId,MAAM,yBAAyBa,UAAU,KAAKC;IAM9C,MAAME,oBAAoBX,eAAeC;IACzC,MAAMW,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBb;IAExB,IAAIW,sBAAsBC,sBAAsB;QAC9CjB,MACE,CAAC,4CAA4C,EAAEK,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEU,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBZ;IAEtB;IAEA,MAAMa,mBAAmBpB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACS,OAAO,QAAQ,CAACW,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIV,MACR,CAAC,wEAAwE,EAAEU,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBlB,MAAM,mBAAmBoB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCnB,MAAM,4BAA4BqB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CnB,MACE,CAAC,mCAAmC,EAAEmB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBhB,kBAAkB;YAC5D,OAAOY;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACjB,kBAAkBC;QACpDU;IACF;AACF;AAEO,eAAeO,+BACpBlB,gBAAwB,EACxBmB,GAEC;IAED,MAAMC,6BAA6BC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA,EAAqBrB;IACxD,MAAMsB,uBAAuB,MAAMjB,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EACjCe;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKG,qBAAqB,KAAK,IACtDH,IAAI,cAAc,CAAC,MAAM,KAAKG,qBAAqB,MAAK,GAE1DrC,WACE,mEACA;QACE,UAAUkC,IAAI,cAAc;QAC5B,QAAQG;IACV;IAIJ,OAAO;QACL,YAAYL,4CAAAA,cAAAA,CAAAA,MAAqB,CAACG,4BAA4BlB,KAAK,GAAG;QACtE,UAAUoB;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASC,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAAA,mBAAAA,CAAAA,iBAAqC,CACzDC,oBAAAA,wBAAwBA;IAE1B,MAAMC,qBAAqBC,2BAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAAA,mBAAAA,CAAAA,qBAAyC,CAACQ,oBAAAA,qBAAqBA,GACjE;IAEFC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAEO,SAASG,YAAYC,KAAc;IACxC,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,OAAS,AAAgB,YAAhB,OAAOA,QAAqBzC,OAAO,QAAQ,CAACyC;AAEtE;AAMO,SAASC,qCACdC,eAA6D;IAE7D,OAAOL,YAAYK,gBAAgB,gBAAgB;AACrD;AAEO,SAASC,qBACdD,eAAwD;IAExD,IAAI,CAACA,iBACH;IAGF,MAAME,OAAOC,AAAAA,IAAAA,sCAAAA,eAAAA,AAAAA,EAAgBH,gBAAgB,gBAAgB;IAE7D,MAAMI,UAAUC,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACdH,MACA,AAAkC,YAAlC,OAAOF,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;IAExC,OAAOI;AACT;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EACE;IAKJ,IAAI;QACF,MAAMT,OACJ,MAAMK,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNhC,KAAK,KAAK,CAAC8B,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpC9B,KAAK,KAAK,CAAC8B,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOO,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAlC;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOkC;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNhC,KAAK,KAAK,CAACgC,QAAQ,MAAM,CAAC,EAAE,GAAGlC;YAC/BE,KAAK,KAAK,CAACgC,QAAQ,MAAM,CAAC,EAAE,GAAGlC;SAChC;QACD,MAAM;YACJ,GAAGkC,QAAQ,IAAI;YACf,MAAMhC,KAAK,KAAK,CAACgC,QAAQ,IAAI,CAAC,IAAI,GAAGlC;YACrC,KAAKE,KAAK,KAAK,CAACgC,QAAQ,IAAI,CAAC,GAAG,GAAGlC;YACnC,OAAOE,KAAK,KAAK,CAACgC,QAAQ,IAAI,CAAC,KAAK,GAAGlC;YACvC,QAAQE,KAAK,KAAK,CAACgC,QAAQ,IAAI,CAAC,MAAM,GAAGlC;QAC3C;IACF;AACF;AAEO,MAAMiD,uCAAuC,CAClDjB,MACAhC;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOgC;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAM9B,KAAK,KAAK,CAAC8B,KAAK,IAAI,GAAGhC;QAC7B,KAAKE,KAAK,KAAK,CAAC8B,KAAK,GAAG,GAAGhC;QAC3B,OAAOE,KAAK,KAAK,CAAC8B,KAAK,KAAK,GAAGhC;QAC/B,QAAQE,KAAK,KAAK,CAAC8B,KAAK,MAAM,GAAGhC;IACnC;AACF"}

package/dist/lib/ai-model/prompt/describe.js

@@ -33,14 +33,16 @@ const examplesMap = {
         '"搜索输入框，placeholder 为"请输入关键词""',
         '"顶部导航栏中文字为"首页"的链接"',
         '"联系表单中的提交按钮"',
-        '"aria-label 为"打开菜单"的菜单图标"'
+        '"aria-label 为"打开菜单"的菜单图标"',
+        '"左侧导航栏中当前分组标题右侧的折叠图标"'
     ],
     English: [
         '"Login button with text \'Sign In\'"',
         '"Search input with placeholder \'Enter keywords\'"',
         '"Navigation link with text \'Home\' in header"',
         '"Submit button in contact form"',
-        '"Menu icon with aria-label \'Open menu\'"'
+        '"Menu icon with aria-label \'Open menu\'"',
+        '"Collapse icon to the right of the current section title in the left sidebar"'
     ]
 };
 const getExamples = (language)=>{
@@ -70,6 +72,7 @@ DESCRIPTION STRUCTURE:
    - Visual features: "blue background", "with icon"
    - Relative position: "below search bar", "in sidebar"
    - Parent context: "in login form", "in main menu"
+   - Neighboring stable text: "to the right of the 'Settings' section title"
 
 GUIDELINES:
 - Keep description under 25 words
@@ -78,6 +81,11 @@ GUIDELINES:
 - Avoid page-specific or temporary content
 - Don't mention the red rectangle or selection box
 - Focus on stable, reusable characteristics
+- If the selected point/box is inside a text input, textarea, search box, or form field, describe the whole field/control, not the individual placeholder character, typed character, caret, or inner text fragment.
+- For icon-only buttons or unlabeled controls, include the nearest stable label, section title, menu item text, row text, or parent region that owns the control.
+- When multiple similar icons or controls appear in a list/sidebar/menu, the description MUST distinguish the selected one by its owning stable text or section, not by generic position such as "bottom", "nearby", or "sidebar button".
+- For expand/collapse, disclosure, chevron, close, menu, and settings icons, describe both the icon purpose and the stable text/section it controls.
+- Use the actual visible neighboring text from the current screenshot when available; do not copy labels from the examples.
 - **Write the description in ${preferredLanguage}**
 
 EXAMPLES:

package/dist/lib/ai-model/prompt/describe.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/describe.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/describe.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@midscene/shared/env';\n\nconst examplesMap: Record<string, string[]> = {\n  Chinese: [\n    '\"登录表单中的\"登录\"按钮\"',\n    '\"搜索输入框，placeholder 为\"请输入关键词\"\"',\n    '\"顶部导航栏中文字为\"首页\"的链接\"',\n    '\"联系表单中的提交按钮\"',\n    '\"aria-label 为\"打开菜单\"的菜单图标\"',\n  ],\n  English: [\n    '\"Login button with text \\'Sign In\\'\"',\n    '\"Search input with placeholder \\'Enter keywords\\'\"',\n    '\"Navigation link with text \\'Home\\' in header\"',\n    '\"Submit button in contact form\"',\n    '\"Menu icon with aria-label \\'Open menu\\'\"',\n  ],\n};\n\nconst getExamples = (language: string) => {\n  const examples = examplesMap[language] || examplesMap.English;\n  return examples.map((e) => `- ${e}`).join('\\n');\n};\n\nexport const elementDescriberInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nDescribe the element in the red rectangle for precise identification.\n\nIMPORTANT: You MUST write the description in ${preferredLanguage}.\n\nCRITICAL REQUIREMENTS:\n1. UNIQUENESS: The description must uniquely identify this element on the current page\n2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts\n3. PRECISION: Be specific enough to distinguish from similar elements\n\nDESCRIPTION STRUCTURE:\n1. Element type (button, input, link, div, etc.)\n2. Primary identifier (in order of preference):\n   - Unique text content: \"with text 'Login'\"\n   - Unique attribute: \"with aria-label 'Search'\"\n   - Unique class/ID: \"with class 'primary-button'\"\n   - Unique position: \"in header navigation\"\n3. Secondary identifiers (if needed for uniqueness):\n   - Visual features: \"blue background\", \"with icon\"\n   - Relative position: \"below search bar\", \"in sidebar\"\n   - Parent context: \"in login form\", \"in main menu\"\n\nGUIDELINES:\n- Keep description under 25 words\n- Prioritize semantic identifiers over visual ones\n- Use consistent terminology across similar elements\n- Avoid page-specific or temporary content\n- Don't mention the red rectangle or selection box\n- Focus on stable, reusable characteristics\n- **Write the description in ${preferredLanguage}**\n\nEXAMPLES:\n${getExamples(preferredLanguage)}\n\nReturn JSON:\n{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","examplesMap","getExamples","language","examples","e","elementDescriberInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACJA,MAAMI,cAAwC;IAC5C,SAAS;QACP;QACA;QACA;QACA;QACA;KACD;IACD,SAAS;QACP;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,MAAMC,cAAc,CAACC;IACnB,MAAMC,WAAWH,WAAW,CAACE,SAAS,IAAIF,YAAY,OAAO;IAC7D,OAAOG,SAAS,GAAG,CAAC,CAACC,IAAM,CAAC,EAAE,EAAEA,GAAG,EAAE,IAAI,CAAC;AAC5C;AAEO,MAAMC,8BAA8B;IACzC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;6CAGmC,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;6BA0BpC,EAAEA,kBAAkB;;;AAGjD,EAAEL,YAAYK,mBAAmB;;;;;;CAMhC,CAAC;AACF"}
+{"version":3,"file":"ai-model/prompt/describe.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/describe.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@midscene/shared/env';\n\nconst examplesMap: Record<string, string[]> = {\n  Chinese: [\n    '\"登录表单中的\"登录\"按钮\"',\n    '\"搜索输入框，placeholder 为\"请输入关键词\"\"',\n    '\"顶部导航栏中文字为\"首页\"的链接\"',\n    '\"联系表单中的提交按钮\"',\n    '\"aria-label 为\"打开菜单\"的菜单图标\"',\n    '\"左侧导航栏中当前分组标题右侧的折叠图标\"',\n  ],\n  English: [\n    '\"Login button with text \\'Sign In\\'\"',\n    '\"Search input with placeholder \\'Enter keywords\\'\"',\n    '\"Navigation link with text \\'Home\\' in header\"',\n    '\"Submit button in contact form\"',\n    '\"Menu icon with aria-label \\'Open menu\\'\"',\n    '\"Collapse icon to the right of the current section title in the left sidebar\"',\n  ],\n};\n\nconst getExamples = (language: string) => {\n  const examples = examplesMap[language] || examplesMap.English;\n  return examples.map((e) => `- ${e}`).join('\\n');\n};\n\nexport const elementDescriberInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nDescribe the element in the red rectangle for precise identification.\n\nIMPORTANT: You MUST write the description in ${preferredLanguage}.\n\nCRITICAL REQUIREMENTS:\n1. UNIQUENESS: The description must uniquely identify this element on the current page\n2. UNIVERSALITY: Use generic, reusable selectors that work across different contexts\n3. PRECISION: Be specific enough to distinguish from similar elements\n\nDESCRIPTION STRUCTURE:\n1. Element type (button, input, link, div, etc.)\n2. Primary identifier (in order of preference):\n   - Unique text content: \"with text 'Login'\"\n   - Unique attribute: \"with aria-label 'Search'\"\n   - Unique class/ID: \"with class 'primary-button'\"\n   - Unique position: \"in header navigation\"\n3. Secondary identifiers (if needed for uniqueness):\n   - Visual features: \"blue background\", \"with icon\"\n   - Relative position: \"below search bar\", \"in sidebar\"\n   - Parent context: \"in login form\", \"in main menu\"\n   - Neighboring stable text: \"to the right of the 'Settings' section title\"\n\nGUIDELINES:\n- Keep description under 25 words\n- Prioritize semantic identifiers over visual ones\n- Use consistent terminology across similar elements\n- Avoid page-specific or temporary content\n- Don't mention the red rectangle or selection box\n- Focus on stable, reusable characteristics\n- If the selected point/box is inside a text input, textarea, search box, or form field, describe the whole field/control, not the individual placeholder character, typed character, caret, or inner text fragment.\n- For icon-only buttons or unlabeled controls, include the nearest stable label, section title, menu item text, row text, or parent region that owns the control.\n- When multiple similar icons or controls appear in a list/sidebar/menu, the description MUST distinguish the selected one by its owning stable text or section, not by generic position such as \"bottom\", \"nearby\", or \"sidebar button\".\n- For expand/collapse, disclosure, chevron, close, menu, and settings icons, describe both the icon purpose and the stable text/section it controls.\n- Use the actual visible neighboring text from the current screenshot when available; do not copy labels from the examples.\n- **Write the description in ${preferredLanguage}**\n\nEXAMPLES:\n${getExamples(preferredLanguage)}\n\nReturn JSON:\n{\n  \"description\": \"unique element identifier\",\n  \"error\"?: \"error message if any\"\n}`;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","examplesMap","getExamples","language","examples","e","elementDescriberInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACJA,MAAMI,cAAwC;IAC5C,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;IACD,SAAS;QACP;QACA;QACA;QACA;QACA;QACA;KACD;AACH;AAEA,MAAMC,cAAc,CAACC;IACnB,MAAMC,WAAWH,WAAW,CAACE,SAAS,IAAIF,YAAY,OAAO;IAC7D,OAAOG,SAAS,GAAG,CAAC,CAACC,IAAM,CAAC,EAAE,EAAEA,GAAG,EAAE,IAAI,CAAC;AAC5C;AAEO,MAAMC,8BAA8B;IACzC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;6CAGmC,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6BAgCpC,EAAEA,kBAAkB;;;AAGjD,EAAEL,YAAYK,mBAAmB;;;;;;CAMhC,CAAC;AACF"}