@midscene/core 1.15.1 → 1.15.2-beta-20261012093542.0

package/dist/es/agent/execution-session.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/execution-session.mjs","sources":["../../../src/agent/execution-session.ts"],"sourcesContent":["import {\n  type ExecutionReferenceImage,\n  type TaskExecutionError,\n  TaskRunner,\n  type TaskRunnerEventListener,\n} from '@/task-runner';\nimport type {\n  ExecutionTaskApply,\n  ExecutionTaskProgressOptions,\n  UIContext,\n} from '@/types';\n\ntype ExecutionSessionOptions = ExecutionTaskProgressOptions & {\n  tasks?: ExecutionTaskApply[];\n  referenceImages?: readonly ExecutionReferenceImage[];\n  onSnapshotChange?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n  onTaskEvent?: TaskRunnerEventListener;\n};\n\n/**\n * Thin wrapper around {@link TaskRunner} that represents a single linear execution run.\n */\nexport class ExecutionSession {\n  private readonly runner: TaskRunner;\n\n  constructor(\n    name: string,\n    contextProvider: () => Promise<UIContext>,\n    options?: ExecutionSessionOptions,\n  ) {\n    this.runner = new TaskRunner(name, contextProvider, options);\n  }\n\n  async append(\n    tasks: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: { allowWhenError?: boolean },\n  ) {\n    await this.runner.append(tasks, options);\n  }\n\n  async appendAndRun(\n    tasks: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: { allowWhenError?: boolean },\n  ) {\n    return this.runner.appendAndFlush(tasks, options);\n  }\n\n  async run(options?: { allowWhenError?: boolean }) {\n    return this.runner.flush(options);\n  }\n\n  isInErrorState() {\n    return this.runner.isInErrorState();\n  }\n\n  latestErrorTask() {\n    return this.runner.latestErrorTask();\n  }\n\n  appendErrorPlan(errorMsg: string) {\n    return this.runner.appendErrorPlan(errorMsg);\n  }\n\n  getRunner() {\n    return this.runner;\n  }\n}\n"],"names":["ExecutionSession","tasks","options","errorMsg","name","contextProvider","TaskRunner"],"mappings":";;;;;;;;;;;AAyBO,MAAMA;IAWX,MAAM,OACJC,KAAgD,EAChDC,OAAsC,EACtC;QACA,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAACD,OAAOC;IAClC;IAEA,MAAM,aACJD,KAAgD,EAChDC,OAAsC,EACtC;QACA,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAACD,OAAOC;IAC3C;IAEA,MAAM,IAAIA,OAAsC,EAAE;QAChD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAACA;IAC3B;IAEA,iBAAiB;QACf,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc;IACnC;IAEA,kBAAkB;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe;IACpC;IAEA,gBAAgBC,QAAgB,EAAE;QAChC,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe,CAACA;IACrC;IAEA,YAAY;QACV,OAAO,IAAI,CAAC,MAAM;IACpB;IAxCA,YACEC,IAAY,EACZC,eAAyC,EACzCH,OAAiC,CACjC;QANF,uBAAiB,UAAjB;QAOE,IAAI,CAAC,MAAM,GAAG,IAAII,WAAWF,MAAMC,iBAAiBH;IACtD;AAmCF"}
+{"version":3,"file":"agent/execution-session.mjs","sources":["../../../src/agent/execution-session.ts"],"sourcesContent":["import {\n  type ExecutionReferenceImage,\n  type TaskExecutionError,\n  TaskRunner,\n  type TaskRunnerEventListener,\n} from '@/task-runner';\nimport type {\n  ExecutionTaskApply,\n  ExecutionTaskProgressOptions,\n  UIContext,\n} from '@/types';\n\ntype ExecutionSessionOptions = ExecutionTaskProgressOptions & {\n  tasks?: ExecutionTaskApply[];\n  abortSignal?: AbortSignal;\n  referenceImages?: readonly ExecutionReferenceImage[];\n  onSnapshotChange?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n  onTaskEvent?: TaskRunnerEventListener;\n};\n\n/**\n * Thin wrapper around {@link TaskRunner} that represents a single linear execution run.\n */\nexport class ExecutionSession {\n  private readonly runner: TaskRunner;\n\n  constructor(\n    name: string,\n    contextProvider: () => Promise<UIContext>,\n    options?: ExecutionSessionOptions,\n  ) {\n    this.runner = new TaskRunner(name, contextProvider, options);\n  }\n\n  async append(\n    tasks: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: { allowWhenError?: boolean },\n  ) {\n    await this.runner.append(tasks, options);\n  }\n\n  async appendAndRun(\n    tasks: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: { allowWhenError?: boolean },\n  ) {\n    return this.runner.appendAndFlush(tasks, options);\n  }\n\n  async run(options?: { allowWhenError?: boolean }) {\n    return this.runner.flush(options);\n  }\n\n  isInErrorState() {\n    return this.runner.isInErrorState();\n  }\n\n  latestErrorTask() {\n    return this.runner.latestErrorTask();\n  }\n\n  appendErrorPlan(errorMsg: string) {\n    return this.runner.appendErrorPlan(errorMsg);\n  }\n\n  getRunner() {\n    return this.runner;\n  }\n}\n"],"names":["ExecutionSession","tasks","options","errorMsg","name","contextProvider","TaskRunner"],"mappings":";;;;;;;;;;;AA0BO,MAAMA;IAWX,MAAM,OACJC,KAAgD,EAChDC,OAAsC,EACtC;QACA,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAACD,OAAOC;IAClC;IAEA,MAAM,aACJD,KAAgD,EAChDC,OAAsC,EACtC;QACA,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc,CAACD,OAAOC;IAC3C;IAEA,MAAM,IAAIA,OAAsC,EAAE;QAChD,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAACA;IAC3B;IAEA,iBAAiB;QACf,OAAO,IAAI,CAAC,MAAM,CAAC,cAAc;IACnC;IAEA,kBAAkB;QAChB,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe;IACpC;IAEA,gBAAgBC,QAAgB,EAAE;QAChC,OAAO,IAAI,CAAC,MAAM,CAAC,eAAe,CAACA;IACrC;IAEA,YAAY;QACV,OAAO,IAAI,CAAC,MAAM;IACpB;IAxCA,YACEC,IAAY,EACZC,eAAyC,EACzCH,OAAiC,CACjC;QANF,uBAAiB,UAAjB;QAOE,IAAI,CAAC,MAAM,GAAG,IAAII,WAAWF,MAAMC,iBAAiBH;IACtD;AAmCF"}

package/dist/es/agent/tasks.mjs

@@ -50,6 +50,7 @@ class TaskExecutor {
         return new ExecutionSession(title, ()=>options?.uiContext ? Promise.resolve(options.uiContext) : Promise.resolve(this.service.contextRetrieverFn()), {
             onTaskStart: this.onTaskStartCallback,
             tasks: options?.tasks,
+            abortSignal: options?.abortSignal,
             referenceImages: options?.referenceImages,
             onSnapshotChange: async (runner, error)=>{
                 await this.hooks?.onSnapshotChange?.(runner, error);
@@ -132,7 +133,9 @@ class TaskExecutor {
         };
     }
     async sleep(ms, abortSignal) {
-        const session = this.createExecutionSession('Sleep');
+        const session = this.createExecutionSession('Sleep', {
+            abortSignal
+        });
         const action = defineActionSleep(abortSignal);
         await session.appendAndRun({
             type: 'Action Space',
@@ -193,6 +196,7 @@ class TaskExecutor {
         const promptDisplay = reportOptions?.prompt || userPromptToString(userPrompt);
         let activeActionReporter;
         const session = this.createExecutionSession(taskTitleStr(reportOptions?.type || 'Act', promptDisplay), {
+            abortSignal,
             referenceImages: userPromptToMultimodalPrompt(userPrompt)?.images,
             onTaskEvent: async (event)=>{
                 await activeActionReporter?.(event);
@@ -525,7 +529,8 @@ class TaskExecutor {
             ...executionOptions?.uiContext ? {
                 uiContext: executionOptions.uiContext
             } : {},
-            referenceImages: multimodalPrompt?.images
+            referenceImages: multimodalPrompt?.images,
+            abortSignal: executionOptions?.abortSignal
         });
         const runner = session.getRunner();
         const executionModelRuntime = {

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/insight';\nimport { prepareUserPrompt } from '@/ai-model/shared/multimodal-prompt';\nimport { standardPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport { type AbstractInterface, defineActionSleep } from '@/device';\nimport type Service from '@/service';\nimport type {\n  ExecutionReferenceImage,\n  TaskRunner,\n  TaskRunnerEvent,\n} from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  AiActEffort,\n  AiActProgressData,\n  AiActProgressPhase,\n  DetailedLocateParam,\n  DeviceAction,\n  ExecutionRecorderItem,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  UIContext,\n} from '@/types';\nimport { ServiceError, aiActProgressScope } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport {\n  type ActionReadiness,\n  isActionReadinessError,\n} from './action-readiness';\nimport { ExecutionSession } from './execution-session';\nimport { withFileChooser } from './file-chooser';\nimport {\n  type AgentProgressPublisher,\n  createAiActActionReporter,\n  errorMessageForAiAct,\n} from './progress';\nimport { renderAIContext } from './prompt-context';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onSnapshotChange?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n  // Publish onto the agent's progress bus. The bus owns sequencing, so the\n  // executor is a pure producer: it names the scope/phase and hands over data.\n  onProgress?: AgentProgressPublisher;\n}\n\nexport type ActionReportOptions = {\n  type?: TaskTitleType;\n  prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap planning feedback by default so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. Actions\n// that require lossless structured feedback can explicitly choose their limit.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(\n  feedback: string,\n  requestedLimit?: number | 'unlimited',\n): string {\n  if (requestedLimit === 'unlimited') {\n    return feedback;\n  }\n  const limit =\n    requestedLimit && Number.isFinite(requestedLimit) && requestedLimit > 0\n      ? Math.floor(requestedLimit)\n      : maxPlanningFeedbackLength;\n  if (feedback.length <= limit) {\n    return feedback;\n  }\n\n  return `${feedback.slice(0, limit)}\n...[truncated, ${feedback.length - limit} more characters]`;\n}\n\nexport { TaskExecutionError };\nexport { withFileChooser } from './file-chooser';\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      actionReadiness?: ActionReadiness;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n      actionReadiness: opts.actionReadiness,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: {\n      tasks?: ExecutionTaskApply[];\n      uiContext?: UIContext;\n      referenceImages?: readonly ExecutionReferenceImage[];\n      onSnapshotChange?: (\n        runner: TaskRunner,\n        error?: TaskExecutionError,\n      ) => Promise<void> | void;\n      onTaskEvent?: (event: TaskRunnerEvent) => Promise<void> | void;\n    },\n  ) {\n    return new ExecutionSession(\n      title,\n      () =>\n        options?.uiContext\n          ? Promise.resolve(options.uiContext)\n          : Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        referenceImages: options?.referenceImages,\n        onSnapshotChange: async (runner, error) => {\n          await this.hooks?.onSnapshotChange?.(runner, error);\n          await options?.onSnapshotChange?.(runner, error);\n        },\n        ...(options?.onTaskEvent ? { onTaskEvent: options.onTaskEvent } : {}),\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Publish one event onto the agent's progress bus. The task layer is a pure\n   * producer here: it names the scope/phase and forwards the structured\n   * payload; the bus stamps the sequence and isolates listener errors. It has\n   * no knowledge of how the event is rendered or consumed.\n   */\n  private async emitProgress(\n    scope: string,\n    phase: string,\n    data: unknown,\n  ): Promise<void> {\n    await this.hooks?.onProgress?.(scope, phase, data);\n  }\n\n  /**\n   * aiAct-flavored convenience over {@link emitProgress}: aiAct is the first\n   * producer (\"pilot\") on the generic bus, so its events are simply tagged with\n   * the `aiAct` scope.\n   */\n  private emitAiActProgress(\n    phase: AiActProgressPhase,\n    data: AiActProgressData,\n  ): Promise<void> {\n    return this.emitProgress(aiActProgressScope, phase, data);\n  }\n\n  /**\n   * Set the pending feedback message consumed by the next planning round.\n   * The message is always prefixed with the current time. When a body is\n   * provided it is appended after the timestamp; otherwise only the time\n   * context is recorded. This is the single entry point for writing\n   * `pendingFeedbackMessage` so the time prefix stays consistent.\n   */\n  private setPendingFeedbackMessage(\n    conversationHistory: ConversationHistory,\n    timeString: string,\n    body?: string,\n  ) {\n    conversationHistory.pendingFeedbackMessage = body\n      ? `Time: ${timeString}, ${body}`\n      : `Current time: ${timeString}`;\n  }\n\n  /**\n   * Collect feedback produced by executed tasks for the next planning round.\n   * Returns undefined when no task reported feedback.\n   */\n  private collectPlanningFeedback(tasks: ExecutionTask[]): string | undefined {\n    const feedbackMessages = tasks.flatMap(\n      ({ planningFeedback, planningFeedbackMaxLength }) =>\n        planningFeedback\n          ? [\n              truncatePlanningFeedback(\n                planningFeedback,\n                planningFeedbackMaxLength,\n              ),\n            ]\n          : [],\n    );\n    return feedbackMessages.length > 0\n      ? feedbackMessages.join('\\n\\n')\n      : undefined;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(\n    userInstruction: TUserPrompt,\n    yamlString: string,\n    reportOptions?: ActionReportOptions,\n  ) {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userInstruction),\n      ),\n      {\n        referenceImages: userPromptToMultimodalPrompt(userInstruction)?.images,\n      },\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n        ...(reportOptions?.prompt\n          ? { userInstructionDisplay: reportOptions.prompt }\n          : {}),\n      },\n      executor: async (executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async sleep(ms: number, abortSignal?: AbortSignal): Promise<void> {\n    const session = this.createExecutionSession('Sleep');\n    const action = defineActionSleep(abortSignal);\n    await session.appendAndRun({\n      type: 'Action Space',\n      subType: action.name,\n      param: { timeMs: ms },\n      executor: async ({ task }) => {\n        assert(\n          Number.isFinite(ms) && ms > 0,\n          `ms for sleep must be a finite number greater than 0, but got ${ms}`,\n        );\n        setTimingFieldOnce(task.timing, 'callActionStart');\n        try {\n          await action.call({ timeMs: ms });\n        } finally {\n          setTimingFieldOnce(task.timing, 'callActionEnd');\n        }\n      },\n    });\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: { uiContext?: UIContext; abortSignal?: AbortSignal },\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title, options);\n    const runner = session.getRunner();\n    const executionPlanningModel = { ...planningModel, executionId: runner.id };\n    const executionDefaultModel = { ...defaultModel, executionId: runner.id };\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      executionPlanningModel,\n      executionDefaultModel,\n      { abortSignal: options?.abortSignal },\n    );\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    effort: AiActEffort = 'balance',\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    abortSignal?.throwIfAborted();\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        effort,\n        deepLocate,\n        abortSignal,\n        reportOptions,\n      );\n    });\n  }\n\n  /**\n   * Called when the task is about to replan. Marks every cache-hit locate task\n   * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n   * did not finish the task, so the element each cache hit produced is suspect.\n   * The upcoming re-locate of the same prompt then replaces the bad entry in\n   * place instead of appending a duplicate that would re-poison the cache on the\n   * next run (#2529).\n   *\n   * Marking a locate that was actually fine is harmless: the step is only ever\n   * replaced if the same prompt is located again (i.e. the step is redone),\n   * which does not happen for a locate that already succeeded.\n   */\n  private invalidateFailedCacheHitLocates(\n    runner: TaskRunner,\n    fromIndex: number,\n  ) {\n    if (!this.taskCache) {\n      return;\n    }\n    for (let i = fromIndex; i < runner.tasks.length; i++) {\n      const task = runner.tasks[i];\n      if (\n        task.type === 'Planning' &&\n        task.subType === 'Locate' &&\n        task.hitBy?.from === 'Cache'\n      ) {\n        const prompt = (task.param as DetailedLocateParam | undefined)?.prompt;\n        if (prompt) {\n          this.taskCache.markLocateCacheStale(prompt);\n        }\n      }\n    }\n  }\n\n  private async runAction(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    effort: AiActEffort = 'balance',\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    const conversationHistory = new ConversationHistory();\n    const promptDisplay =\n      reportOptions?.prompt || userPromptToString(userPrompt);\n\n    // Per-call reporter that maps the runner's native task events to aiAct\n    // action progress for the action batch currently running. Kept local (not\n    // on the instance) so concurrent aiAct() calls on the same executor stay\n    // isolated; it is set while a planned batch runs and cleared afterwards.\n    let activeActionReporter:\n      | ((event: TaskRunnerEvent) => Promise<void>)\n      | undefined;\n\n    const session = this.createExecutionSession(\n      taskTitleStr(reportOptions?.type || 'Act', promptDisplay),\n      {\n        referenceImages: userPromptToMultimodalPrompt(userPrompt)?.images,\n        onTaskEvent: async (event) => {\n          await activeActionReporter?.(event);\n        },\n      },\n    );\n    const runner = session.getRunner();\n    planningModel = { ...planningModel, executionId: runner.id };\n    defaultModel = { ...defaultModel, executionId: runner.id };\n\n    const noIndividualLocateModel = planningModel.config.slot === 'default';\n    const includeLocateInPlanning =\n      effort !== 'deepThink' && noIndividualLocateModel;\n    const imagesIncludeCount = effort === 'deepThink' ? 2 : 1;\n\n    debug('setting includeLocateInPlanning to', includeLocateInPlanning, {\n      effort,\n      noIndividualLocateModel,\n    });\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    if (replanningCycleLimit === undefined) {\n      throw new Error(\n        'replanningCycleLimit is required for TaskExecutor.action',\n      );\n    }\n\n    await this.emitAiActProgress('start', {\n      prompt: promptDisplay,\n      planLimit: replanningCycleLimit,\n    });\n\n    const appendFailedPlan = async (\n      errorMsg: string,\n      planIndex?: number,\n    ): Promise<{\n      output: undefined;\n      runner: TaskRunner;\n    }> => {\n      await this.emitAiActProgress('failed', {\n        ...(planIndex ? { planIndex } : {}),\n        planLimit: replanningCycleLimit,\n        error: errorMsg,\n      });\n      return session.appendErrorPlan(errorMsg);\n    };\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n    let latestPlanResult: PlanningAIResponse | undefined;\n    let latestPlanIndex = 0;\n\n    if (abortSignal?.aborted) {\n      return appendFailedPlan(\n        `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n      );\n    }\n    const getPreparedUserPrompt = (() => {\n      let promise: ReturnType<typeof prepareUserPrompt> | undefined;\n\n      return () => {\n        promise ??= prepareUserPrompt(userPrompt);\n        return promise;\n      };\n    })();\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      const planIndex = replanCount + 1;\n      latestPlanIndex = planIndex;\n\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return appendFailedPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n          planIndex,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            ...(reportOptions?.prompt\n              ? { userInstructionDisplay: reportOptions.prompt }\n              : {}),\n            replanningCycleLimit,\n            aiActContext,\n            imagesIncludeCount,\n            effort,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const planningUiContext = uiContext as UIContext;\n            const timing = executorContext.task.timing;\n            await this.emitAiActProgress('plan_thinking', {\n              planIndex,\n              planLimit: replanningCycleLimit,\n              screenshot: planningUiContext.screenshot,\n            });\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : standardPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              const preparedUserPrompt = await getPreparedUserPrompt();\n\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(preparedUserPrompt, {\n                context: planningUiContext,\n                actionContext: renderAIContext(aiActContext),\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                effort,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                  rawAssistantOutput: planError.rawAssistantOutput,\n                };\n              }\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessageForAiAct(planError),\n              });\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              rawAssistantOutput,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n              rawAssistantOutput,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = planningUiContext;\n\n            // Forward the raw in-progress reasoning the model produced; the\n            // consumer decides which field to surface and how to format it.\n            // `finalizeMessage` is intentionally left for the `complete` event.\n            if (log || thought) {\n              await this.emitAiActProgress('plan_planned', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                ...(log ? { log } : {}),\n                ...(thought ? { thought } : {}),\n              });\n            }\n\n            if (error) {\n              const errorMessage = `Failed to continue: ${error}\\n${log || ''}`;\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessage,\n              });\n            }\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              const errorMessage = `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`;\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessage,\n              });\n              assert(false, errorMessage);\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n      latestPlanResult = planResult;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return appendFailedPlan(\n          `Error converting plans to executable tasks: ${errorMessageForAiAct(error)}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n          planIndex,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Capture the time context for the next planning call before running.\n      const initialTimeString = await this.getTimeString();\n\n      const taskCountBeforeRun = runner.tasks.length;\n      // Scope a reporter to this replanning round so native task events from the\n      // runner are mapped to aiAct progress with the right plan context; cleared\n      // in `finally` so events fired between batches are ignored.\n      activeActionReporter = createAiActActionReporter(\n        planIndex,\n        replanningCycleLimit,\n        (phase, data) => this.emitAiActProgress(phase, data),\n      );\n      try {\n        await session.appendAndRun(executables.tasks);\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          initialTimeString,\n          this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n        );\n      } catch (error: any) {\n        // A completed action must not be replayed because readiness failed.\n        if (isActionReadinessError(error)) throw error;\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          timeString,\n          `Error executing running tasks: ${error?.message || String(error)}`,\n        );\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      } finally {\n        activeActionReporter = undefined;\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return appendFailedPlan(\n          'Too many errors in one planning loop',\n          planIndex,\n        );\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return appendFailedPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n          planIndex,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // We are about to replan, which means the batch we just ran did not finish\n      // the task. Any locate task in that batch that was served from cache\n      // produced an element that failed to complete the step (the action threw,\n      // or it clicked the wrong element and the goal was not reached). Mark those\n      // cache entries stale so the re-locate of the same prompt replaces them in\n      // place instead of appending a poisoning duplicate that would be matched\n      // first on the next run (#2529).\n      this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return appendFailedPlan(errorMsg, planIndex);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    // Carry the raw final text; the consumer formats it. `outputString` is the\n    // model's finalize message, with the latest plan's log/thought as fallback\n    // context for consumers that want to show why the run completed.\n    await this.emitAiActProgress('complete', {\n      planIndex: latestPlanIndex,\n      planLimit: replanningCycleLimit,\n      ...((outputString ?? latestPlanResult?.output)\n        ? { output: outputString ?? latestPlanResult?.output }\n        : {}),\n      ...(latestPlanResult?.log ? { log: latestPlanResult.log } : {}),\n      ...(latestPlanResult?.thought\n        ? { thought: latestPlanResult.thought }\n        : {}),\n    });\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        ...(opt?.context !== undefined ? { context: opt.context } : {}),\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n            executionOptions,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        } finally {\n          // Surface the captured screenshot sequence in the report, then release\n          // it from the UIContext so its base64 is not retained twice.\n          recordAndReleaseScreenshotSequence(task, uiContext);\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        // Keep assertion failures in the task result so TaskRunner can persist\n        // the model dump before aiAssert turns the result into a thrown error.\n        if (type === 'Assert') {\n          outputResult = Boolean(outputResult);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n      uiContext?: UIContext;\n    },\n  ): Promise<ExecutionResult<T>> {\n    executionOptions?.abortSignal?.throwIfAborted();\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      {\n        ...(executionOptions?.uiContext\n          ? { uiContext: executionOptions.uiContext }\n          : {}),\n        referenceImages: multimodalPrompt?.images,\n      },\n    );\n\n    const runner = session.getRunner();\n    const executionModelRuntime = { ...modelRuntime, executionId: runner.id };\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      executionModelRuntime,\n      opt,\n      multimodalPrompt,\n      executionOptions,\n    );\n\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n      { referenceImages: multimodalPrompt?.images },\n    );\n    const runner = session.getRunner();\n    const executionModelRuntime = { ...modelRuntime, executionId: runner.id };\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        executionModelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n/**\n * Surface a captured screenshot sequence in the report timeline, then release\n * it from the UIContext.\n *\n * When a UIObservation insight runs, the observed frames live on\n * `uiContext.screenshotSequence` only as a transient model input. This attaches\n * them to the task recorder so the report renders the full sequence the model\n * saw (the report timeline builds one screenshot per recorder item), then drops\n * the field from the UIContext so its base64 is not retained twice for the\n * lifetime of the dump.\n *\n * The last frame is the representative `uiContext.screenshot`, already shown in\n * the report, so only the earlier frames are recorded to avoid duplication.\n * Observed frames are PREPENDED to `task.recorder` (rather than appended) so\n * array order matches chronological order — they were captured before the\n * assertion's before/after screenshots.\n */\nexport function recordAndReleaseScreenshotSequence(\n  task: ExecutionTask,\n  uiContext: UIContext | undefined,\n): void {\n  const frames = uiContext?.screenshotSequence;\n  if (frames && frames.length > 1) {\n    const recorderItems: ExecutionRecorderItem[] = [];\n    for (let i = 0; i < frames.length - 1; i++) {\n      const frame = frames[i];\n      recorderItems.push({\n        type: 'screenshot',\n        ts: frame.capturedAt,\n        screenshot: frame,\n        description: `Observed frame ${i + 1}/${frames.length}`,\n        timing: 'observed-frame',\n      });\n    }\n    // Prepend observed frames so they appear before the task's own\n    // before/after screenshots in array order. Observed frames have earlier\n    // timestamps than the assertion task itself, so this keeps array order\n    // aligned with chronological order for consumers that iterate in-place.\n    task.recorder = task.recorder\n      ? [...recorderItems, ...task.recorder]\n      : recorderItems;\n  }\n  if (uiContext?.screenshotSequence) {\n    uiContext.screenshotSequence = undefined;\n  }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","requestedLimit","limit","Number","Math","TaskExecutor","title","options","ExecutionSession","Promise","runner","error","scope","phase","data","aiActProgressScope","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","planningFeedbackMaxLength","undefined","format","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","userPromptToMultimodalPrompt","task","executorContext","uiContext","assert","ms","abortSignal","action","defineActionSleep","setTimingFieldOnce","executionPlanningModel","executionDefaultModel","result","output","userPrompt","aiActContext","cacheable","replanningCycleLimitOverride","effort","fileChooserAccept","deepLocate","withFileChooser","fromIndex","i","prompt","ConversationHistory","promptDisplay","activeActionReporter","event","noIndividualLocateModel","includeLocateInPlanning","imagesIncludeCount","replanCount","yamlFlow","replanningCycleLimit","Error","appendFailedPlan","errorMsg","planIndex","errorCountInOnePlanningLoop","outputString","latestPlanResult","latestPlanIndex","getPreparedUserPrompt","promise","prepareUserPrompt","subGoalStatus","memoriesStatus","planningUiContext","timing","actionSpace","Array","console","planImpl","standardPlan","planResult","preparedUserPrompt","renderAIContext","planError","AIResponseParseError","withUsageIntent","errorMessageForAiAct","JSON","actions","thought","log","memory","usage","rawResponse","rawAssistantOutput","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","errorMessage","executables","initialTimeString","taskCountBeforeRun","createAiActActionReporter","isActionReadinessError","String","type","demand","modelRuntime","opt","multimodalPrompt","executionOptions","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","recordAndReleaseScreenshotSequence","outputResult","Boolean","executionModelRuntime","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","frames","recorderItems","frame"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsFA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAK9C,MAAMC,4BAA4B;AAElC,SAASC,yBACPC,QAAgB,EAChBC,cAAqC;IAErC,IAAIA,AAAmB,gBAAnBA,gBACF,OAAOD;IAET,MAAME,QACJD,kBAAkBE,OAAO,QAAQ,CAACF,mBAAmBA,iBAAiB,IAClEG,KAAK,KAAK,CAACH,kBACXH;IACN,IAAIE,SAAS,MAAM,IAAIE,OACrB,OAAOF;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGE,OAAO;eACtB,EAAEF,SAAS,MAAM,GAAGE,MAAM,iBAAiB,CAAC;AAC3D;AAKO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAmCQ,uBACNC,KAAa,EACbC,OASC,EACD;QACA,OAAO,IAAIC,iBACTF,OACA,IACEC,SAAS,YACLE,QAAQ,OAAO,CAACF,QAAQ,SAAS,IACjCE,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,iBAAiBA,SAAS;YAC1B,kBAAkB,OAAOG,QAAQC;gBAC/B,MAAM,IAAI,CAAC,KAAK,EAAE,mBAAmBD,QAAQC;gBAC7C,MAAMJ,SAAS,mBAAmBG,QAAQC;YAC5C;YACA,GAAIJ,SAAS,cAAc;gBAAE,aAAaA,QAAQ,WAAW;YAAC,IAAI,CAAC,CAAC;QACtE;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IAQA,MAAc,aACZK,KAAa,EACbC,KAAa,EACbC,IAAa,EACE;QACf,MAAM,IAAI,CAAC,KAAK,EAAE,aAAaF,OAAOC,OAAOC;IAC/C;IAOQ,kBACND,KAAyB,EACzBC,IAAuB,EACR;QACf,OAAO,IAAI,CAAC,YAAY,CAACC,oBAAoBF,OAAOC;IACtD;IASQ,0BACNE,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CACpC,CAAC,EAAEE,gBAAgB,EAAEC,yBAAyB,EAAE,GAC9CD,mBACI;gBACEtB,yBACEsB,kBACAC;aAEH,GACD,EAAE;QAEV,OAAOF,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBG;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOb,OAAO;YACdf,QACE,CAAC,gEAAgE,EAAEe,OAAO;QAE9E;aAEAf,QACE;QAKN,OAAO6B,sBAAsBD;IAC/B;IAEA,MAAa,wBACXE,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BrB,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACmB,OAAOC,eAAeC,cAAcrB;IACpE;IAEA,MAAM,uBACJsB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,mBAAmBL,mBAE9C;YACE,iBAAiBM,6BAA6BN,kBAAkB;QAClE;QAGF,MAAMO,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLP;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOM;gBACf,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMpB,SAASsB,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACI;QAE3B,OAAO;YACL1B;QACF;IACF;IAEA,MAAM,MAAM8B,EAAU,EAAEC,WAAyB,EAAiB;QAChE,MAAMT,UAAU,IAAI,CAAC,sBAAsB,CAAC;QAC5C,MAAMU,SAASC,kBAAkBF;QACjC,MAAMT,QAAQ,YAAY,CAAC;YACzB,MAAM;YACN,SAASU,OAAO,IAAI;YACpB,OAAO;gBAAE,QAAQF;YAAG;YACpB,UAAU,OAAO,EAAEJ,IAAI,EAAE;gBACvBG,OACEpC,OAAO,QAAQ,CAACqC,OAAOA,KAAK,GAC5B,CAAC,6DAA6D,EAAEA,IAAI;gBAEtEI,mBAAmBR,KAAK,MAAM,EAAE;gBAChC,IAAI;oBACF,MAAMM,OAAO,IAAI,CAAC;wBAAE,QAAQF;oBAAG;gBACjC,SAAU;oBACRI,mBAAmBR,KAAK,MAAM,EAAE;gBAClC;YACF;QACF;IACF;IAEA,MAAM,SACJ9B,KAAa,EACboB,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BrB,OAA8D,EACpC;QAC1B,MAAMyB,UAAU,IAAI,CAAC,sBAAsB,CAAC1B,OAAOC;QACnD,MAAMG,SAASsB,QAAQ,SAAS;QAChC,MAAMa,yBAAyB;YAAE,GAAGlB,aAAa;YAAE,aAAajB,OAAO,EAAE;QAAC;QAC1E,MAAMoC,wBAAwB;YAAE,GAAGlB,YAAY;YAAE,aAAalB,OAAO,EAAE;QAAC;QACxE,MAAM,EAAES,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDO,OACAmB,wBACAC,uBACA;YAAE,aAAavC,SAAS;QAAY;QAEtC,MAAMwC,SAAS,MAAMf,QAAQ,YAAY,CAACb;QAC1C,MAAM,EAAE6B,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAtC;QACF;IACF;IAEA,MAAM,OACJuC,UAAuB,EACvBtB,aAA2B,EAC3BC,YAA0B,EAC1BsB,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,SAAsB,SAAS,EAC/BC,iBAA4B,EAC5BC,UAAoB,EACpBd,WAAyB,EACzBV,aAAmC,EASnC;QACAU,aAAa;QACb,OAAOe,gBAAgB,IAAI,CAAC,SAAS,EAAEF,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBL,YACAtB,eACAC,cACAsB,cACAC,WACAC,8BACAC,QACAE,YACAd,aACAV;IAGN;IAcQ,gCACNrB,MAAkB,EAClB+C,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAIhD,OAAO,KAAK,CAAC,MAAM,EAAEgD,IAAK;YACpD,MAAMtB,OAAO1B,OAAO,KAAK,CAACgD,EAAE;YAC5B,IACEtB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMuB,SAAUvB,KAAK,KAAK,EAAsC;gBAChE,IAAIuB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZV,UAAuB,EACvBtB,aAA2B,EAC3BC,YAA0B,EAC1BsB,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,SAAsB,SAAS,EAC/BE,UAAoB,EACpBd,WAAyB,EACzBV,aAAmC,EASnC;QACA,MAAMf,sBAAsB,IAAI4C;QAChC,MAAMC,gBACJ9B,eAAe,UAAUG,mBAAmBe;QAM9C,IAAIa;QAIJ,MAAM9B,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAaF,eAAe,QAAQ,OAAO8B,gBAC3C;YACE,iBAAiB1B,6BAA6Bc,aAAa;YAC3D,aAAa,OAAOc;gBAClB,MAAMD,uBAAuBC;YAC/B;QACF;QAEF,MAAMrD,SAASsB,QAAQ,SAAS;QAChCL,gBAAgB;YAAE,GAAGA,aAAa;YAAE,aAAajB,OAAO,EAAE;QAAC;QAC3DkB,eAAe;YAAE,GAAGA,YAAY;YAAE,aAAalB,OAAO,EAAE;QAAC;QAEzD,MAAMsD,0BAA0BrC,AAA8B,cAA9BA,cAAc,MAAM,CAAC,IAAI;QACzD,MAAMsC,0BACJZ,AAAW,gBAAXA,UAA0BW;QAC5B,MAAME,qBAAqBb,AAAW,gBAAXA,SAAyB,IAAI;QAExD3D,MAAM,sCAAsCuE,yBAAyB;YACnEZ;YACAW;QACF;QAEA,IAAIG,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJjB,gCAAgC,IAAI,CAAC,oBAAoB;QAC3D,IAAIiB,AAAyB9C,WAAzB8C,sBACF,MAAM,IAAIC,MACR;QAIJ,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS;YACpC,QAAQT;YACR,WAAWQ;QACb;QAEA,MAAME,mBAAmB,OACvBC,UACAC;YAKA,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU;gBACrC,GAAIA,YAAY;oBAAEA;gBAAU,IAAI,CAAC,CAAC;gBAClC,WAAWJ;gBACX,OAAOG;YACT;YACA,OAAOxC,QAAQ,eAAe,CAACwC;QACjC;QAEA,IAAIE,8BAA8B;QAClC,IAAIC;QACJ,IAAIC;QACJ,IAAIC,kBAAkB;QAEtB,IAAIpC,aAAa,SACf,OAAO8B,iBACL,CAAC,cAAc,EAAE9B,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMqC,wBAAyB,AAAC;YAC9B,IAAIC;YAEJ,OAAO;gBACLA,YAAYC,kBAAkB/B;gBAC9B,OAAO8B;YACT;QACF;QAGA,MAAO,KAAM;YACX,MAAMN,YAAYN,cAAc;YAChCU,kBAAkBJ;YAGlB,IAAIhC,aAAa,SACf,OAAO8B,iBACL,CAAC,cAAc,EAAE9B,YAAY,MAAM,IAAI,yBAAyB,EAChEgC;YAKJ,MAAMQ,gBAAgBjE,oBAAoB,cAAc,MAAMO;YAG9D,MAAM2D,iBAAiBlE,oBAAoB,cAAc,MAAMO;YAE/D,MAAMwB,SAAS,MAAMf,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBiB;oBACjB,GAAIlB,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACNsC;oBACAnB;oBACAgB;oBACAb;oBACA,GAAI4B,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO7C;oBACf,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM6C,oBAAoB7C;oBAC1B,MAAM8C,SAAS/C,gBAAgB,IAAI,CAAC,MAAM;oBAC1C,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB;wBAC5CoC;wBACA,WAAWJ;wBACX,YAAYc,kBAAkB,UAAU;oBAC1C;oBAEA,MAAME,cAAc,IAAI,CAAC,cAAc;oBACvC3F,MACE,sCACA2F,YAAY,GAAG,CAAC,CAAC3C,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhDH,OAAO+C,MAAM,OAAO,CAACD,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBE,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJ7D,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrC8D;oBAEN,IAAIC;oBACJ,IAAI;wBACF,MAAMC,qBAAqB,MAAMb;wBAEjClC,mBAAmBwC,QAAQ;wBAC3BM,aAAa,MAAMF,SAASG,oBAAoB;4BAC9C,SAASR;4BACT,eAAeS,gBAAgB1C;4BAC/BmC;4BACA,cAAc1D;4BACdX;4BACAiD;4BACAC;4BACAb;4BACAZ;wBACF;oBACF,EAAE,OAAOoD,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7CzD,gBAAgB,IAAI,CAAC,KAAK,GAAG0D,gBAC3BF,UAAU,KAAK,EACf;4BAEFxD,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAawD,UAAU,WAAW;gCAClC,oBAAoBA,UAAU,kBAAkB;4BAClD;wBACF;wBACA,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1CpB;4BACA,WAAWJ;4BACX,OAAO2B,qBAAqBH;wBAC9B;wBACA,MAAMA;oBACR,SAAU;wBACRjD,mBAAmBwC,QAAQ;oBAC7B;oBACA1F,MAAM,cAAcuG,KAAK,SAAS,CAACP,YAAY,MAAM;oBAErD,MAAM,EACJQ,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN1F,KAAK,EACL2F,KAAK,EACLC,WAAW,EACXC,kBAAkB,EAClBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGnB;oBACJf,eAAegC;oBAEftE,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCkE;wBACAC;oBACF;oBACAnE,gBAAgB,IAAI,CAAC,KAAK,GAAG0D,gBAAgBO,OAAO;oBACpDjE,gBAAgB,IAAI,CAAC,iBAAiB,GAAGoE;oBACzCpE,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS6D,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUX,WAAW,QAAQ;wBAC7B,QAAQiB;wBACR,wBAAwBjB,WAAW,sBAAsB;wBACzDkB;wBACAC;oBACF;oBACAxE,gBAAgB,SAAS,GAAG8C;oBAK5B,IAAIiB,OAAOD,SACT,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB;wBAC3C1B;wBACA,WAAWJ;wBACX,GAAI+B,MAAM;4BAAEA;wBAAI,IAAI,CAAC,CAAC;wBACtB,GAAID,UAAU;4BAAEA;wBAAQ,IAAI,CAAC,CAAC;oBAChC;oBAGF,IAAIxF,OAAO;wBACT,MAAMmG,eAAe,CAAC,oBAAoB,EAAEnG,MAAM,EAAE,EAAEyF,OAAO,IAAI;wBACjE,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1C3B;4BACA,WAAWJ;4BACX,OAAOyC;wBACT;oBACF;oBAEAvE,OAAO,CAAC5B,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEyF,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBAA2B;wBAC7B,MAAMI,eAAe,CAAC,aAAa,EAAEH,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;wBACnG,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1C3B;4BACA,WAAWJ;4BACX,OAAOyC;wBACT;wBACAvE,OAAO,OAAOuE;oBAChB;oBAEA,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMpB,aAAa3C,QAAQ;YAC3B6B,mBAAmBc;YAGnB,MAAMhE,QAAQgE,YAAY,WAAW,EAAE;YACvCtB,SAAS,IAAI,IAAKsB,YAAY,YAAY,EAAE;YAE5C,IAAIqB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CrF,OACAC,eACAC,cACA;oBACEuB;oBACAI;oBACAd;gBACF;YAEJ,EAAE,OAAO9B,OAAO;gBACd,OAAO4D,iBACL,CAAC,4CAA4C,EAAEyB,qBAAqBrF,OAAO,SAAS,EAAEsF,KAAK,SAAS,CAClGvE,QACC,EACH+C;YAEJ;YACA,IAAIzD,oBAAoB,sBAAsB,EAC5CuE,QAAQ,IAAI,CACV,8FACAvE,oBAAoB,sBAAsB;YAK9C,MAAMgG,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqBvG,OAAO,KAAK,CAAC,MAAM;YAI9CoD,uBAAuBoD,0BACrBzC,WACAJ,sBACA,CAACxD,OAAOC,OAAS,IAAI,CAAC,iBAAiB,CAACD,OAAOC;YAEjD,IAAI;gBACF,MAAMkB,QAAQ,YAAY,CAAC+E,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5B/F,qBACAgG,mBACA,IAAI,CAAC,uBAAuB,CAACtG,OAAO,KAAK,CAAC,KAAK,CAACuG;YAEpD,EAAE,OAAOtG,OAAY;gBAEnB,IAAIwG,uBAAuBxG,QAAQ,MAAMA;gBAEzC+D;gBACA,MAAMzD,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEN,OAAO,WAAWyG,OAAOzG,QAAQ;gBAErEjB,MACE,yFACAiB,iBAAiB2D,QAAQ3D,MAAM,OAAO,GAAGyG,OAAOzG,QAChD,6CACA+D;YAEJ,SAAU;gBACRZ,uBAAuBvC;YACzB;YAEA,IAAImD,8BAA8B7E,uCAChC,OAAO0E,iBACL,wCACAE;YAKJ,IAAIhC,aAAa,SACf,OAAO8B,iBACL,CAAC,cAAc,EAAE9B,YAAY,MAAM,IAAI,yBAAyB,EAChEgC;YAKJ,IAAI,CAACiB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAAChF,QAAQuG;YAG7C,EAAE9C;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMG,WAAW,CAAC,UAAU,EAAEH,qBAAqB,4JAA4J,CAAC;gBAChN,OAAOE,iBAAiBC,UAAUC;YACpC;YAEA,IAAI,CAACzD,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAKA,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY;YACvC,WAAW4D;YACX,WAAWR;YACX,GAAKM,gBAAgBC,kBAAkB,SACnC;gBAAE,QAAQD,gBAAgBC,kBAAkB;YAAO,IACnD,CAAC,CAAC;YACN,GAAIA,kBAAkB,MAAM;gBAAE,KAAKA,iBAAiB,GAAG;YAAC,IAAI,CAAC,CAAC;YAC9D,GAAIA,kBAAkB,UAClB;gBAAE,SAASA,iBAAiB,OAAO;YAAC,IACpC,CAAC,CAAC;QACR;QAEA,OAAO;YACL,QAAQ;gBACNR;gBACA,QAAQO;YACV;YACAjE;QACF;IACF;IAEQ,oBACN2G,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAEC,EACD;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASN;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,GAAIA,KAAK,YAAYjG,SAAY;oBAAE,SAASiG,IAAI,OAAO;gBAAC,IAAI,CAAC,CAAC;gBAC9D,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOM;gBACf,MAAM,EAAExF,IAAI,EAAE,GAAGwF;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ3F,KAAK,GAAG,GAAG;wBACT2F;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA3F,KAAK,KAAK,GAAG2D,gBAAgBgC,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjB3F,KAAK,iBAAiB,GAAG2F,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMzF,YAAYsF,YAAY,SAAS;gBACvCrF,OAAOD,WAAW;gBAElB,MAAM0F,mBAAmBX,AAAS,YAATA;gBACzB,IAAIY,cAAcX;gBAClB,IAAIY,cAAc;gBAClB,IAAIF,oBAAqBX,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEa,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF,OAAO,IAAIU,kBAAkB;oBAC3BE,cAAcb;oBACdY,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF;gBAEA,IAAIc;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIb,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1D9H,MAAM;oBACN,MAAM4I,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAd,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAV,cACAC,KACAa,sBACAZ,kBACAnF,WACAoF;gBAEJ,EAAE,OAAO/G,OAAO;oBACd,IAAIA,iBAAiB6H,cACnBV,UAAUnH,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBAGR8H,mCAAmCrG,MAAME;gBAC3C;gBAEA,MAAM,EAAExB,IAAI,EAAEqF,OAAO,EAAE4B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAe5H;gBACnB,IAAIkH,kBAEF,IAAI,AAAgB,YAAhB,OAAOlH,MACT4H,eAAe5H;qBACV,IAAIuG,AAAS,cAATA,MAEPqB,eADE5H,QAAAA,OACa,QAECA,IAAY,CAACoH,YAAY;qBAEtC,IAAIpH,QAAAA,MACT4H,eAAe;qBAGf,IAAI5H,MAAM,CAACoH,YAAY,KAAK3G,QAC1BmH,eAAgB5H,IAAY,CAACoH,YAAY;qBACpC,IAAIpH,MAAM,WAAWS,QAC1BmH,eAAgB5H,KAAa,MAAM;qBAEnCyB,OAAO,OAAO;gBAOpB,IAAI8E,AAAS,aAATA,MACFqB,eAAeC,QAAQD;gBAGzB,OAAO;oBACL,QAAQA;oBACR,KAAKb;oBACL1B;gBACF;YACF;QACF;QAEA,OAAOwB;IACT;IACA,MAAM,yBACJN,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAGC,EAC4B;QAC7BA,kBAAkB,aAAa;QAC/B,MAAM1F,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEoF,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASrB,KAAK,SAAS,CAACqB,UAEvD;YACE,GAAII,kBAAkB,YAClB;gBAAE,WAAWA,iBAAiB,SAAS;YAAC,IACxC,CAAC,CAAC;YACN,iBAAiBD,kBAAkB;QACrC;QAGF,MAAM/G,SAASsB,QAAQ,SAAS;QAChC,MAAM4G,wBAAwB;YAAE,GAAGrB,YAAY;YAAE,aAAa7G,OAAO,EAAE;QAAC;QACxE,MAAMiH,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CN,MACAC,QACAsB,uBACApB,KACAC,kBACAC;QAGF,MAAM3E,SAAS,MAAMf,QAAQ,YAAY,CAAC2F;QAE1C,IAAI,CAAC5E,QACH,MAAM,IAAIuB,MACR;QAIJ,MAAM,EAAEtB,MAAM,EAAEmD,OAAO,EAAE,GAAGpD;QAE5B,OAAO;YACLC;YACAmD;YACAzF;QACF;IACF;IAEA,MAAM,QACJmI,SAAsB,EACtBrB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEuB,UAAU,EAAErB,gBAAgB,EAAE,GAAGsB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM9G,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW+G,cACxB;YAAE,iBAAiBvB,kBAAkB;QAAO;QAE9C,MAAM/G,SAASsB,QAAQ,SAAS;QAChC,MAAM4G,wBAAwB;YAAE,GAAGrB,YAAY;YAAE,aAAa7G,OAAO,EAAE;QAAC;QACxE,MAAM,EACJuI,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG7B;QACJ,MAAM8B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA9G,OAAOsG,WAAW;QAClBtG,OAAO0G,WAAW;QAClB1G,OAAO2G,iBAAiB;QAExB3G,OACE2G,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAMhC,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAmB,YACAF,uBACAU,mBACA7B;YAGF,MAAM1E,SAAU,MAAMf,QAAQ,YAAY,CAAC2F;YAO3C,IAAI5E,QAAQ,QACV,OAAO;gBACL,QAAQxB;gBACRb;YACF;YAGFgJ,eACE3G,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAE+F,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAM1D,UAAU,CAAC,kBAAkB,EAAE+C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAG3D;oBAAQ;iBAAE,EAC9DoB,cACAA;gBAEF,IAAIwC,UAAU,CAAC,EAAE,EACf,MAAM/H,QAAQ,YAAY,CAAC+H,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO/H,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAE0H,cAAc;IACnE;IA/hCA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IASC,CACD;QAtCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAqBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;YACrC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAigCF;AAkBO,SAASzB,mCACdrG,IAAmB,EACnBE,SAAgC;IAEhC,MAAM8H,SAAS9H,WAAW;IAC1B,IAAI8H,UAAUA,OAAO,MAAM,GAAG,GAAG;QAC/B,MAAMC,gBAAyC,EAAE;QACjD,IAAK,IAAI3G,IAAI,GAAGA,IAAI0G,OAAO,MAAM,GAAG,GAAG1G,IAAK;YAC1C,MAAM4G,QAAQF,MAAM,CAAC1G,EAAE;YACvB2G,cAAc,IAAI,CAAC;gBACjB,MAAM;gBACN,IAAIC,MAAM,UAAU;gBACpB,YAAYA;gBACZ,aAAa,CAAC,eAAe,EAAE5G,IAAI,EAAE,CAAC,EAAE0G,OAAO,MAAM,EAAE;gBACvD,QAAQ;YACV;QACF;QAKAhI,KAAK,QAAQ,GAAGA,KAAK,QAAQ,GACzB;eAAIiI;eAAkBjI,KAAK,QAAQ;SAAC,GACpCiI;IACN;IACA,IAAI/H,WAAW,oBACbA,UAAU,kBAAkB,GAAGf;AAEnC"}
+{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/insight';\nimport { prepareUserPrompt } from '@/ai-model/shared/multimodal-prompt';\nimport { standardPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport { type AbstractInterface, defineActionSleep } from '@/device';\nimport type Service from '@/service';\nimport type {\n  ExecutionReferenceImage,\n  TaskRunner,\n  TaskRunnerEvent,\n} from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  AiActEffort,\n  AiActProgressData,\n  AiActProgressPhase,\n  DetailedLocateParam,\n  DeviceAction,\n  ExecutionRecorderItem,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  UIContext,\n} from '@/types';\nimport { ServiceError, aiActProgressScope } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport {\n  type ActionReadiness,\n  isActionReadinessError,\n} from './action-readiness';\nimport { ExecutionSession } from './execution-session';\nimport { withFileChooser } from './file-chooser';\nimport {\n  type AgentProgressPublisher,\n  createAiActActionReporter,\n  errorMessageForAiAct,\n} from './progress';\nimport { renderAIContext } from './prompt-context';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onSnapshotChange?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n  // Publish onto the agent's progress bus. The bus owns sequencing, so the\n  // executor is a pure producer: it names the scope/phase and hands over data.\n  onProgress?: AgentProgressPublisher;\n}\n\nexport type ActionReportOptions = {\n  type?: TaskTitleType;\n  prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap planning feedback by default so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. Actions\n// that require lossless structured feedback can explicitly choose their limit.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(\n  feedback: string,\n  requestedLimit?: number | 'unlimited',\n): string {\n  if (requestedLimit === 'unlimited') {\n    return feedback;\n  }\n  const limit =\n    requestedLimit && Number.isFinite(requestedLimit) && requestedLimit > 0\n      ? Math.floor(requestedLimit)\n      : maxPlanningFeedbackLength;\n  if (feedback.length <= limit) {\n    return feedback;\n  }\n\n  return `${feedback.slice(0, limit)}\n...[truncated, ${feedback.length - limit} more characters]`;\n}\n\nexport { TaskExecutionError };\nexport { withFileChooser } from './file-chooser';\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      actionReadiness?: ActionReadiness;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n      actionReadiness: opts.actionReadiness,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: {\n      tasks?: ExecutionTaskApply[];\n      abortSignal?: AbortSignal;\n      uiContext?: UIContext;\n      referenceImages?: readonly ExecutionReferenceImage[];\n      onSnapshotChange?: (\n        runner: TaskRunner,\n        error?: TaskExecutionError,\n      ) => Promise<void> | void;\n      onTaskEvent?: (event: TaskRunnerEvent) => Promise<void> | void;\n    },\n  ) {\n    return new ExecutionSession(\n      title,\n      () =>\n        options?.uiContext\n          ? Promise.resolve(options.uiContext)\n          : Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        abortSignal: options?.abortSignal,\n        referenceImages: options?.referenceImages,\n        onSnapshotChange: async (runner, error) => {\n          await this.hooks?.onSnapshotChange?.(runner, error);\n          await options?.onSnapshotChange?.(runner, error);\n        },\n        ...(options?.onTaskEvent ? { onTaskEvent: options.onTaskEvent } : {}),\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Publish one event onto the agent's progress bus. The task layer is a pure\n   * producer here: it names the scope/phase and forwards the structured\n   * payload; the bus stamps the sequence and isolates listener errors. It has\n   * no knowledge of how the event is rendered or consumed.\n   */\n  private async emitProgress(\n    scope: string,\n    phase: string,\n    data: unknown,\n  ): Promise<void> {\n    await this.hooks?.onProgress?.(scope, phase, data);\n  }\n\n  /**\n   * aiAct-flavored convenience over {@link emitProgress}: aiAct is the first\n   * producer (\"pilot\") on the generic bus, so its events are simply tagged with\n   * the `aiAct` scope.\n   */\n  private emitAiActProgress(\n    phase: AiActProgressPhase,\n    data: AiActProgressData,\n  ): Promise<void> {\n    return this.emitProgress(aiActProgressScope, phase, data);\n  }\n\n  /**\n   * Set the pending feedback message consumed by the next planning round.\n   * The message is always prefixed with the current time. When a body is\n   * provided it is appended after the timestamp; otherwise only the time\n   * context is recorded. This is the single entry point for writing\n   * `pendingFeedbackMessage` so the time prefix stays consistent.\n   */\n  private setPendingFeedbackMessage(\n    conversationHistory: ConversationHistory,\n    timeString: string,\n    body?: string,\n  ) {\n    conversationHistory.pendingFeedbackMessage = body\n      ? `Time: ${timeString}, ${body}`\n      : `Current time: ${timeString}`;\n  }\n\n  /**\n   * Collect feedback produced by executed tasks for the next planning round.\n   * Returns undefined when no task reported feedback.\n   */\n  private collectPlanningFeedback(tasks: ExecutionTask[]): string | undefined {\n    const feedbackMessages = tasks.flatMap(\n      ({ planningFeedback, planningFeedbackMaxLength }) =>\n        planningFeedback\n          ? [\n              truncatePlanningFeedback(\n                planningFeedback,\n                planningFeedbackMaxLength,\n              ),\n            ]\n          : [],\n    );\n    return feedbackMessages.length > 0\n      ? feedbackMessages.join('\\n\\n')\n      : undefined;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(\n    userInstruction: TUserPrompt,\n    yamlString: string,\n    reportOptions?: ActionReportOptions,\n  ) {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userInstruction),\n      ),\n      {\n        referenceImages: userPromptToMultimodalPrompt(userInstruction)?.images,\n      },\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n        ...(reportOptions?.prompt\n          ? { userInstructionDisplay: reportOptions.prompt }\n          : {}),\n      },\n      executor: async (executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async sleep(ms: number, abortSignal?: AbortSignal): Promise<void> {\n    const session = this.createExecutionSession('Sleep', { abortSignal });\n    const action = defineActionSleep(abortSignal);\n    await session.appendAndRun({\n      type: 'Action Space',\n      subType: action.name,\n      param: { timeMs: ms },\n      executor: async ({ task }) => {\n        assert(\n          Number.isFinite(ms) && ms > 0,\n          `ms for sleep must be a finite number greater than 0, but got ${ms}`,\n        );\n        setTimingFieldOnce(task.timing, 'callActionStart');\n        try {\n          await action.call({ timeMs: ms });\n        } finally {\n          setTimingFieldOnce(task.timing, 'callActionEnd');\n        }\n      },\n    });\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: { uiContext?: UIContext; abortSignal?: AbortSignal },\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title, options);\n    const runner = session.getRunner();\n    const executionPlanningModel = { ...planningModel, executionId: runner.id };\n    const executionDefaultModel = { ...defaultModel, executionId: runner.id };\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      executionPlanningModel,\n      executionDefaultModel,\n      { abortSignal: options?.abortSignal },\n    );\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    effort: AiActEffort = 'balance',\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    abortSignal?.throwIfAborted();\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        effort,\n        deepLocate,\n        abortSignal,\n        reportOptions,\n      );\n    });\n  }\n\n  /**\n   * Called when the task is about to replan. Marks every cache-hit locate task\n   * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n   * did not finish the task, so the element each cache hit produced is suspect.\n   * The upcoming re-locate of the same prompt then replaces the bad entry in\n   * place instead of appending a duplicate that would re-poison the cache on the\n   * next run (#2529).\n   *\n   * Marking a locate that was actually fine is harmless: the step is only ever\n   * replaced if the same prompt is located again (i.e. the step is redone),\n   * which does not happen for a locate that already succeeded.\n   */\n  private invalidateFailedCacheHitLocates(\n    runner: TaskRunner,\n    fromIndex: number,\n  ) {\n    if (!this.taskCache) {\n      return;\n    }\n    for (let i = fromIndex; i < runner.tasks.length; i++) {\n      const task = runner.tasks[i];\n      if (\n        task.type === 'Planning' &&\n        task.subType === 'Locate' &&\n        task.hitBy?.from === 'Cache'\n      ) {\n        const prompt = (task.param as DetailedLocateParam | undefined)?.prompt;\n        if (prompt) {\n          this.taskCache.markLocateCacheStale(prompt);\n        }\n      }\n    }\n  }\n\n  private async runAction(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    effort: AiActEffort = 'balance',\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    const conversationHistory = new ConversationHistory();\n    const promptDisplay =\n      reportOptions?.prompt || userPromptToString(userPrompt);\n\n    // Per-call reporter that maps the runner's native task events to aiAct\n    // action progress for the action batch currently running. Kept local (not\n    // on the instance) so concurrent aiAct() calls on the same executor stay\n    // isolated; it is set while a planned batch runs and cleared afterwards.\n    let activeActionReporter:\n      | ((event: TaskRunnerEvent) => Promise<void>)\n      | undefined;\n\n    const session = this.createExecutionSession(\n      taskTitleStr(reportOptions?.type || 'Act', promptDisplay),\n      {\n        abortSignal,\n        referenceImages: userPromptToMultimodalPrompt(userPrompt)?.images,\n        onTaskEvent: async (event) => {\n          await activeActionReporter?.(event);\n        },\n      },\n    );\n    const runner = session.getRunner();\n    planningModel = { ...planningModel, executionId: runner.id };\n    defaultModel = { ...defaultModel, executionId: runner.id };\n\n    const noIndividualLocateModel = planningModel.config.slot === 'default';\n    const includeLocateInPlanning =\n      effort !== 'deepThink' && noIndividualLocateModel;\n    const imagesIncludeCount = effort === 'deepThink' ? 2 : 1;\n\n    debug('setting includeLocateInPlanning to', includeLocateInPlanning, {\n      effort,\n      noIndividualLocateModel,\n    });\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    if (replanningCycleLimit === undefined) {\n      throw new Error(\n        'replanningCycleLimit is required for TaskExecutor.action',\n      );\n    }\n\n    await this.emitAiActProgress('start', {\n      prompt: promptDisplay,\n      planLimit: replanningCycleLimit,\n    });\n\n    const appendFailedPlan = async (\n      errorMsg: string,\n      planIndex?: number,\n    ): Promise<{\n      output: undefined;\n      runner: TaskRunner;\n    }> => {\n      await this.emitAiActProgress('failed', {\n        ...(planIndex ? { planIndex } : {}),\n        planLimit: replanningCycleLimit,\n        error: errorMsg,\n      });\n      return session.appendErrorPlan(errorMsg);\n    };\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n    let latestPlanResult: PlanningAIResponse | undefined;\n    let latestPlanIndex = 0;\n\n    if (abortSignal?.aborted) {\n      return appendFailedPlan(\n        `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n      );\n    }\n    const getPreparedUserPrompt = (() => {\n      let promise: ReturnType<typeof prepareUserPrompt> | undefined;\n\n      return () => {\n        promise ??= prepareUserPrompt(userPrompt);\n        return promise;\n      };\n    })();\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      const planIndex = replanCount + 1;\n      latestPlanIndex = planIndex;\n\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return appendFailedPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n          planIndex,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            ...(reportOptions?.prompt\n              ? { userInstructionDisplay: reportOptions.prompt }\n              : {}),\n            replanningCycleLimit,\n            aiActContext,\n            imagesIncludeCount,\n            effort,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const planningUiContext = uiContext as UIContext;\n            const timing = executorContext.task.timing;\n            await this.emitAiActProgress('plan_thinking', {\n              planIndex,\n              planLimit: replanningCycleLimit,\n              screenshot: planningUiContext.screenshot,\n            });\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : standardPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              const preparedUserPrompt = await getPreparedUserPrompt();\n\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(preparedUserPrompt, {\n                context: planningUiContext,\n                actionContext: renderAIContext(aiActContext),\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                effort,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                  rawAssistantOutput: planError.rawAssistantOutput,\n                };\n              }\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessageForAiAct(planError),\n              });\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              rawAssistantOutput,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n              rawAssistantOutput,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = planningUiContext;\n\n            // Forward the raw in-progress reasoning the model produced; the\n            // consumer decides which field to surface and how to format it.\n            // `finalizeMessage` is intentionally left for the `complete` event.\n            if (log || thought) {\n              await this.emitAiActProgress('plan_planned', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                ...(log ? { log } : {}),\n                ...(thought ? { thought } : {}),\n              });\n            }\n\n            if (error) {\n              const errorMessage = `Failed to continue: ${error}\\n${log || ''}`;\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessage,\n              });\n            }\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              const errorMessage = `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`;\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessage,\n              });\n              assert(false, errorMessage);\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n      latestPlanResult = planResult;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return appendFailedPlan(\n          `Error converting plans to executable tasks: ${errorMessageForAiAct(error)}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n          planIndex,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Capture the time context for the next planning call before running.\n      const initialTimeString = await this.getTimeString();\n\n      const taskCountBeforeRun = runner.tasks.length;\n      // Scope a reporter to this replanning round so native task events from the\n      // runner are mapped to aiAct progress with the right plan context; cleared\n      // in `finally` so events fired between batches are ignored.\n      activeActionReporter = createAiActActionReporter(\n        planIndex,\n        replanningCycleLimit,\n        (phase, data) => this.emitAiActProgress(phase, data),\n      );\n      try {\n        await session.appendAndRun(executables.tasks);\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          initialTimeString,\n          this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n        );\n      } catch (error: any) {\n        // A completed action must not be replayed because readiness failed.\n        if (isActionReadinessError(error)) throw error;\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          timeString,\n          `Error executing running tasks: ${error?.message || String(error)}`,\n        );\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      } finally {\n        activeActionReporter = undefined;\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return appendFailedPlan(\n          'Too many errors in one planning loop',\n          planIndex,\n        );\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return appendFailedPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n          planIndex,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // We are about to replan, which means the batch we just ran did not finish\n      // the task. Any locate task in that batch that was served from cache\n      // produced an element that failed to complete the step (the action threw,\n      // or it clicked the wrong element and the goal was not reached). Mark those\n      // cache entries stale so the re-locate of the same prompt replaces them in\n      // place instead of appending a poisoning duplicate that would be matched\n      // first on the next run (#2529).\n      this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return appendFailedPlan(errorMsg, planIndex);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    // Carry the raw final text; the consumer formats it. `outputString` is the\n    // model's finalize message, with the latest plan's log/thought as fallback\n    // context for consumers that want to show why the run completed.\n    await this.emitAiActProgress('complete', {\n      planIndex: latestPlanIndex,\n      planLimit: replanningCycleLimit,\n      ...((outputString ?? latestPlanResult?.output)\n        ? { output: outputString ?? latestPlanResult?.output }\n        : {}),\n      ...(latestPlanResult?.log ? { log: latestPlanResult.log } : {}),\n      ...(latestPlanResult?.thought\n        ? { thought: latestPlanResult.thought }\n        : {}),\n    });\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        ...(opt?.context !== undefined ? { context: opt.context } : {}),\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n            executionOptions,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        } finally {\n          // Surface the captured screenshot sequence in the report, then release\n          // it from the UIContext so its base64 is not retained twice.\n          recordAndReleaseScreenshotSequence(task, uiContext);\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        // Keep assertion failures in the task result so TaskRunner can persist\n        // the model dump before aiAssert turns the result into a thrown error.\n        if (type === 'Assert') {\n          outputResult = Boolean(outputResult);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n      uiContext?: UIContext;\n    },\n  ): Promise<ExecutionResult<T>> {\n    executionOptions?.abortSignal?.throwIfAborted();\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      {\n        ...(executionOptions?.uiContext\n          ? { uiContext: executionOptions.uiContext }\n          : {}),\n        referenceImages: multimodalPrompt?.images,\n        abortSignal: executionOptions?.abortSignal,\n      },\n    );\n\n    const runner = session.getRunner();\n    const executionModelRuntime = { ...modelRuntime, executionId: runner.id };\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      executionModelRuntime,\n      opt,\n      multimodalPrompt,\n      executionOptions,\n    );\n\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n      { referenceImages: multimodalPrompt?.images },\n    );\n    const runner = session.getRunner();\n    const executionModelRuntime = { ...modelRuntime, executionId: runner.id };\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        executionModelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n/**\n * Surface a captured screenshot sequence in the report timeline, then release\n * it from the UIContext.\n *\n * When a UIObservation insight runs, the observed frames live on\n * `uiContext.screenshotSequence` only as a transient model input. This attaches\n * them to the task recorder so the report renders the full sequence the model\n * saw (the report timeline builds one screenshot per recorder item), then drops\n * the field from the UIContext so its base64 is not retained twice for the\n * lifetime of the dump.\n *\n * The last frame is the representative `uiContext.screenshot`, already shown in\n * the report, so only the earlier frames are recorded to avoid duplication.\n * Observed frames are PREPENDED to `task.recorder` (rather than appended) so\n * array order matches chronological order — they were captured before the\n * assertion's before/after screenshots.\n */\nexport function recordAndReleaseScreenshotSequence(\n  task: ExecutionTask,\n  uiContext: UIContext | undefined,\n): void {\n  const frames = uiContext?.screenshotSequence;\n  if (frames && frames.length > 1) {\n    const recorderItems: ExecutionRecorderItem[] = [];\n    for (let i = 0; i < frames.length - 1; i++) {\n      const frame = frames[i];\n      recorderItems.push({\n        type: 'screenshot',\n        ts: frame.capturedAt,\n        screenshot: frame,\n        description: `Observed frame ${i + 1}/${frames.length}`,\n        timing: 'observed-frame',\n      });\n    }\n    // Prepend observed frames so they appear before the task's own\n    // before/after screenshots in array order. Observed frames have earlier\n    // timestamps than the assertion task itself, so this keeps array order\n    // aligned with chronological order for consumers that iterate in-place.\n    task.recorder = task.recorder\n      ? [...recorderItems, ...task.recorder]\n      : recorderItems;\n  }\n  if (uiContext?.screenshotSequence) {\n    uiContext.screenshotSequence = undefined;\n  }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","requestedLimit","limit","Number","Math","TaskExecutor","title","options","ExecutionSession","Promise","runner","error","scope","phase","data","aiActProgressScope","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","planningFeedbackMaxLength","undefined","format","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","userPromptToMultimodalPrompt","task","executorContext","uiContext","assert","ms","abortSignal","action","defineActionSleep","setTimingFieldOnce","executionPlanningModel","executionDefaultModel","result","output","userPrompt","aiActContext","cacheable","replanningCycleLimitOverride","effort","fileChooserAccept","deepLocate","withFileChooser","fromIndex","i","prompt","ConversationHistory","promptDisplay","activeActionReporter","event","noIndividualLocateModel","includeLocateInPlanning","imagesIncludeCount","replanCount","yamlFlow","replanningCycleLimit","Error","appendFailedPlan","errorMsg","planIndex","errorCountInOnePlanningLoop","outputString","latestPlanResult","latestPlanIndex","getPreparedUserPrompt","promise","prepareUserPrompt","subGoalStatus","memoriesStatus","planningUiContext","timing","actionSpace","Array","console","planImpl","standardPlan","planResult","preparedUserPrompt","renderAIContext","planError","AIResponseParseError","withUsageIntent","errorMessageForAiAct","JSON","actions","thought","log","memory","usage","rawResponse","rawAssistantOutput","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","errorMessage","executables","initialTimeString","taskCountBeforeRun","createAiActActionReporter","isActionReadinessError","String","type","demand","modelRuntime","opt","multimodalPrompt","executionOptions","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","recordAndReleaseScreenshotSequence","outputResult","Boolean","executionModelRuntime","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","frames","recorderItems","frame"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsFA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAK9C,MAAMC,4BAA4B;AAElC,SAASC,yBACPC,QAAgB,EAChBC,cAAqC;IAErC,IAAIA,AAAmB,gBAAnBA,gBACF,OAAOD;IAET,MAAME,QACJD,kBAAkBE,OAAO,QAAQ,CAACF,mBAAmBA,iBAAiB,IAClEG,KAAK,KAAK,CAACH,kBACXH;IACN,IAAIE,SAAS,MAAM,IAAIE,OACrB,OAAOF;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGE,OAAO;eACtB,EAAEF,SAAS,MAAM,GAAGE,MAAM,iBAAiB,CAAC;AAC3D;AAKO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAmCQ,uBACNC,KAAa,EACbC,OAUC,EACD;QACA,OAAO,IAAIC,iBACTF,OACA,IACEC,SAAS,YACLE,QAAQ,OAAO,CAACF,QAAQ,SAAS,IACjCE,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,aAAaA,SAAS;YACtB,iBAAiBA,SAAS;YAC1B,kBAAkB,OAAOG,QAAQC;gBAC/B,MAAM,IAAI,CAAC,KAAK,EAAE,mBAAmBD,QAAQC;gBAC7C,MAAMJ,SAAS,mBAAmBG,QAAQC;YAC5C;YACA,GAAIJ,SAAS,cAAc;gBAAE,aAAaA,QAAQ,WAAW;YAAC,IAAI,CAAC,CAAC;QACtE;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IAQA,MAAc,aACZK,KAAa,EACbC,KAAa,EACbC,IAAa,EACE;QACf,MAAM,IAAI,CAAC,KAAK,EAAE,aAAaF,OAAOC,OAAOC;IAC/C;IAOQ,kBACND,KAAyB,EACzBC,IAAuB,EACR;QACf,OAAO,IAAI,CAAC,YAAY,CAACC,oBAAoBF,OAAOC;IACtD;IASQ,0BACNE,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CACpC,CAAC,EAAEE,gBAAgB,EAAEC,yBAAyB,EAAE,GAC9CD,mBACI;gBACEtB,yBACEsB,kBACAC;aAEH,GACD,EAAE;QAEV,OAAOF,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBG;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOb,OAAO;YACdf,QACE,CAAC,gEAAgE,EAAEe,OAAO;QAE9E;aAEAf,QACE;QAKN,OAAO6B,sBAAsBD;IAC/B;IAEA,MAAa,wBACXE,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BrB,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACmB,OAAOC,eAAeC,cAAcrB;IACpE;IAEA,MAAM,uBACJsB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,mBAAmBL,mBAE9C;YACE,iBAAiBM,6BAA6BN,kBAAkB;QAClE;QAGF,MAAMO,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLP;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOM;gBACf,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMpB,SAASsB,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACI;QAE3B,OAAO;YACL1B;QACF;IACF;IAEA,MAAM,MAAM8B,EAAU,EAAEC,WAAyB,EAAiB;QAChE,MAAMT,UAAU,IAAI,CAAC,sBAAsB,CAAC,SAAS;YAAES;QAAY;QACnE,MAAMC,SAASC,kBAAkBF;QACjC,MAAMT,QAAQ,YAAY,CAAC;YACzB,MAAM;YACN,SAASU,OAAO,IAAI;YACpB,OAAO;gBAAE,QAAQF;YAAG;YACpB,UAAU,OAAO,EAAEJ,IAAI,EAAE;gBACvBG,OACEpC,OAAO,QAAQ,CAACqC,OAAOA,KAAK,GAC5B,CAAC,6DAA6D,EAAEA,IAAI;gBAEtEI,mBAAmBR,KAAK,MAAM,EAAE;gBAChC,IAAI;oBACF,MAAMM,OAAO,IAAI,CAAC;wBAAE,QAAQF;oBAAG;gBACjC,SAAU;oBACRI,mBAAmBR,KAAK,MAAM,EAAE;gBAClC;YACF;QACF;IACF;IAEA,MAAM,SACJ9B,KAAa,EACboB,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BrB,OAA8D,EACpC;QAC1B,MAAMyB,UAAU,IAAI,CAAC,sBAAsB,CAAC1B,OAAOC;QACnD,MAAMG,SAASsB,QAAQ,SAAS;QAChC,MAAMa,yBAAyB;YAAE,GAAGlB,aAAa;YAAE,aAAajB,OAAO,EAAE;QAAC;QAC1E,MAAMoC,wBAAwB;YAAE,GAAGlB,YAAY;YAAE,aAAalB,OAAO,EAAE;QAAC;QACxE,MAAM,EAAES,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDO,OACAmB,wBACAC,uBACA;YAAE,aAAavC,SAAS;QAAY;QAEtC,MAAMwC,SAAS,MAAMf,QAAQ,YAAY,CAACb;QAC1C,MAAM,EAAE6B,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAtC;QACF;IACF;IAEA,MAAM,OACJuC,UAAuB,EACvBtB,aAA2B,EAC3BC,YAA0B,EAC1BsB,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,SAAsB,SAAS,EAC/BC,iBAA4B,EAC5BC,UAAoB,EACpBd,WAAyB,EACzBV,aAAmC,EASnC;QACAU,aAAa;QACb,OAAOe,gBAAgB,IAAI,CAAC,SAAS,EAAEF,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBL,YACAtB,eACAC,cACAsB,cACAC,WACAC,8BACAC,QACAE,YACAd,aACAV;IAGN;IAcQ,gCACNrB,MAAkB,EAClB+C,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAIhD,OAAO,KAAK,CAAC,MAAM,EAAEgD,IAAK;YACpD,MAAMtB,OAAO1B,OAAO,KAAK,CAACgD,EAAE;YAC5B,IACEtB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMuB,SAAUvB,KAAK,KAAK,EAAsC;gBAChE,IAAIuB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZV,UAAuB,EACvBtB,aAA2B,EAC3BC,YAA0B,EAC1BsB,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,SAAsB,SAAS,EAC/BE,UAAoB,EACpBd,WAAyB,EACzBV,aAAmC,EASnC;QACA,MAAMf,sBAAsB,IAAI4C;QAChC,MAAMC,gBACJ9B,eAAe,UAAUG,mBAAmBe;QAM9C,IAAIa;QAIJ,MAAM9B,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAaF,eAAe,QAAQ,OAAO8B,gBAC3C;YACEpB;YACA,iBAAiBN,6BAA6Bc,aAAa;YAC3D,aAAa,OAAOc;gBAClB,MAAMD,uBAAuBC;YAC/B;QACF;QAEF,MAAMrD,SAASsB,QAAQ,SAAS;QAChCL,gBAAgB;YAAE,GAAGA,aAAa;YAAE,aAAajB,OAAO,EAAE;QAAC;QAC3DkB,eAAe;YAAE,GAAGA,YAAY;YAAE,aAAalB,OAAO,EAAE;QAAC;QAEzD,MAAMsD,0BAA0BrC,AAA8B,cAA9BA,cAAc,MAAM,CAAC,IAAI;QACzD,MAAMsC,0BACJZ,AAAW,gBAAXA,UAA0BW;QAC5B,MAAME,qBAAqBb,AAAW,gBAAXA,SAAyB,IAAI;QAExD3D,MAAM,sCAAsCuE,yBAAyB;YACnEZ;YACAW;QACF;QAEA,IAAIG,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJjB,gCAAgC,IAAI,CAAC,oBAAoB;QAC3D,IAAIiB,AAAyB9C,WAAzB8C,sBACF,MAAM,IAAIC,MACR;QAIJ,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS;YACpC,QAAQT;YACR,WAAWQ;QACb;QAEA,MAAME,mBAAmB,OACvBC,UACAC;YAKA,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU;gBACrC,GAAIA,YAAY;oBAAEA;gBAAU,IAAI,CAAC,CAAC;gBAClC,WAAWJ;gBACX,OAAOG;YACT;YACA,OAAOxC,QAAQ,eAAe,CAACwC;QACjC;QAEA,IAAIE,8BAA8B;QAClC,IAAIC;QACJ,IAAIC;QACJ,IAAIC,kBAAkB;QAEtB,IAAIpC,aAAa,SACf,OAAO8B,iBACL,CAAC,cAAc,EAAE9B,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMqC,wBAAyB,AAAC;YAC9B,IAAIC;YAEJ,OAAO;gBACLA,YAAYC,kBAAkB/B;gBAC9B,OAAO8B;YACT;QACF;QAGA,MAAO,KAAM;YACX,MAAMN,YAAYN,cAAc;YAChCU,kBAAkBJ;YAGlB,IAAIhC,aAAa,SACf,OAAO8B,iBACL,CAAC,cAAc,EAAE9B,YAAY,MAAM,IAAI,yBAAyB,EAChEgC;YAKJ,MAAMQ,gBAAgBjE,oBAAoB,cAAc,MAAMO;YAG9D,MAAM2D,iBAAiBlE,oBAAoB,cAAc,MAAMO;YAE/D,MAAMwB,SAAS,MAAMf,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBiB;oBACjB,GAAIlB,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACNsC;oBACAnB;oBACAgB;oBACAb;oBACA,GAAI4B,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO7C;oBACf,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM6C,oBAAoB7C;oBAC1B,MAAM8C,SAAS/C,gBAAgB,IAAI,CAAC,MAAM;oBAC1C,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB;wBAC5CoC;wBACA,WAAWJ;wBACX,YAAYc,kBAAkB,UAAU;oBAC1C;oBAEA,MAAME,cAAc,IAAI,CAAC,cAAc;oBACvC3F,MACE,sCACA2F,YAAY,GAAG,CAAC,CAAC3C,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhDH,OAAO+C,MAAM,OAAO,CAACD,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBE,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJ7D,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrC8D;oBAEN,IAAIC;oBACJ,IAAI;wBACF,MAAMC,qBAAqB,MAAMb;wBAEjClC,mBAAmBwC,QAAQ;wBAC3BM,aAAa,MAAMF,SAASG,oBAAoB;4BAC9C,SAASR;4BACT,eAAeS,gBAAgB1C;4BAC/BmC;4BACA,cAAc1D;4BACdX;4BACAiD;4BACAC;4BACAb;4BACAZ;wBACF;oBACF,EAAE,OAAOoD,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7CzD,gBAAgB,IAAI,CAAC,KAAK,GAAG0D,gBAC3BF,UAAU,KAAK,EACf;4BAEFxD,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAawD,UAAU,WAAW;gCAClC,oBAAoBA,UAAU,kBAAkB;4BAClD;wBACF;wBACA,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1CpB;4BACA,WAAWJ;4BACX,OAAO2B,qBAAqBH;wBAC9B;wBACA,MAAMA;oBACR,SAAU;wBACRjD,mBAAmBwC,QAAQ;oBAC7B;oBACA1F,MAAM,cAAcuG,KAAK,SAAS,CAACP,YAAY,MAAM;oBAErD,MAAM,EACJQ,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN1F,KAAK,EACL2F,KAAK,EACLC,WAAW,EACXC,kBAAkB,EAClBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGnB;oBACJf,eAAegC;oBAEftE,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCkE;wBACAC;oBACF;oBACAnE,gBAAgB,IAAI,CAAC,KAAK,GAAG0D,gBAAgBO,OAAO;oBACpDjE,gBAAgB,IAAI,CAAC,iBAAiB,GAAGoE;oBACzCpE,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS6D,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUX,WAAW,QAAQ;wBAC7B,QAAQiB;wBACR,wBAAwBjB,WAAW,sBAAsB;wBACzDkB;wBACAC;oBACF;oBACAxE,gBAAgB,SAAS,GAAG8C;oBAK5B,IAAIiB,OAAOD,SACT,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB;wBAC3C1B;wBACA,WAAWJ;wBACX,GAAI+B,MAAM;4BAAEA;wBAAI,IAAI,CAAC,CAAC;wBACtB,GAAID,UAAU;4BAAEA;wBAAQ,IAAI,CAAC,CAAC;oBAChC;oBAGF,IAAIxF,OAAO;wBACT,MAAMmG,eAAe,CAAC,oBAAoB,EAAEnG,MAAM,EAAE,EAAEyF,OAAO,IAAI;wBACjE,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1C3B;4BACA,WAAWJ;4BACX,OAAOyC;wBACT;oBACF;oBAEAvE,OAAO,CAAC5B,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEyF,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBAA2B;wBAC7B,MAAMI,eAAe,CAAC,aAAa,EAAEH,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;wBACnG,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1C3B;4BACA,WAAWJ;4BACX,OAAOyC;wBACT;wBACAvE,OAAO,OAAOuE;oBAChB;oBAEA,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMpB,aAAa3C,QAAQ;YAC3B6B,mBAAmBc;YAGnB,MAAMhE,QAAQgE,YAAY,WAAW,EAAE;YACvCtB,SAAS,IAAI,IAAKsB,YAAY,YAAY,EAAE;YAE5C,IAAIqB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CrF,OACAC,eACAC,cACA;oBACEuB;oBACAI;oBACAd;gBACF;YAEJ,EAAE,OAAO9B,OAAO;gBACd,OAAO4D,iBACL,CAAC,4CAA4C,EAAEyB,qBAAqBrF,OAAO,SAAS,EAAEsF,KAAK,SAAS,CAClGvE,QACC,EACH+C;YAEJ;YACA,IAAIzD,oBAAoB,sBAAsB,EAC5CuE,QAAQ,IAAI,CACV,8FACAvE,oBAAoB,sBAAsB;YAK9C,MAAMgG,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqBvG,OAAO,KAAK,CAAC,MAAM;YAI9CoD,uBAAuBoD,0BACrBzC,WACAJ,sBACA,CAACxD,OAAOC,OAAS,IAAI,CAAC,iBAAiB,CAACD,OAAOC;YAEjD,IAAI;gBACF,MAAMkB,QAAQ,YAAY,CAAC+E,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5B/F,qBACAgG,mBACA,IAAI,CAAC,uBAAuB,CAACtG,OAAO,KAAK,CAAC,KAAK,CAACuG;YAEpD,EAAE,OAAOtG,OAAY;gBAEnB,IAAIwG,uBAAuBxG,QAAQ,MAAMA;gBAEzC+D;gBACA,MAAMzD,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEN,OAAO,WAAWyG,OAAOzG,QAAQ;gBAErEjB,MACE,yFACAiB,iBAAiB2D,QAAQ3D,MAAM,OAAO,GAAGyG,OAAOzG,QAChD,6CACA+D;YAEJ,SAAU;gBACRZ,uBAAuBvC;YACzB;YAEA,IAAImD,8BAA8B7E,uCAChC,OAAO0E,iBACL,wCACAE;YAKJ,IAAIhC,aAAa,SACf,OAAO8B,iBACL,CAAC,cAAc,EAAE9B,YAAY,MAAM,IAAI,yBAAyB,EAChEgC;YAKJ,IAAI,CAACiB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAAChF,QAAQuG;YAG7C,EAAE9C;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMG,WAAW,CAAC,UAAU,EAAEH,qBAAqB,4JAA4J,CAAC;gBAChN,OAAOE,iBAAiBC,UAAUC;YACpC;YAEA,IAAI,CAACzD,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAKA,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY;YACvC,WAAW4D;YACX,WAAWR;YACX,GAAKM,gBAAgBC,kBAAkB,SACnC;gBAAE,QAAQD,gBAAgBC,kBAAkB;YAAO,IACnD,CAAC,CAAC;YACN,GAAIA,kBAAkB,MAAM;gBAAE,KAAKA,iBAAiB,GAAG;YAAC,IAAI,CAAC,CAAC;YAC9D,GAAIA,kBAAkB,UAClB;gBAAE,SAASA,iBAAiB,OAAO;YAAC,IACpC,CAAC,CAAC;QACR;QAEA,OAAO;YACL,QAAQ;gBACNR;gBACA,QAAQO;YACV;YACAjE;QACF;IACF;IAEQ,oBACN2G,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAEC,EACD;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASN;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,GAAIA,KAAK,YAAYjG,SAAY;oBAAE,SAASiG,IAAI,OAAO;gBAAC,IAAI,CAAC,CAAC;gBAC9D,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOM;gBACf,MAAM,EAAExF,IAAI,EAAE,GAAGwF;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ3F,KAAK,GAAG,GAAG;wBACT2F;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA3F,KAAK,KAAK,GAAG2D,gBAAgBgC,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjB3F,KAAK,iBAAiB,GAAG2F,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMzF,YAAYsF,YAAY,SAAS;gBACvCrF,OAAOD,WAAW;gBAElB,MAAM0F,mBAAmBX,AAAS,YAATA;gBACzB,IAAIY,cAAcX;gBAClB,IAAIY,cAAc;gBAClB,IAAIF,oBAAqBX,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEa,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF,OAAO,IAAIU,kBAAkB;oBAC3BE,cAAcb;oBACdY,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF;gBAEA,IAAIc;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIb,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1D9H,MAAM;oBACN,MAAM4I,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAd,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAV,cACAC,KACAa,sBACAZ,kBACAnF,WACAoF;gBAEJ,EAAE,OAAO/G,OAAO;oBACd,IAAIA,iBAAiB6H,cACnBV,UAAUnH,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBAGR8H,mCAAmCrG,MAAME;gBAC3C;gBAEA,MAAM,EAAExB,IAAI,EAAEqF,OAAO,EAAE4B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAe5H;gBACnB,IAAIkH,kBAEF,IAAI,AAAgB,YAAhB,OAAOlH,MACT4H,eAAe5H;qBACV,IAAIuG,AAAS,cAATA,MAEPqB,eADE5H,QAAAA,OACa,QAECA,IAAY,CAACoH,YAAY;qBAEtC,IAAIpH,QAAAA,MACT4H,eAAe;qBAGf,IAAI5H,MAAM,CAACoH,YAAY,KAAK3G,QAC1BmH,eAAgB5H,IAAY,CAACoH,YAAY;qBACpC,IAAIpH,MAAM,WAAWS,QAC1BmH,eAAgB5H,KAAa,MAAM;qBAEnCyB,OAAO,OAAO;gBAOpB,IAAI8E,AAAS,aAATA,MACFqB,eAAeC,QAAQD;gBAGzB,OAAO;oBACL,QAAQA;oBACR,KAAKb;oBACL1B;gBACF;YACF;QACF;QAEA,OAAOwB;IACT;IACA,MAAM,yBACJN,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAGC,EAC4B;QAC7BA,kBAAkB,aAAa;QAC/B,MAAM1F,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEoF,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASrB,KAAK,SAAS,CAACqB,UAEvD;YACE,GAAII,kBAAkB,YAClB;gBAAE,WAAWA,iBAAiB,SAAS;YAAC,IACxC,CAAC,CAAC;YACN,iBAAiBD,kBAAkB;YACnC,aAAaC,kBAAkB;QACjC;QAGF,MAAMhH,SAASsB,QAAQ,SAAS;QAChC,MAAM4G,wBAAwB;YAAE,GAAGrB,YAAY;YAAE,aAAa7G,OAAO,EAAE;QAAC;QACxE,MAAMiH,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CN,MACAC,QACAsB,uBACApB,KACAC,kBACAC;QAGF,MAAM3E,SAAS,MAAMf,QAAQ,YAAY,CAAC2F;QAE1C,IAAI,CAAC5E,QACH,MAAM,IAAIuB,MACR;QAIJ,MAAM,EAAEtB,MAAM,EAAEmD,OAAO,EAAE,GAAGpD;QAE5B,OAAO;YACLC;YACAmD;YACAzF;QACF;IACF;IAEA,MAAM,QACJmI,SAAsB,EACtBrB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEuB,UAAU,EAAErB,gBAAgB,EAAE,GAAGsB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM9G,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW+G,cACxB;YAAE,iBAAiBvB,kBAAkB;QAAO;QAE9C,MAAM/G,SAASsB,QAAQ,SAAS;QAChC,MAAM4G,wBAAwB;YAAE,GAAGrB,YAAY;YAAE,aAAa7G,OAAO,EAAE;QAAC;QACxE,MAAM,EACJuI,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG7B;QACJ,MAAM8B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA9G,OAAOsG,WAAW;QAClBtG,OAAO0G,WAAW;QAClB1G,OAAO2G,iBAAiB;QAExB3G,OACE2G,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAMhC,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAmB,YACAF,uBACAU,mBACA7B;YAGF,MAAM1E,SAAU,MAAMf,QAAQ,YAAY,CAAC2F;YAO3C,IAAI5E,QAAQ,QACV,OAAO;gBACL,QAAQxB;gBACRb;YACF;YAGFgJ,eACE3G,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAE+F,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAM1D,UAAU,CAAC,kBAAkB,EAAE+C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAG3D;oBAAQ;iBAAE,EAC9DoB,cACAA;gBAEF,IAAIwC,UAAU,CAAC,EAAE,EACf,MAAM/H,QAAQ,YAAY,CAAC+H,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO/H,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAE0H,cAAc;IACnE;IAniCA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IASC,CACD;QAtCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAqBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;YACrC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAqgCF;AAkBO,SAASzB,mCACdrG,IAAmB,EACnBE,SAAgC;IAEhC,MAAM8H,SAAS9H,WAAW;IAC1B,IAAI8H,UAAUA,OAAO,MAAM,GAAG,GAAG;QAC/B,MAAMC,gBAAyC,EAAE;QACjD,IAAK,IAAI3G,IAAI,GAAGA,IAAI0G,OAAO,MAAM,GAAG,GAAG1G,IAAK;YAC1C,MAAM4G,QAAQF,MAAM,CAAC1G,EAAE;YACvB2G,cAAc,IAAI,CAAC;gBACjB,MAAM;gBACN,IAAIC,MAAM,UAAU;gBACpB,YAAYA;gBACZ,aAAa,CAAC,eAAe,EAAE5G,IAAI,EAAE,CAAC,EAAE0G,OAAO,MAAM,EAAE;gBACvD,QAAQ;YACV;QACF;QAKAhI,KAAK,QAAQ,GAAGA,KAAK,QAAQ,GACzB;eAAIiI;eAAkBjI,KAAK,QAAQ;SAAC,GACpCiI;IACN;IACA,IAAI/H,WAAW,oBACbA,UAAU,kBAAkB,GAAGf;AAEnC"}

package/dist/es/agent/utils.mjs

@@ -153,7 +153,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.15.1";
+const getMidsceneVersion = ()=>"1.15.2-beta-20261012093542.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/prompt/planning/system-prompt.mjs

@@ -205,6 +205,8 @@ ${hasRunAdbShell ? "- If the user's task can be completed with the RunAdbShell a
 
 ### Interpreting Scroll and Swipe Directions
 
+- For a relative Swipe, express distance in pixels of the current screenshot. The runtime converts this model-planned distance to device logical units. This rule applies to Swipe distance, not Scroll distance.
+
 Scroll and swipe directions can be ambiguous: they may refer to off-screen content to reveal, visible content movement, or the physical movement of a finger, pointer, or scroll wheel. Infer the intended result from the user's goal, the interaction target, the current UI, and common usage in the user's language, rather than mechanically copying the user's direction word.
 
 In the current Action Space, \`direction\` has the following meanings:

package/dist/es/ai-model/prompt/planning/system-prompt.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/planning/system-prompt.mjs","sources":["../../../../../src/ai-model/prompt/planning/system-prompt.ts"],"sourcesContent":["import type { DeviceAction } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport type { StandardPlanningProtocol } from '../../model-adapter/planning-protocol';\nimport type { LocateResultPromptSpec } from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport { locateGroundingRules } from '../locate';\nimport {\n  buildActionOutputExample,\n  createSampleTapAction,\n} from './action-output-example';\nimport { buildPlanningActionSpaceDescription } from './action-space-description';\nimport { buildPlanningMultiTurnExample } from './multi-turn-example';\n\ntype BuildStandardPlanningSystemPromptInput = {\n  actionSpace: DeviceAction<any>[];\n  includeSubGoals?: boolean;\n  includeThought?: boolean;\n  includeLog?: boolean;\n  planningProtocol: StandardPlanningProtocol;\n} & (\n  | {\n      includeLocateInPlanning: true;\n      locatePromptSpec: LocateResultPromptSpec;\n    }\n  | {\n      includeLocateInPlanning: false;\n      locatePromptSpec?: never;\n    }\n);\n\nexport async function buildStandardPlanningSystemPrompt(\n  input: BuildStandardPlanningSystemPromptInput,\n) {\n  const {\n    actionSpace,\n    includeLocateInPlanning,\n    locatePromptSpec,\n    includeSubGoals,\n    includeThought = true,\n    includeLog = true,\n    planningProtocol,\n  } = input;\n  const actionOutputProtocol = planningProtocol.actionOutputProtocol;\n  const preferredLanguage = getPreferredLanguage();\n\n  if (includeLocateInPlanning && !locatePromptSpec) {\n    throw new Error(planningModelFamilyRequiredForLocateMessage());\n  }\n\n  const actionSpaceDescription = buildPlanningActionSpaceDescription({\n    actionSpace,\n    locatePromptSpec,\n    planningProtocol,\n  });\n  const hasRunAdbShell = actionSpace.some(\n    (action) => action.name === 'RunAdbShell',\n  );\n  const actionOutputTagsText = actionOutputProtocol.actionOutputTagNames\n    .map((tagName) => `<${tagName}>`)\n    .join(', ');\n\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n  const renderSubGoalsContent = (content: string, fallbackContent = '') =>\n    shouldIncludeSubGoals ? content : fallbackContent;\n  const renderThoughtContent = (content: string, fallbackContent = '') =>\n    includeThought ? content : fallbackContent;\n  const renderLogContent = (content: string, fallbackContent = '') =>\n    includeLog ? content : fallbackContent;\n\n  // Step numbering adjusts based on whether sub-goals are included\n  // When includeSubGoals=false, memory step is skipped\n  const completionCheckStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = completionCheckStepNumber + 1;\n\n  return `\nTarget: You are an expert to manipulate the UI to accomplish the user's instruction. User will give you an instruction, some screenshots, background knowledge and previous logs indicating what have been done. Your task is to accomplish the instruction by thinking through the path to complete the task and give the next action to execute.\n\n## Step 1: ${renderSubGoalsContent(\n    'Observe and Plan (related tags: <planning>, <update-plan-content>, <mark-sub-goal-done>)',\n    renderThoughtContent('Observe (related tags: <planning>)', 'Observe'),\n  )}\n\nFirst, observe the current screenshot and previous logs${renderSubGoalsContent(\n    \", then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot.\",\n    ' to understand the current state.',\n  )}\n\n${renderSubGoalsContent(`### Observation Guidelines\n\n- Treat visible summaries, thumbnails, cropped content, and partially visible lists as potentially incomplete when the task depends on precise details.\n- If the current view does not provide enough information to decide safely, use available UI affordances such as opening details, expanding content, previewing, enlarging, zooming, or scrolling before acting.`)}\n\n${renderThoughtContent(`### <planning> tag (REQUIRED)\n\nREQUIRED: You MUST always output the <planning> tag. Never skip it.\n\nWrite the content of the <planning> tag in ${preferredLanguage}.\n\nInclude your planning details in the <planning> tag. It should answer: ${renderSubGoalsContent(\n  \"What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action?\",\n  'What is the current state based on the screenshot? What should be the next action?',\n)} Write it naturally without numbering or section headers.\n\nCRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: \"fill out the form\" means only fill fields, do NOT submit; \"click the button\" means only click, do NOT wait for page load or verify results; \"type 'hello'\" means only type, do NOT press Enter.`)}\n\n${renderSubGoalsContent(`### <update-plan-content> tag\n\nUse this structure to give or update your plan:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">sub goal description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">sub goal description</sub-goal>\n  ...\n</update-plan-content>\n\n### <mark-sub-goal-done> tag\n\nUse this structure to mark a sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: You MUST only mark a sub-goal as \"finished\" AFTER you have confirmed the task is actually completed by observing the result in the screenshot. Do NOT mark a sub-goal as done just because you expect the next action will complete it. Wait until you see visual confirmation in the screenshot that the sub-goal has been achieved.\n\n### Note\n\nDuring execution, you can call <update-plan-content> at any time to update the plan based on the latest screenshot and completed sub-goals.\n\n### Example\n\nIf the user wants to \"log in to a system using username and password, complete all to-do items, and submit a registration form\", you can break it down into the following sub-goals:\n\n<planning>...</planning>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Log in to the system</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Complete all to-do items</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Submit the registration form</sub-goal>\n</update-plan-content>\n\nAfter logging in and seeing the to-do items, you can mark the sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nAt this point, the status of all sub-goals is:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished\" />\n  <sub-goal index=\"2\" status=\"pending\" />\n  <sub-goal index=\"3\" status=\"pending\" />\n</update-plan-content>\n\nAfter some time, when the last sub-goal is also completed, you can mark it as done as well:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>`)}\n\n${renderSubGoalsContent(`## Step 2: Memory Data from Current Screenshot (related tags: <memory>)\n\nUse <memory> to record clear, task-relevant information from the current screenshot that may be needed in later steps. The current screenshot will not be available later, so memory should preserve enough detail for future reasoning, verification, or action.\n\n- Record information completely and exactly as shown. Do not summarize, translate, normalize, or merge values that may matter later.\n- When recording an item, include the item itself, its exact task-relevant details, and the visible cue or UI context that identifies where it came from when relevant.\n- Keep similar or repeated items as separate memory entries unless their task-relevant details are confirmed to be the same.\n- After navigation, scrolling, editing, deletion, saving, or other screen changes, treat remembered positions, order, indexes, and UI bindings as references only. Re-check the current screen before acting on them.\n\nExamples:\n- If you need to find an item and later assert its details, record the item name and the exact details needed for the assertion, such as status, price, date, owner, description, or other visible fields.\n- If you need to compare multiple similar results, record each candidate separately with its exact distinguishing details and visible context.\n- If you need to copy information from one place to another, record the exact source value and the target field or UI cue it should be mapped to.\n\nDon't use this tag if no information needs to be preserved.`)}\n\n## Step ${completionCheckStepNumber}: ${renderSubGoalsContent(\n    'Check if Goal is Accomplished',\n    'Check if the Instruction is Fulfilled',\n  )} (related tags: <complete>)\n\nDetermine if the entire task is completed${renderSubGoalsContent(' based on the current screenshot and the status of all sub-goals')}.\n\n### CRITICAL: The User's Instruction is the Supreme Authority\n\nThe user's instruction defines the EXACT scope of what you must accomplish. You MUST follow it precisely - nothing more, nothing less. Violating this rule may cause severe consequences such as data loss, unintended operations, or system failures.\n\n**Explicit instructions vs. High-level goals:**\n- If the user gives you **explicit operation steps** (e.g., \"click X\", \"type Y\", \"fill out the form\"), treat them as exact commands. Execute ONLY those steps, nothing more.\n- If the user gives you a **high-level goal** (e.g., \"log in to the system\", \"complete the purchase\"), you may determine the necessary steps to achieve it.\n\n**What \"${renderSubGoalsContent('goal accomplished', 'instruction fulfilled')}\" means:**\n- The ${renderSubGoalsContent('goal is accomplished', 'instruction is fulfilled')} when you have done EXACTLY what the user asked - no extra steps, no assumptions.\n- Do NOT perform any action beyond the explicit instruction, even if it seems logical or helpful.\n\n**Examples - Explicit instructions (execute exactly, no extra steps):**\n- \"fill out the form\" → ${renderSubGoalsContent('Goal accomplished', 'Instruction fulfilled')} when all fields are filled. Do NOT submit the form.\n- \"click the login button\" → ${renderSubGoalsContent('Goal accomplished', 'Instruction fulfilled')} once clicked. Do NOT wait for page load or verify login success.\n- \"type 'hello' in the search box\" → ${renderSubGoalsContent('Goal accomplished', 'Instruction fulfilled')} when 'hello' is typed. Do NOT press Enter or trigger search.\n- \"select the first item\" → ${renderSubGoalsContent('Goal accomplished', 'Instruction fulfilled')} when selected. Do NOT proceed to checkout.\n\n**Change completion:**\n- If the requested outcome is a durable change, such as create, edit, update, delete, save, send, submit, apply, or publish, do not stop at an unsaved draft, open editor, temporary input, transient selection, or staged value. Continue through the app/page's normal completion control such as Save, Done, Confirm, OK, Submit, Apply, Send, or Publish before completing, so the result remains after leaving the screen.\n- If the user only asks for an intermediate UI state, such as typing text, selecting an option, filling fields, or opening a screen without saving/submitting/applying, stop once that exact state is reached.\n\n**Special case - Scrollable option lists and dropdowns:**\n- When choosing an item from a scrollable select, dropdown, listbox, menu, or similar option list, first open the control if it is closed. Once the list is open, interact with the list itself, not the page.\n- If the target option is visible in the open list, Tap that exact option immediately.\n- If the list is open but the target option is not visible, try to find it by scrolling the open list/dropdown before giving up or interacting with other elements.\n- For an open dropdown/list, prefer small incremental Scroll actions with an explicit distance (typically 50-120 pixels) and a locate target describing the open list/dropdown. Do NOT omit distance while searching within a list, because the default scroll distance can skip over relevant options and cause oscillation.\n- While searching within an open dropdown/list, use short scrolls so intermediate options are not skipped.\n- After selecting the target, if the trigger text or result text shows the requested option, treat the current selection step as fulfilled and continue evaluating the remaining user instruction.\n\n**Special case - Text hidden by a narrow input field:**\n- CRITICAL PRIORITY OVERRIDE - Input verification after an input action:\n- This rule overrides the general requirement to verify the exact target text from the screenshot.\n- If the previous step already executed an input action, and the current input field is not empty, you MUST directly treat that input as successful.\n- In this situation, do NOT use the visible text in the screenshot to decide that the input is incomplete, incorrect, missing characters, out of order, or needs correction.\n- Do NOT perform ClearInput, another Input, KeyboardPress, or any other action intended to \"fix the input content\" just because the visible text looks different from the target text, has abnormal character order, missing characters, abnormal spacing, suspicious visual recognition results, or appears to have selected/highlighted text.\n- The general rule \"do EXACTLY what the user asked\" still applies to the intended input value you execute, but it MUST NOT be enforced by re-validating the visible text in the screenshot after the input action.\n- Differences in visible text must be assumed to be caused by clipping, horizontal scrolling, narrow input fields, text selection, caret position, or visual recognition errors rather than input failure.\n- Retry input only when the input field is clearly still empty, or when the page shows an explicit error message.\n\n**Special case - Assertion instructions:**\n- If the user's instruction includes an assertion (e.g., \"verify that...\", \"check that...\", \"assert...\"), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark ${renderSubGoalsContent('the goal', 'it')} as failed (success=\"false\").\n- If the page is still loading (e.g., you see a loading spinner, skeleton screen, or progress bar), do NOT assert yet. Wait for the page to finish loading before evaluating the assertion.\n\n### Completion Criteria for Process-required Instructions\n\nIf the user's instruction includes explicit operation steps, ordering requirements, or action requirements, it is a process-required instruction.\n\nFor process-required instructions, do NOT treat the task as complete only because the current screenshot already shows the final expected state. Do NOT infer that earlier steps were executed from the final UI state.\n\nYou may output <complete success=\"true\"> only when the current execution history, previous logs, or the screenshot after the most recent action proves that every explicit step required by the user has been completed, and the final check condition is also satisfied.\n\nIf any explicit step lacks completion evidence in the current execution history, continue with the next missing step instead of outputting <complete>, even if the current screenshot appears to satisfy the final condition.\n\n${renderSubGoalsContent(\n  '',\n  `**Page navigation restriction:**\n- Unless the user's instruction explicitly asks you to click a link, jump to another page, or navigate to a URL, you MUST complete the task on the current page only.\n- Do NOT navigate away from the current page on your own initiative (e.g., do not click links that lead to other pages, do not use browser back/forward, do not open new URLs).\n- If the task cannot be accomplished on the current page and the user has not instructed you to navigate, report it as a failure (success=\"false\") instead of attempting to navigate to other pages.`,\n)}\n\n### Output Rules\n\n- If the task is NOT complete, skip this section and continue to Step ${actionStepNumber}.\n- Use the <complete success=\"true|false\">message</complete> tag to output the result if the goal is accomplished or failed.\n  - the 'success' attribute is required. ${renderSubGoalsContent('It means whether the expected goal is accomplished based on what you observe in the current screenshot and the current execution history. ')}No matter what errors occurred during execution, set success=\"true\" only when the current execution history shows that all steps required by the user have been completed and the final state satisfies the requirement. If the user asks for explicit operation steps or an ordered workflow, do not treat those steps as completed only because the current screenshot already shows the final expected state. If the ${renderSubGoalsContent('expected goal is not accomplished and cannot be accomplished', 'instruction is not fulfilled and cannot be fulfilled')}, set success=\"false\".\n  - the 'message' is the information that will be provided to the user. If the user asks for a specific format, strictly follow that.\n- If you output <complete>, do NOT output ${actionOutputTagsText}. The task ends here.\n\n## Step ${actionStepNumber}: Determine Next Action (related tags: ${renderLogContent('<log>, ')}${actionOutputTagsText}, <error>)\n\nONLY if the task is not complete: Think what the next action is according to the current screenshot${renderSubGoalsContent(' and the plan')}.\n\n- Don't give extra actions or plans beyond the instruction or the plan. For example, don't try to submit the form if the instruction is only to fill something.\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully. Otherwise, retry or do something else to recover.\n- Give just the next ONE action you should do (if any)\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 3 times, you should think this is an error and set the \"error\" field to the error message.\n\n### Action Guidelines\n\n${\n  hasRunAdbShell\n    ? \"- If the user's task can be completed with the RunAdbShell action, prefer using the RunAdbShell action.\"\n    : ''\n}\n- If Swipe is available in the current Action Space, for touch continuous controls that set a value along a track, such as a slider, prefer Swipe from the current handle or filled position to the requested track endpoint instead of tapping the endpoint.\n- When editing existing text in a UI field, preserve all existing text by moving the cursor and typing/deleting the minimal necessary characters.\n- For insert/prepend/append edits, use CursorMove when the caret must be adjusted precisely, then use Input with mode \"typeOnly\" for inserted characters and KeyboardPress for newlines or deletion. If the caret lands in the wrong position, recover with CursorMove, KeyboardPress, or undo and retry cursor placement; do not switch to replace as a fallback for cursor placement failures.\n\n### Interpreting Scroll and Swipe Directions\n\nScroll and swipe directions can be ambiguous: they may refer to off-screen content to reveal, visible content movement, or the physical movement of a finger, pointer, or scroll wheel. Infer the intended result from the user's goal, the interaction target, the current UI, and common usage in the user's language, rather than mechanically copying the user's direction word.\n\nIn the current Action Space, \\`direction\\` has the following meanings:\n\n- Scroll follows the traditional mouse-wheel convention (with natural scrolling disabled). Its direction refers to the off-screen content to reveal; under this convention, it matches the wheel movement direction and is opposite to the content movement direction.\n- Swipe follows the finger-movement convention. Its direction refers to finger movement; for controls where content follows the finger, it matches the content movement direction and is opposite to the direction from which off-screen content is revealed.\n\nUse the following priority order:\n\n1. Prioritize the user's goal or expected result, even when it conflicts with a direction word. For example, for \"swipe the date picker down to increase the date\", if dates increase from top to bottom, use Swipe up to move a larger date from below into the selected position.\n2. Otherwise, if the user specifies a moving object or movement path, follow that object's physical movement. For example, \"swipe the finger to the left\" means Swipe left; \"move the page content from top to bottom\" means moving content downward, corresponding to Swipe down on a page where content follows the finger.\n3. Otherwise, consider the interaction target and common usage in the user's language. For content browsing, such as pages and lists, scroll directions usually refer to the off-screen content the user wants to see. For directly manipulated controls, such as sliders and wheel pickers, swipe directions usually refer to the intended movement of the control's movable part. For example, the Chinese \"往下滑一下页面\" and English \"scroll down the page\" usually mean revealing content below, corresponding to Scroll down; the Chinese \"向右滑动滑块\" and English \"slide the slider to the right\" usually mean moving the slider right, corresponding to Swipe right if using Swipe.\n4. If the intent remains ambiguous, use the Scroll and Swipe direction definitions above.\n\nAfter interpreting the intent, choose an appropriate Action from the current Action Space and use its supported parameters to achieve the intended result. Some actions support start and end points instead of \\`direction\\`. When using \\`direction\\`, ensure its value achieves the intended result according to the selected Action's definition.\n\n${includeLocateInPlanning ? locateGroundingRules() : ''}\n\n### ${planningProtocol.actionSpaceProtocol.title}\n\n${actionSpaceDescription}\n\n${renderLogContent(`### Log to give user feedback (preamble message)\n\nThe <log> tag is a brief preamble message to the user explaining what you're about to do. It should follow these principles and examples:\n\n- **Use ${preferredLanguage}**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what's been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button in popup</log>\n- <log>Previous actions failed to find the 'Yes' button, i will try again</log>\n- <log>Go back to find the login button</log>`)}\n\n### If there is some action to do ...\n\n${actionOutputProtocol.actionOutputRules}\n\nFor example:\n${buildActionOutputExample(\n  createSampleTapAction('Add to cart button for Sauce Labs Backpack'),\n  {\n    locatePromptSpec,\n    locateResultExampleIndex: 1,\n    buildActionOutput: actionOutputProtocol.buildActionOutput,\n  },\n)}\n\n### If you think there is an error ...\n\n- Use the <error> tag to output the error message.\n\nFor example:\n<error>Unable to find the required element on the page</error>\n\n### If there is no action to do ...\n\n- Don't output ${actionOutputTagsText} if there is no action to do.\n\n## Return Format\n\nReturn in XML format following this decision flow:\n\n${planningProtocol.responsePrefix ?? ''}\n\n${renderThoughtContent(`**Always include (REQUIRED):**\n<!-- Step 1: Observe and Plan -->\n<planning>Your planning details here. NEVER skip this tag.</planning>`)}\n${renderSubGoalsContent(`<!-- required when no update-plan-content is provided in the previous response -->\n<update-plan-content>...</update-plan-content>\n\n<!-- required when any sub-goal is completed -->\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\n<!-- Step 2: Memory data from current screenshot if needed -->\n<memory>...</memory>`)}\n\n**Then choose ONE of the following paths:**\n\n**Path A: If the ${renderSubGoalsContent('goal is accomplished', 'instruction is fulfilled')} or failed (Step ${completionCheckStepNumber})**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If the ${renderSubGoalsContent('goal is NOT complete', 'instruction is NOT fulfilled')} yet (Step ${actionStepNumber})**\n<!-- Determine next action -->\n${renderLogContent('<log>...</log>')}\n${actionOutputProtocol.actionOutputPlaceholder}\n\n<!-- OR if there's an error -->\n<error>...</error>\n\n${buildPlanningMultiTurnExample({\n  includeSubGoals: shouldIncludeSubGoals,\n  includeThought,\n  includeLog,\n  locatePromptSpec,\n  actionOutputProtocol,\n  prefix: planningProtocol.responsePrefix,\n})}`;\n}\n"],"names":["buildStandardPlanningSystemPrompt","input","actionSpace","includeLocateInPlanning","locatePromptSpec","includeSubGoals","includeThought","includeLog","planningProtocol","actionOutputProtocol","preferredLanguage","getPreferredLanguage","Error","planningModelFamilyRequiredForLocateMessage","actionSpaceDescription","buildPlanningActionSpaceDescription","hasRunAdbShell","action","actionOutputTagsText","tagName","shouldIncludeSubGoals","renderSubGoalsContent","content","fallbackContent","renderThoughtContent","renderLogContent","completionCheckStepNumber","actionStepNumber","locateGroundingRules","buildActionOutputExample","createSampleTapAction","buildPlanningMultiTurnExample"],"mappings":";;;;;;AA8BO,eAAeA,kCACpBC,KAA6C;IAE7C,MAAM,EACJC,WAAW,EACXC,uBAAuB,EACvBC,gBAAgB,EAChBC,eAAe,EACfC,iBAAiB,IAAI,EACrBC,aAAa,IAAI,EACjBC,gBAAgB,EACjB,GAAGP;IACJ,MAAMQ,uBAAuBD,iBAAiB,oBAAoB;IAClE,MAAME,oBAAoBC;IAE1B,IAAIR,2BAA2B,CAACC,kBAC9B,MAAM,IAAIQ,MAAMC;IAGlB,MAAMC,yBAAyBC,oCAAoC;QACjEb;QACAE;QACAI;IACF;IACA,MAAMQ,iBAAiBd,YAAY,IAAI,CACrC,CAACe,SAAWA,AAAgB,kBAAhBA,OAAO,IAAI;IAEzB,MAAMC,uBAAuBT,qBAAqB,oBAAoB,CACnE,GAAG,CAAC,CAACU,UAAY,CAAC,CAAC,EAAEA,QAAQ,CAAC,CAAC,EAC/B,IAAI,CAAC;IAER,MAAMC,wBAAwBf,mBAAmB;IACjD,MAAMgB,wBAAwB,CAACC,SAAiBC,kBAAkB,EAAE,GAClEH,wBAAwBE,UAAUC;IACpC,MAAMC,uBAAuB,CAACF,SAAiBC,kBAAkB,EAAE,GACjEjB,iBAAiBgB,UAAUC;IAC7B,MAAME,mBAAmB,CAACH,SAAiBC,kBAAkB,EAAE,GAC7DhB,aAAae,UAAUC;IAIzB,MAAMG,4BAA4BN,wBAAwB,IAAI;IAC9D,MAAMO,mBAAmBD,4BAA4B;IAErD,OAAO,CAAC;;;WAGC,EAAEL,sBACT,4FACAG,qBAAqB,sCAAsC,YAC3D;;uDAEmD,EAAEH,sBACrD,gKACA,qCACA;;AAEJ,EAAEA,sBAAsB,CAAC;;;gNAGuL,CAAC,EAAE;;AAEnN,EAAEG,qBAAqB,CAAC;;;;2CAImB,EAAEd,kBAAkB;;uEAEQ,EAAEW,sBACvE,2JACA,sFACA;;4bAE0b,CAAC,EAAE;;AAE/b,EAAEA,sBAAsB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAqDJ,CAAC,EAAE;;AAExB,EAAEA,sBAAsB,CAAC;;;;;;;;;;;;;;2DAckC,CAAC,EAAE;;QAEtD,EAAEK,0BAA0B,EAAE,EAAEL,sBACpC,iCACA,yCACA;;yCAEqC,EAAEA,sBAAsB,oEAAoE;;;;;;;;;;QAU7H,EAAEA,sBAAsB,qBAAqB,yBAAyB;MACxE,EAAEA,sBAAsB,wBAAwB,4BAA4B;;;;wBAI1D,EAAEA,sBAAsB,qBAAqB,yBAAyB;6BACjE,EAAEA,sBAAsB,qBAAqB,yBAAyB;qCAC9D,EAAEA,sBAAsB,qBAAqB,yBAAyB;4BAC/E,EAAEA,sBAAsB,qBAAqB,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;0NAyBwH,EAAEA,sBAAsB,YAAY,MAAM;;;;;;;;;;;;;AAapQ,EAAEA,sBACA,IACA,CAAC;;;oMAGiM,CAAC,EACnM;;;;sEAIoE,EAAEM,iBAAiB;;yCAEhD,EAAEN,sBAAsB,8IAA8I,wZAAwZ,EAAEA,sBAAsB,gEAAgE,wDAAwD;;0CAE7sB,EAAEH,qBAAqB;;QAEzD,EAAES,iBAAiB,uCAAuC,EAAEF,iBAAiB,aAAaP,qBAAqB;;mGAEpB,EAAEG,sBAAsB,iBAAiB;;;;;;;;;;AAU5I,EACEL,iBACI,4GACA,GACL;;;;;;;;;;;;;;;;;;;;;;;AAuBD,EAAEb,0BAA0ByB,yBAAyB,GAAG;;IAEpD,EAAEpB,iBAAiB,mBAAmB,CAAC,KAAK,CAAC;;AAEjD,EAAEM,uBAAuB;;AAEzB,EAAEW,iBAAiB,CAAC;;;;QAIZ,EAAEf,kBAAkB;;;;;;;;;6CASiB,CAAC,EAAE;;;;AAIhD,EAAED,qBAAqB,iBAAiB,CAAC;;;AAGzC,EAAEoB,yBACAC,sBAAsB,+CACtB;QACE1B;QACA,0BAA0B;QAC1B,mBAAmBK,qBAAqB,iBAAiB;IAC3D,GACA;;;;;;;;;;;eAWa,EAAES,qBAAqB;;;;;;AAMtC,EAAEV,iBAAiB,cAAc,IAAI,GAAG;;AAExC,EAAEgB,qBAAqB,CAAC;;qEAE6C,CAAC,EAAE;AACxE,EAAEH,sBAAsB,CAAC;;;;;;;;;oBASL,CAAC,EAAE;;;;iBAIN,EAAEA,sBAAsB,wBAAwB,4BAA4B,iBAAiB,EAAEK,0BAA0B;;;iBAGzH,EAAEL,sBAAsB,wBAAwB,gCAAgC,WAAW,EAAEM,iBAAiB;;AAE/H,EAAEF,iBAAiB,kBAAkB;AACrC,EAAEhB,qBAAqB,uBAAuB,CAAC;;;;;AAK/C,EAAEsB,8BAA8B;QAC9B,iBAAiBX;QACjBd;QACAC;QACAH;QACAK;QACA,QAAQD,iBAAiB,cAAc;IACzC,IAAI;AACJ"}
+{"version":3,"file":"ai-model/prompt/planning/system-prompt.mjs","sources":["../../../../../src/ai-model/prompt/planning/system-prompt.ts"],"sourcesContent":["import type { DeviceAction } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport type { StandardPlanningProtocol } from '../../model-adapter/planning-protocol';\nimport type { LocateResultPromptSpec } from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport { locateGroundingRules } from '../locate';\nimport {\n  buildActionOutputExample,\n  createSampleTapAction,\n} from './action-output-example';\nimport { buildPlanningActionSpaceDescription } from './action-space-description';\nimport { buildPlanningMultiTurnExample } from './multi-turn-example';\n\ntype BuildStandardPlanningSystemPromptInput = {\n  actionSpace: DeviceAction<any>[];\n  includeSubGoals?: boolean;\n  includeThought?: boolean;\n  includeLog?: boolean;\n  planningProtocol: StandardPlanningProtocol;\n} & (\n  | {\n      includeLocateInPlanning: true;\n      locatePromptSpec: LocateResultPromptSpec;\n    }\n  | {\n      includeLocateInPlanning: false;\n      locatePromptSpec?: never;\n    }\n);\n\nexport async function buildStandardPlanningSystemPrompt(\n  input: BuildStandardPlanningSystemPromptInput,\n) {\n  const {\n    actionSpace,\n    includeLocateInPlanning,\n    locatePromptSpec,\n    includeSubGoals,\n    includeThought = true,\n    includeLog = true,\n    planningProtocol,\n  } = input;\n  const actionOutputProtocol = planningProtocol.actionOutputProtocol;\n  const preferredLanguage = getPreferredLanguage();\n\n  if (includeLocateInPlanning && !locatePromptSpec) {\n    throw new Error(planningModelFamilyRequiredForLocateMessage());\n  }\n\n  const actionSpaceDescription = buildPlanningActionSpaceDescription({\n    actionSpace,\n    locatePromptSpec,\n    planningProtocol,\n  });\n  const hasRunAdbShell = actionSpace.some(\n    (action) => action.name === 'RunAdbShell',\n  );\n  const actionOutputTagsText = actionOutputProtocol.actionOutputTagNames\n    .map((tagName) => `<${tagName}>`)\n    .join(', ');\n\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n  const renderSubGoalsContent = (content: string, fallbackContent = '') =>\n    shouldIncludeSubGoals ? content : fallbackContent;\n  const renderThoughtContent = (content: string, fallbackContent = '') =>\n    includeThought ? content : fallbackContent;\n  const renderLogContent = (content: string, fallbackContent = '') =>\n    includeLog ? content : fallbackContent;\n\n  // Step numbering adjusts based on whether sub-goals are included\n  // When includeSubGoals=false, memory step is skipped\n  const completionCheckStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = completionCheckStepNumber + 1;\n\n  return `\nTarget: You are an expert to manipulate the UI to accomplish the user's instruction. User will give you an instruction, some screenshots, background knowledge and previous logs indicating what have been done. Your task is to accomplish the instruction by thinking through the path to complete the task and give the next action to execute.\n\n## Step 1: ${renderSubGoalsContent(\n    'Observe and Plan (related tags: <planning>, <update-plan-content>, <mark-sub-goal-done>)',\n    renderThoughtContent('Observe (related tags: <planning>)', 'Observe'),\n  )}\n\nFirst, observe the current screenshot and previous logs${renderSubGoalsContent(\n    \", then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot.\",\n    ' to understand the current state.',\n  )}\n\n${renderSubGoalsContent(`### Observation Guidelines\n\n- Treat visible summaries, thumbnails, cropped content, and partially visible lists as potentially incomplete when the task depends on precise details.\n- If the current view does not provide enough information to decide safely, use available UI affordances such as opening details, expanding content, previewing, enlarging, zooming, or scrolling before acting.`)}\n\n${renderThoughtContent(`### <planning> tag (REQUIRED)\n\nREQUIRED: You MUST always output the <planning> tag. Never skip it.\n\nWrite the content of the <planning> tag in ${preferredLanguage}.\n\nInclude your planning details in the <planning> tag. It should answer: ${renderSubGoalsContent(\n  \"What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action?\",\n  'What is the current state based on the screenshot? What should be the next action?',\n)} Write it naturally without numbering or section headers.\n\nCRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: \"fill out the form\" means only fill fields, do NOT submit; \"click the button\" means only click, do NOT wait for page load or verify results; \"type 'hello'\" means only type, do NOT press Enter.`)}\n\n${renderSubGoalsContent(`### <update-plan-content> tag\n\nUse this structure to give or update your plan:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">sub goal description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">sub goal description</sub-goal>\n  ...\n</update-plan-content>\n\n### <mark-sub-goal-done> tag\n\nUse this structure to mark a sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: You MUST only mark a sub-goal as \"finished\" AFTER you have confirmed the task is actually completed by observing the result in the screenshot. Do NOT mark a sub-goal as done just because you expect the next action will complete it. Wait until you see visual confirmation in the screenshot that the sub-goal has been achieved.\n\n### Note\n\nDuring execution, you can call <update-plan-content> at any time to update the plan based on the latest screenshot and completed sub-goals.\n\n### Example\n\nIf the user wants to \"log in to a system using username and password, complete all to-do items, and submit a registration form\", you can break it down into the following sub-goals:\n\n<planning>...</planning>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Log in to the system</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Complete all to-do items</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Submit the registration form</sub-goal>\n</update-plan-content>\n\nAfter logging in and seeing the to-do items, you can mark the sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nAt this point, the status of all sub-goals is:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished\" />\n  <sub-goal index=\"2\" status=\"pending\" />\n  <sub-goal index=\"3\" status=\"pending\" />\n</update-plan-content>\n\nAfter some time, when the last sub-goal is also completed, you can mark it as done as well:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>`)}\n\n${renderSubGoalsContent(`## Step 2: Memory Data from Current Screenshot (related tags: <memory>)\n\nUse <memory> to record clear, task-relevant information from the current screenshot that may be needed in later steps. The current screenshot will not be available later, so memory should preserve enough detail for future reasoning, verification, or action.\n\n- Record information completely and exactly as shown. Do not summarize, translate, normalize, or merge values that may matter later.\n- When recording an item, include the item itself, its exact task-relevant details, and the visible cue or UI context that identifies where it came from when relevant.\n- Keep similar or repeated items as separate memory entries unless their task-relevant details are confirmed to be the same.\n- After navigation, scrolling, editing, deletion, saving, or other screen changes, treat remembered positions, order, indexes, and UI bindings as references only. Re-check the current screen before acting on them.\n\nExamples:\n- If you need to find an item and later assert its details, record the item name and the exact details needed for the assertion, such as status, price, date, owner, description, or other visible fields.\n- If you need to compare multiple similar results, record each candidate separately with its exact distinguishing details and visible context.\n- If you need to copy information from one place to another, record the exact source value and the target field or UI cue it should be mapped to.\n\nDon't use this tag if no information needs to be preserved.`)}\n\n## Step ${completionCheckStepNumber}: ${renderSubGoalsContent(\n    'Check if Goal is Accomplished',\n    'Check if the Instruction is Fulfilled',\n  )} (related tags: <complete>)\n\nDetermine if the entire task is completed${renderSubGoalsContent(' based on the current screenshot and the status of all sub-goals')}.\n\n### CRITICAL: The User's Instruction is the Supreme Authority\n\nThe user's instruction defines the EXACT scope of what you must accomplish. You MUST follow it precisely - nothing more, nothing less. Violating this rule may cause severe consequences such as data loss, unintended operations, or system failures.\n\n**Explicit instructions vs. High-level goals:**\n- If the user gives you **explicit operation steps** (e.g., \"click X\", \"type Y\", \"fill out the form\"), treat them as exact commands. Execute ONLY those steps, nothing more.\n- If the user gives you a **high-level goal** (e.g., \"log in to the system\", \"complete the purchase\"), you may determine the necessary steps to achieve it.\n\n**What \"${renderSubGoalsContent('goal accomplished', 'instruction fulfilled')}\" means:**\n- The ${renderSubGoalsContent('goal is accomplished', 'instruction is fulfilled')} when you have done EXACTLY what the user asked - no extra steps, no assumptions.\n- Do NOT perform any action beyond the explicit instruction, even if it seems logical or helpful.\n\n**Examples - Explicit instructions (execute exactly, no extra steps):**\n- \"fill out the form\" → ${renderSubGoalsContent('Goal accomplished', 'Instruction fulfilled')} when all fields are filled. Do NOT submit the form.\n- \"click the login button\" → ${renderSubGoalsContent('Goal accomplished', 'Instruction fulfilled')} once clicked. Do NOT wait for page load or verify login success.\n- \"type 'hello' in the search box\" → ${renderSubGoalsContent('Goal accomplished', 'Instruction fulfilled')} when 'hello' is typed. Do NOT press Enter or trigger search.\n- \"select the first item\" → ${renderSubGoalsContent('Goal accomplished', 'Instruction fulfilled')} when selected. Do NOT proceed to checkout.\n\n**Change completion:**\n- If the requested outcome is a durable change, such as create, edit, update, delete, save, send, submit, apply, or publish, do not stop at an unsaved draft, open editor, temporary input, transient selection, or staged value. Continue through the app/page's normal completion control such as Save, Done, Confirm, OK, Submit, Apply, Send, or Publish before completing, so the result remains after leaving the screen.\n- If the user only asks for an intermediate UI state, such as typing text, selecting an option, filling fields, or opening a screen without saving/submitting/applying, stop once that exact state is reached.\n\n**Special case - Scrollable option lists and dropdowns:**\n- When choosing an item from a scrollable select, dropdown, listbox, menu, or similar option list, first open the control if it is closed. Once the list is open, interact with the list itself, not the page.\n- If the target option is visible in the open list, Tap that exact option immediately.\n- If the list is open but the target option is not visible, try to find it by scrolling the open list/dropdown before giving up or interacting with other elements.\n- For an open dropdown/list, prefer small incremental Scroll actions with an explicit distance (typically 50-120 pixels) and a locate target describing the open list/dropdown. Do NOT omit distance while searching within a list, because the default scroll distance can skip over relevant options and cause oscillation.\n- While searching within an open dropdown/list, use short scrolls so intermediate options are not skipped.\n- After selecting the target, if the trigger text or result text shows the requested option, treat the current selection step as fulfilled and continue evaluating the remaining user instruction.\n\n**Special case - Text hidden by a narrow input field:**\n- CRITICAL PRIORITY OVERRIDE - Input verification after an input action:\n- This rule overrides the general requirement to verify the exact target text from the screenshot.\n- If the previous step already executed an input action, and the current input field is not empty, you MUST directly treat that input as successful.\n- In this situation, do NOT use the visible text in the screenshot to decide that the input is incomplete, incorrect, missing characters, out of order, or needs correction.\n- Do NOT perform ClearInput, another Input, KeyboardPress, or any other action intended to \"fix the input content\" just because the visible text looks different from the target text, has abnormal character order, missing characters, abnormal spacing, suspicious visual recognition results, or appears to have selected/highlighted text.\n- The general rule \"do EXACTLY what the user asked\" still applies to the intended input value you execute, but it MUST NOT be enforced by re-validating the visible text in the screenshot after the input action.\n- Differences in visible text must be assumed to be caused by clipping, horizontal scrolling, narrow input fields, text selection, caret position, or visual recognition errors rather than input failure.\n- Retry input only when the input field is clearly still empty, or when the page shows an explicit error message.\n\n**Special case - Assertion instructions:**\n- If the user's instruction includes an assertion (e.g., \"verify that...\", \"check that...\", \"assert...\"), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark ${renderSubGoalsContent('the goal', 'it')} as failed (success=\"false\").\n- If the page is still loading (e.g., you see a loading spinner, skeleton screen, or progress bar), do NOT assert yet. Wait for the page to finish loading before evaluating the assertion.\n\n### Completion Criteria for Process-required Instructions\n\nIf the user's instruction includes explicit operation steps, ordering requirements, or action requirements, it is a process-required instruction.\n\nFor process-required instructions, do NOT treat the task as complete only because the current screenshot already shows the final expected state. Do NOT infer that earlier steps were executed from the final UI state.\n\nYou may output <complete success=\"true\"> only when the current execution history, previous logs, or the screenshot after the most recent action proves that every explicit step required by the user has been completed, and the final check condition is also satisfied.\n\nIf any explicit step lacks completion evidence in the current execution history, continue with the next missing step instead of outputting <complete>, even if the current screenshot appears to satisfy the final condition.\n\n${renderSubGoalsContent(\n  '',\n  `**Page navigation restriction:**\n- Unless the user's instruction explicitly asks you to click a link, jump to another page, or navigate to a URL, you MUST complete the task on the current page only.\n- Do NOT navigate away from the current page on your own initiative (e.g., do not click links that lead to other pages, do not use browser back/forward, do not open new URLs).\n- If the task cannot be accomplished on the current page and the user has not instructed you to navigate, report it as a failure (success=\"false\") instead of attempting to navigate to other pages.`,\n)}\n\n### Output Rules\n\n- If the task is NOT complete, skip this section and continue to Step ${actionStepNumber}.\n- Use the <complete success=\"true|false\">message</complete> tag to output the result if the goal is accomplished or failed.\n  - the 'success' attribute is required. ${renderSubGoalsContent('It means whether the expected goal is accomplished based on what you observe in the current screenshot and the current execution history. ')}No matter what errors occurred during execution, set success=\"true\" only when the current execution history shows that all steps required by the user have been completed and the final state satisfies the requirement. If the user asks for explicit operation steps or an ordered workflow, do not treat those steps as completed only because the current screenshot already shows the final expected state. If the ${renderSubGoalsContent('expected goal is not accomplished and cannot be accomplished', 'instruction is not fulfilled and cannot be fulfilled')}, set success=\"false\".\n  - the 'message' is the information that will be provided to the user. If the user asks for a specific format, strictly follow that.\n- If you output <complete>, do NOT output ${actionOutputTagsText}. The task ends here.\n\n## Step ${actionStepNumber}: Determine Next Action (related tags: ${renderLogContent('<log>, ')}${actionOutputTagsText}, <error>)\n\nONLY if the task is not complete: Think what the next action is according to the current screenshot${renderSubGoalsContent(' and the plan')}.\n\n- Don't give extra actions or plans beyond the instruction or the plan. For example, don't try to submit the form if the instruction is only to fill something.\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully. Otherwise, retry or do something else to recover.\n- Give just the next ONE action you should do (if any)\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 3 times, you should think this is an error and set the \"error\" field to the error message.\n\n### Action Guidelines\n\n${\n  hasRunAdbShell\n    ? \"- If the user's task can be completed with the RunAdbShell action, prefer using the RunAdbShell action.\"\n    : ''\n}\n- If Swipe is available in the current Action Space, for touch continuous controls that set a value along a track, such as a slider, prefer Swipe from the current handle or filled position to the requested track endpoint instead of tapping the endpoint.\n- When editing existing text in a UI field, preserve all existing text by moving the cursor and typing/deleting the minimal necessary characters.\n- For insert/prepend/append edits, use CursorMove when the caret must be adjusted precisely, then use Input with mode \"typeOnly\" for inserted characters and KeyboardPress for newlines or deletion. If the caret lands in the wrong position, recover with CursorMove, KeyboardPress, or undo and retry cursor placement; do not switch to replace as a fallback for cursor placement failures.\n\n### Interpreting Scroll and Swipe Directions\n\n- For a relative Swipe, express distance in pixels of the current screenshot. The runtime converts this model-planned distance to device logical units. This rule applies to Swipe distance, not Scroll distance.\n\nScroll and swipe directions can be ambiguous: they may refer to off-screen content to reveal, visible content movement, or the physical movement of a finger, pointer, or scroll wheel. Infer the intended result from the user's goal, the interaction target, the current UI, and common usage in the user's language, rather than mechanically copying the user's direction word.\n\nIn the current Action Space, \\`direction\\` has the following meanings:\n\n- Scroll follows the traditional mouse-wheel convention (with natural scrolling disabled). Its direction refers to the off-screen content to reveal; under this convention, it matches the wheel movement direction and is opposite to the content movement direction.\n- Swipe follows the finger-movement convention. Its direction refers to finger movement; for controls where content follows the finger, it matches the content movement direction and is opposite to the direction from which off-screen content is revealed.\n\nUse the following priority order:\n\n1. Prioritize the user's goal or expected result, even when it conflicts with a direction word. For example, for \"swipe the date picker down to increase the date\", if dates increase from top to bottom, use Swipe up to move a larger date from below into the selected position.\n2. Otherwise, if the user specifies a moving object or movement path, follow that object's physical movement. For example, \"swipe the finger to the left\" means Swipe left; \"move the page content from top to bottom\" means moving content downward, corresponding to Swipe down on a page where content follows the finger.\n3. Otherwise, consider the interaction target and common usage in the user's language. For content browsing, such as pages and lists, scroll directions usually refer to the off-screen content the user wants to see. For directly manipulated controls, such as sliders and wheel pickers, swipe directions usually refer to the intended movement of the control's movable part. For example, the Chinese \"往下滑一下页面\" and English \"scroll down the page\" usually mean revealing content below, corresponding to Scroll down; the Chinese \"向右滑动滑块\" and English \"slide the slider to the right\" usually mean moving the slider right, corresponding to Swipe right if using Swipe.\n4. If the intent remains ambiguous, use the Scroll and Swipe direction definitions above.\n\nAfter interpreting the intent, choose an appropriate Action from the current Action Space and use its supported parameters to achieve the intended result. Some actions support start and end points instead of \\`direction\\`. When using \\`direction\\`, ensure its value achieves the intended result according to the selected Action's definition.\n\n${includeLocateInPlanning ? locateGroundingRules() : ''}\n\n### ${planningProtocol.actionSpaceProtocol.title}\n\n${actionSpaceDescription}\n\n${renderLogContent(`### Log to give user feedback (preamble message)\n\nThe <log> tag is a brief preamble message to the user explaining what you're about to do. It should follow these principles and examples:\n\n- **Use ${preferredLanguage}**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what's been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button in popup</log>\n- <log>Previous actions failed to find the 'Yes' button, i will try again</log>\n- <log>Go back to find the login button</log>`)}\n\n### If there is some action to do ...\n\n${actionOutputProtocol.actionOutputRules}\n\nFor example:\n${buildActionOutputExample(\n  createSampleTapAction('Add to cart button for Sauce Labs Backpack'),\n  {\n    locatePromptSpec,\n    locateResultExampleIndex: 1,\n    buildActionOutput: actionOutputProtocol.buildActionOutput,\n  },\n)}\n\n### If you think there is an error ...\n\n- Use the <error> tag to output the error message.\n\nFor example:\n<error>Unable to find the required element on the page</error>\n\n### If there is no action to do ...\n\n- Don't output ${actionOutputTagsText} if there is no action to do.\n\n## Return Format\n\nReturn in XML format following this decision flow:\n\n${planningProtocol.responsePrefix ?? ''}\n\n${renderThoughtContent(`**Always include (REQUIRED):**\n<!-- Step 1: Observe and Plan -->\n<planning>Your planning details here. NEVER skip this tag.</planning>`)}\n${renderSubGoalsContent(`<!-- required when no update-plan-content is provided in the previous response -->\n<update-plan-content>...</update-plan-content>\n\n<!-- required when any sub-goal is completed -->\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\n<!-- Step 2: Memory data from current screenshot if needed -->\n<memory>...</memory>`)}\n\n**Then choose ONE of the following paths:**\n\n**Path A: If the ${renderSubGoalsContent('goal is accomplished', 'instruction is fulfilled')} or failed (Step ${completionCheckStepNumber})**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If the ${renderSubGoalsContent('goal is NOT complete', 'instruction is NOT fulfilled')} yet (Step ${actionStepNumber})**\n<!-- Determine next action -->\n${renderLogContent('<log>...</log>')}\n${actionOutputProtocol.actionOutputPlaceholder}\n\n<!-- OR if there's an error -->\n<error>...</error>\n\n${buildPlanningMultiTurnExample({\n  includeSubGoals: shouldIncludeSubGoals,\n  includeThought,\n  includeLog,\n  locatePromptSpec,\n  actionOutputProtocol,\n  prefix: planningProtocol.responsePrefix,\n})}`;\n}\n"],"names":["buildStandardPlanningSystemPrompt","input","actionSpace","includeLocateInPlanning","locatePromptSpec","includeSubGoals","includeThought","includeLog","planningProtocol","actionOutputProtocol","preferredLanguage","getPreferredLanguage","Error","planningModelFamilyRequiredForLocateMessage","actionSpaceDescription","buildPlanningActionSpaceDescription","hasRunAdbShell","action","actionOutputTagsText","tagName","shouldIncludeSubGoals","renderSubGoalsContent","content","fallbackContent","renderThoughtContent","renderLogContent","completionCheckStepNumber","actionStepNumber","locateGroundingRules","buildActionOutputExample","createSampleTapAction","buildPlanningMultiTurnExample"],"mappings":";;;;;;AA8BO,eAAeA,kCACpBC,KAA6C;IAE7C,MAAM,EACJC,WAAW,EACXC,uBAAuB,EACvBC,gBAAgB,EAChBC,eAAe,EACfC,iBAAiB,IAAI,EACrBC,aAAa,IAAI,EACjBC,gBAAgB,EACjB,GAAGP;IACJ,MAAMQ,uBAAuBD,iBAAiB,oBAAoB;IAClE,MAAME,oBAAoBC;IAE1B,IAAIR,2BAA2B,CAACC,kBAC9B,MAAM,IAAIQ,MAAMC;IAGlB,MAAMC,yBAAyBC,oCAAoC;QACjEb;QACAE;QACAI;IACF;IACA,MAAMQ,iBAAiBd,YAAY,IAAI,CACrC,CAACe,SAAWA,AAAgB,kBAAhBA,OAAO,IAAI;IAEzB,MAAMC,uBAAuBT,qBAAqB,oBAAoB,CACnE,GAAG,CAAC,CAACU,UAAY,CAAC,CAAC,EAAEA,QAAQ,CAAC,CAAC,EAC/B,IAAI,CAAC;IAER,MAAMC,wBAAwBf,mBAAmB;IACjD,MAAMgB,wBAAwB,CAACC,SAAiBC,kBAAkB,EAAE,GAClEH,wBAAwBE,UAAUC;IACpC,MAAMC,uBAAuB,CAACF,SAAiBC,kBAAkB,EAAE,GACjEjB,iBAAiBgB,UAAUC;IAC7B,MAAME,mBAAmB,CAACH,SAAiBC,kBAAkB,EAAE,GAC7DhB,aAAae,UAAUC;IAIzB,MAAMG,4BAA4BN,wBAAwB,IAAI;IAC9D,MAAMO,mBAAmBD,4BAA4B;IAErD,OAAO,CAAC;;;WAGC,EAAEL,sBACT,4FACAG,qBAAqB,sCAAsC,YAC3D;;uDAEmD,EAAEH,sBACrD,gKACA,qCACA;;AAEJ,EAAEA,sBAAsB,CAAC;;;gNAGuL,CAAC,EAAE;;AAEnN,EAAEG,qBAAqB,CAAC;;;;2CAImB,EAAEd,kBAAkB;;uEAEQ,EAAEW,sBACvE,2JACA,sFACA;;4bAE0b,CAAC,EAAE;;AAE/b,EAAEA,sBAAsB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAqDJ,CAAC,EAAE;;AAExB,EAAEA,sBAAsB,CAAC;;;;;;;;;;;;;;2DAckC,CAAC,EAAE;;QAEtD,EAAEK,0BAA0B,EAAE,EAAEL,sBACpC,iCACA,yCACA;;yCAEqC,EAAEA,sBAAsB,oEAAoE;;;;;;;;;;QAU7H,EAAEA,sBAAsB,qBAAqB,yBAAyB;MACxE,EAAEA,sBAAsB,wBAAwB,4BAA4B;;;;wBAI1D,EAAEA,sBAAsB,qBAAqB,yBAAyB;6BACjE,EAAEA,sBAAsB,qBAAqB,yBAAyB;qCAC9D,EAAEA,sBAAsB,qBAAqB,yBAAyB;4BAC/E,EAAEA,sBAAsB,qBAAqB,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;0NAyBwH,EAAEA,sBAAsB,YAAY,MAAM;;;;;;;;;;;;;AAapQ,EAAEA,sBACA,IACA,CAAC;;;oMAGiM,CAAC,EACnM;;;;sEAIoE,EAAEM,iBAAiB;;yCAEhD,EAAEN,sBAAsB,8IAA8I,wZAAwZ,EAAEA,sBAAsB,gEAAgE,wDAAwD;;0CAE7sB,EAAEH,qBAAqB;;QAEzD,EAAES,iBAAiB,uCAAuC,EAAEF,iBAAiB,aAAaP,qBAAqB;;mGAEpB,EAAEG,sBAAsB,iBAAiB;;;;;;;;;;AAU5I,EACEL,iBACI,4GACA,GACL;;;;;;;;;;;;;;;;;;;;;;;;;AAyBD,EAAEb,0BAA0ByB,yBAAyB,GAAG;;IAEpD,EAAEpB,iBAAiB,mBAAmB,CAAC,KAAK,CAAC;;AAEjD,EAAEM,uBAAuB;;AAEzB,EAAEW,iBAAiB,CAAC;;;;QAIZ,EAAEf,kBAAkB;;;;;;;;;6CASiB,CAAC,EAAE;;;;AAIhD,EAAED,qBAAqB,iBAAiB,CAAC;;;AAGzC,EAAEoB,yBACAC,sBAAsB,+CACtB;QACE1B;QACA,0BAA0B;QAC1B,mBAAmBK,qBAAqB,iBAAiB;IAC3D,GACA;;;;;;;;;;;eAWa,EAAES,qBAAqB;;;;;;AAMtC,EAAEV,iBAAiB,cAAc,IAAI,GAAG;;AAExC,EAAEgB,qBAAqB,CAAC;;qEAE6C,CAAC,EAAE;AACxE,EAAEH,sBAAsB,CAAC;;;;;;;;;oBASL,CAAC,EAAE;;;;iBAIN,EAAEA,sBAAsB,wBAAwB,4BAA4B,iBAAiB,EAAEK,0BAA0B;;;iBAGzH,EAAEL,sBAAsB,wBAAwB,gCAAgC,WAAW,EAAEM,iBAAiB;;AAE/H,EAAEF,iBAAiB,kBAAkB;AACrC,EAAEhB,qBAAqB,uBAAuB,CAAC;;;;;AAK/C,EAAEsB,8BAA8B;QAC9B,iBAAiBX;QACjBd;QACAC;QACAH;QACAK;QACA,QAAQD,iBAAiB,cAAc;IACzC,IAAI;AACJ"}

package/dist/es/ai-model/workflows/planning/custom-planning.mjs

@@ -3,6 +3,7 @@ import { prepareModelImage } from "../../model-adapter/image-preprocess.mjs";
 import { AIResponseParseError, callAIWithStringResponse } from "../../service-caller/index.mjs";
 import { preparedReferenceImagesToMessages } from "../../shared/multimodal-prompt.mjs";
 import { normalizePlanningActionLocateFields } from "./locate-normalization.mjs";
+import { normalizePlanningSwipeDistances } from "./swipe-distance.mjs";
 function appendActionContext(systemPrompt, actionContext) {
     return systemPrompt + (actionContext ? `${actionContext}\n` : '');
 }
@@ -101,6 +102,7 @@ async function runCustomPlanning(userInstruction, options, config) {
                 contentSize: preparedImage.contentSize
             }
         });
+        normalizePlanningSwipeDistances(actions, context.shrunkShotToLogicalRatio);
         shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);
     } catch (parseError) {
         const errorMessage = `Parse error: ${parseError instanceof Error ? parseError.message : String(parseError)}`;

package/dist/es/ai-model/workflows/planning/custom-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport type { PlanOptions } from './types';\n\nfunction appendActionContext(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return systemPrompt + (actionContext ? `${actionContext}\\n` : '');\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ModelCallMessages {\n  const { options } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendActionContext(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstructionText = input.userInstruction.text;\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    input.userInstruction.referenceImages,\n  );\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.appendMessage({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.appendMessage({\n    role: 'user',\n    content: [\n      {\n        type: 'image',\n        url: context.screenshot.base64,\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...referenceImageMessages,\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...referenceImageMessages,\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: PreparedUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawAssistantOutput } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultCodec: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    });\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawAssistantOutput,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  // UI-TARS and Auto-GLM define their own assistant history format.\n  if (\n    rawAssistantOutput?.type === 'responses' &&\n    options.modelRuntime.adapter.responses.replayRawAssistantOutput\n  ) {\n    options.conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else if (assistantContent) {\n    options.conversationHistory.appendMessage({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawAssistantOutput,\n  };\n}\n"],"names":["appendActionContext","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawAssistantOutput","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;AAoBA,SAASA,oBACPC,YAAoB,EACpBC,aAAsB;IAEtB,OAAOD,eAAgBC,CAAAA,gBAAgB,GAAGA,cAAc,EAAE,CAAC,GAAG,EAAC;AACjE;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,oBACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,kCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,aAAa,CAAC;QAChC,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,KAAKC,QAAQ,UAAU,CAAC,MAAM;YAChC;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,kBAAkB,EAAE,GAAG,MAAMC,yBACnDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;QACF;QACAW,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOG,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACb,SAASc,QAAW,IACnCb,OACAC;IAEJ;IAEA,MAAMa,mBAAmB7B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IACEiB,oBAAoB,SAAS,eAC7Bf,QAAQ,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,wBAAwB,EAE/DA,QAAQ,mBAAmB,CAAC,iBAAiB,CAACe;SACzC,IAAIa,kBACT5B,QAAQ,mBAAmB,CAAC,aAAa,CAAC;QACxC,MAAM;QACN,SAAS4B;IACX;IAGF,OAAO;QACLV;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaO,KAAK,SAAS,CAACb,SAASc,QAAW;QAChDZ;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/planning/custom-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/custom-planning.ts"],"sourcesContent":["import type { PlanningAIResponse, PlanningAction } from '@/types';\nimport { ScreenshotItem } from '../../../screenshot-item';\nimport type {\n  CustomPlanningInput,\n  CustomPlanningMessageConfig,\n  ResolvedCustomPlanningDefinition,\n} from '../../model-adapter/custom-planning-types';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport {\n  AIResponseParseError,\n  callAIWithStringResponse,\n} from '../../service-caller/index';\nimport type { ModelCallMessages } from '../../service-caller/types';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport { normalizePlanningSwipeDistances } from './swipe-distance';\nimport type { PlanOptions } from './types';\n\nfunction appendActionContext(\n  systemPrompt: string,\n  actionContext?: string,\n): string {\n  return systemPrompt + (actionContext ? `${actionContext}\\n` : '');\n}\n\nexport function buildCustomPlanningMessages<TParsed>(\n  input: CustomPlanningInput,\n  config: CustomPlanningMessageConfig<TParsed>,\n): ModelCallMessages {\n  const { options } = input;\n  const { conversationHistory, context, actionContext } = options;\n  const systemPrompt = appendActionContext(\n    config.buildSystemPrompt(),\n    actionContext,\n  );\n  const userInstructionText = input.userInstruction.text;\n  const userInstruction = config.buildUserInstruction\n    ? config.buildUserInstruction(userInstructionText)\n    : userInstructionText;\n  const referenceImageMessages = preparedReferenceImagesToMessages(\n    input.userInstruction.referenceImages,\n  );\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    conversationHistory.appendMessage({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.`,\n        },\n      ],\n    });\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  }\n\n  conversationHistory.appendMessage({\n    role: 'user',\n    content: [\n      {\n        type: 'image',\n        url: context.screenshot.base64,\n      },\n    ],\n  });\n\n  if (config.systemPromptPlacement === 'system-message') {\n    return [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [{ type: 'text', text: userInstruction }],\n      },\n      ...referenceImageMessages,\n      ...conversationHistory.snapshot(config.historyImageLimit),\n    ];\n  }\n\n  return [\n    {\n      role: 'user',\n      content: `${systemPrompt}${userInstruction}`,\n    },\n    ...referenceImageMessages,\n    ...conversationHistory.snapshot(config.historyImageLimit),\n  ];\n}\n\nexport async function runCustomPlanning<TParsed>(\n  userInstruction: PreparedUserPrompt,\n  options: PlanOptions,\n  config: ResolvedCustomPlanningDefinition<TParsed>,\n): Promise<PlanningAIResponse> {\n  const { context } = options;\n  const preparedImage = await prepareModelImage({\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: options.modelRuntime.adapter.imagePreprocess,\n  });\n  const preparedOptions: PlanOptions = {\n    ...options,\n    context: {\n      ...context,\n      screenshot: ScreenshotItem.create(\n        preparedImage.imageBase64,\n        context.screenshot.capturedAt,\n      ),\n      shotSize: preparedImage.preparedSize,\n    },\n  };\n  const input: CustomPlanningInput = {\n    userInstruction,\n    options: preparedOptions,\n    coordinateSystem: config.coordinateSystem,\n  };\n\n  const messages = buildCustomPlanningMessages(input, config.messages);\n  const { content, usage, rawAssistantOutput } = await callAIWithStringResponse(\n    messages,\n    preparedOptions.modelRuntime,\n    {\n      abortSignal: preparedOptions.abortSignal,\n      requiresOriginalImageDetail: preparedOptions.includeLocateInPlanning,\n    },\n  );\n\n  let parsed: TParsed;\n  let actions: PlanningAction[];\n  let shouldContinuePlanning: boolean;\n\n  try {\n    parsed = config.parseResponse(content, input);\n    actions = config.transformActions(parsed, input);\n    normalizePlanningActionLocateFields(actions, {\n      actionSpace: preparedOptions.actionSpace,\n      includeLocateInPlanning: preparedOptions.includeLocateInPlanning,\n      locateResultCodec: config.coordinateNormalizer,\n      locateResultContext: {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    });\n    normalizePlanningSwipeDistances(actions, context.shrunkShotToLogicalRatio);\n    shouldContinuePlanning = config.shouldContinuePlanning(parsed, actions);\n  } catch (parseError) {\n    const errorMessage = `Parse error: ${\n      parseError instanceof Error ? parseError.message : String(parseError)\n    }`;\n    throw new AIResponseParseError(\n      errorMessage,\n      JSON.stringify(content, undefined, 2),\n      usage,\n      rawAssistantOutput,\n    );\n  }\n\n  const assistantContent = config.messages.buildAssistantContent?.(\n    parsed,\n    content,\n    input,\n  );\n  // UI-TARS and Auto-GLM define their own assistant history format.\n  if (\n    rawAssistantOutput?.type === 'responses' &&\n    options.modelRuntime.adapter.responses.replayRawAssistantOutput\n  ) {\n    options.conversationHistory.appendModelOutput(rawAssistantOutput);\n  } else if (assistantContent) {\n    options.conversationHistory.appendMessage({\n      role: 'assistant',\n      content: assistantContent,\n    });\n  }\n\n  return {\n    actions,\n    log: config.buildResponseLog(parsed, content),\n    usage,\n    shouldContinuePlanning,\n    rawResponse: JSON.stringify(content, undefined, 2),\n    rawAssistantOutput,\n  };\n}\n"],"names":["appendActionContext","systemPrompt","actionContext","buildCustomPlanningMessages","input","config","options","conversationHistory","context","userInstructionText","userInstruction","referenceImageMessages","preparedReferenceImagesToMessages","runCustomPlanning","preparedImage","prepareModelImage","preparedOptions","ScreenshotItem","messages","content","usage","rawAssistantOutput","callAIWithStringResponse","parsed","actions","shouldContinuePlanning","normalizePlanningActionLocateFields","normalizePlanningSwipeDistances","parseError","errorMessage","Error","String","AIResponseParseError","JSON","undefined","assistantContent"],"mappings":";;;;;;AAqBA,SAASA,oBACPC,YAAoB,EACpBC,aAAsB;IAEtB,OAAOD,eAAgBC,CAAAA,gBAAgB,GAAGA,cAAc,EAAE,CAAC,GAAG,EAAC;AACjE;AAEO,SAASC,4BACdC,KAA0B,EAC1BC,MAA4C;IAE5C,MAAM,EAAEC,OAAO,EAAE,GAAGF;IACpB,MAAM,EAAEG,mBAAmB,EAAEC,OAAO,EAAEN,aAAa,EAAE,GAAGI;IACxD,MAAML,eAAeD,oBACnBK,OAAO,iBAAiB,IACxBH;IAEF,MAAMO,sBAAsBL,MAAM,eAAe,CAAC,IAAI;IACtD,MAAMM,kBAAkBL,OAAO,oBAAoB,GAC/CA,OAAO,oBAAoB,CAACI,uBAC5BA;IACJ,MAAME,yBAAyBC,kCAC7BR,MAAM,eAAe,CAAC,eAAe;IAGvC,IAAIG,oBAAoB,sBAAsB,EAAE;QAC9CA,oBAAoB,aAAa,CAAC;YAChC,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGA,oBAAoB,sBAAsB,CAAC,qHAAqH,CAAC;gBAC5K;aACD;QACH;QACAA,oBAAoB,mCAAmC;IACzD;IAEAA,oBAAoB,aAAa,CAAC;QAChC,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,KAAKC,QAAQ,UAAU,CAAC,MAAM;YAChC;SACD;IACH;IAEA,IAAIH,AAAiC,qBAAjCA,OAAO,qBAAqB,EAC9B,OAAO;QACL;YAAE,MAAM;YAAU,SAASJ;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBAAC;oBAAE,MAAM;oBAAQ,MAAMS;gBAAgB;aAAE;QACpD;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;IAGH,OAAO;QACL;YACE,MAAM;YACN,SAAS,GAAGJ,eAAeS,iBAAiB;QAC9C;WACGC;WACAJ,oBAAoB,QAAQ,CAACF,OAAO,iBAAiB;KACzD;AACH;AAEO,eAAeQ,kBACpBH,eAAmC,EACnCJ,OAAoB,EACpBD,MAAiD;IAEjD,MAAM,EAAEG,OAAO,EAAE,GAAGF;IACpB,MAAMQ,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQF,QAAQ,YAAY,CAAC,OAAO,CAAC,eAAe;IACtD;IACA,MAAMU,kBAA+B;QACnC,GAAGV,OAAO;QACV,SAAS;YACP,GAAGE,OAAO;YACV,YAAYS,eAAe,MAAM,CAC/BH,cAAc,WAAW,EACzBN,QAAQ,UAAU,CAAC,UAAU;YAE/B,UAAUM,cAAc,YAAY;QACtC;IACF;IACA,MAAMV,QAA6B;QACjCM;QACA,SAASM;QACT,kBAAkBX,OAAO,gBAAgB;IAC3C;IAEA,MAAMa,WAAWf,4BAA4BC,OAAOC,OAAO,QAAQ;IACnE,MAAM,EAAEc,OAAO,EAAEC,KAAK,EAAEC,kBAAkB,EAAE,GAAG,MAAMC,yBACnDJ,UACAF,gBAAgB,YAAY,EAC5B;QACE,aAAaA,gBAAgB,WAAW;QACxC,6BAA6BA,gBAAgB,uBAAuB;IACtE;IAGF,IAAIO;IACJ,IAAIC;IACJ,IAAIC;IAEJ,IAAI;QACFF,SAASlB,OAAO,aAAa,CAACc,SAASf;QACvCoB,UAAUnB,OAAO,gBAAgB,CAACkB,QAAQnB;QAC1CsB,oCAAoCF,SAAS;YAC3C,aAAaR,gBAAgB,WAAW;YACxC,yBAAyBA,gBAAgB,uBAAuB;YAChE,mBAAmBX,OAAO,oBAAoB;YAC9C,qBAAqB;gBACnB,cAAcS,cAAc,YAAY;gBACxC,aAAaA,cAAc,WAAW;YACxC;QACF;QACAa,gCAAgCH,SAAShB,QAAQ,wBAAwB;QACzEiB,yBAAyBpB,OAAO,sBAAsB,CAACkB,QAAQC;IACjE,EAAE,OAAOI,YAAY;QACnB,MAAMC,eAAe,CAAC,aAAa,EACjCD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH,aAC1D;QACF,MAAM,IAAII,qBACRH,cACAI,KAAK,SAAS,CAACd,SAASe,QAAW,IACnCd,OACAC;IAEJ;IAEA,MAAMc,mBAAmB9B,OAAO,QAAQ,CAAC,qBAAqB,GAC5DkB,QACAJ,SACAf;IAGF,IACEiB,oBAAoB,SAAS,eAC7Bf,QAAQ,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,wBAAwB,EAE/DA,QAAQ,mBAAmB,CAAC,iBAAiB,CAACe;SACzC,IAAIc,kBACT7B,QAAQ,mBAAmB,CAAC,aAAa,CAAC;QACxC,MAAM;QACN,SAAS6B;IACX;IAGF,OAAO;QACLX;QACA,KAAKnB,OAAO,gBAAgB,CAACkB,QAAQJ;QACrCC;QACAK;QACA,aAAaQ,KAAK,SAAS,CAACd,SAASe,QAAW;QAChDb;IACF;AACF"}