@midscene/core 1.10.8-beta-20260724035136.0 → 1.10.8

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner, TaskRunnerEvent } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  AiActProgressData,\n  AiActProgressPhase,\n  DeviceAction,\n  ExecutionRecorderItem,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  PlanningLocateParam,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  UIContext,\n} from '@/types';\nimport { ServiceError, aiActProgressScope } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport {\n  type AgentProgressPublisher,\n  createAiActActionReporter,\n  errorMessageForAiAct,\n} from './progress';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onSnapshotChange?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n  // Publish onto the agent's progress bus. The bus owns sequencing, so the\n  // executor is a pure producer: it names the scope/phase and hands over data.\n  onProgress?: AgentProgressPublisher;\n}\n\nexport type ActionReportOptions = {\n  type?: TaskTitleType;\n  prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap each task's planning feedback so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. This is the\n// single place that truncates feedback before it is sent to the model; action\n// implementations should hand over the untruncated value.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(feedback: string): string {\n  if (feedback.length <= maxPlanningFeedbackLength) {\n    return feedback;\n  }\n\n  return `${feedback.slice(0, maxPlanningFeedbackLength)}\n...[truncated, ${feedback.length - maxPlanningFeedbackLength} more characters]`;\n}\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: {\n      tasks?: ExecutionTaskApply[];\n      uiContext?: UIContext;\n      onSnapshotChange?: (\n        runner: TaskRunner,\n        error?: TaskExecutionError,\n      ) => Promise<void> | void;\n      onTaskEvent?: (event: TaskRunnerEvent) => Promise<void> | void;\n    },\n  ) {\n    return new ExecutionSession(\n      title,\n      () =>\n        options?.uiContext\n          ? Promise.resolve(options.uiContext)\n          : Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onSnapshotChange: async (runner, error) => {\n          await this.hooks?.onSnapshotChange?.(runner, error);\n          await options?.onSnapshotChange?.(runner, error);\n        },\n        ...(options?.onTaskEvent ? { onTaskEvent: options.onTaskEvent } : {}),\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Publish one event onto the agent's progress bus. The task layer is a pure\n   * producer here: it names the scope/phase and forwards the structured\n   * payload; the bus stamps the sequence and isolates listener errors. It has\n   * no knowledge of how the event is rendered or consumed.\n   */\n  private async emitProgress(\n    scope: string,\n    phase: string,\n    data: unknown,\n  ): Promise<void> {\n    await this.hooks?.onProgress?.(scope, phase, data);\n  }\n\n  /**\n   * aiAct-flavored convenience over {@link emitProgress}: aiAct is the first\n   * producer (\"pilot\") on the generic bus, so its events are simply tagged with\n   * the `aiAct` scope.\n   */\n  private emitAiActProgress(\n    phase: AiActProgressPhase,\n    data: AiActProgressData,\n  ): Promise<void> {\n    return this.emitProgress(aiActProgressScope, phase, data);\n  }\n\n  /**\n   * Set the pending feedback message consumed by the next planning round.\n   * The message is always prefixed with the current time. When a body is\n   * provided it is appended after the timestamp; otherwise only the time\n   * context is recorded. This is the single entry point for writing\n   * `pendingFeedbackMessage` so the time prefix stays consistent.\n   */\n  private setPendingFeedbackMessage(\n    conversationHistory: ConversationHistory,\n    timeString: string,\n    body?: string,\n  ) {\n    conversationHistory.pendingFeedbackMessage = body\n      ? `Time: ${timeString}, ${body}`\n      : `Current time: ${timeString}`;\n  }\n\n  /**\n   * Collect feedback produced by executed tasks for the next planning round.\n   * Returns undefined when no task reported feedback.\n   */\n  private collectPlanningFeedback(tasks: ExecutionTask[]): string | undefined {\n    const feedbackMessages = tasks.flatMap(({ planningFeedback }) =>\n      planningFeedback ? [truncatePlanningFeedback(planningFeedback)] : [],\n    );\n    return feedbackMessages.length > 0\n      ? feedbackMessages.join('\\n\\n')\n      : undefined;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(\n    userInstruction: TUserPrompt,\n    yamlString: string,\n    reportOptions?: ActionReportOptions,\n  ) {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userInstruction),\n      ),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n        ...(reportOptions?.prompt\n          ? { userInstructionDisplay: reportOptions.prompt }\n          : {}),\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: { uiContext?: UIContext },\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title, options);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      planningModel,\n      defaultModel,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        includeLocateInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n        reportOptions,\n      );\n    });\n  }\n\n  /**\n   * Called when the task is about to replan. Marks every cache-hit locate task\n   * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n   * did not finish the task, so the element each cache hit produced is suspect.\n   * The upcoming re-locate of the same prompt then replaces the bad entry in\n   * place instead of appending a duplicate that would re-poison the cache on the\n   * next run (#2529).\n   *\n   * Marking a locate that was actually fine is harmless: the step is only ever\n   * replaced if the same prompt is located again (i.e. the step is redone),\n   * which does not happen for a locate that already succeeded.\n   */\n  private invalidateFailedCacheHitLocates(\n    runner: TaskRunner,\n    fromIndex: number,\n  ) {\n    if (!this.taskCache) {\n      return;\n    }\n    for (let i = fromIndex; i < runner.tasks.length; i++) {\n      const task = runner.tasks[i];\n      if (\n        task.type === 'Planning' &&\n        task.subType === 'Locate' &&\n        task.hitBy?.from === 'Cache'\n      ) {\n        const prompt = (task.param as PlanningLocateParam | undefined)?.prompt;\n        if (prompt) {\n          this.taskCache.markLocateCacheStale(prompt);\n        }\n      }\n    }\n  }\n\n  private async runAction(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    const conversationHistory = new ConversationHistory();\n    const promptDisplay =\n      reportOptions?.prompt || userPromptToString(userPrompt);\n\n    // Per-call reporter that maps the runner's native task events to aiAct\n    // action progress for the action batch currently running. Kept local (not\n    // on the instance) so concurrent aiAct() calls on the same executor stay\n    // isolated; it is set while a planned batch runs and cleared afterwards.\n    let activeActionReporter:\n      | ((event: TaskRunnerEvent) => Promise<void>)\n      | undefined;\n\n    const session = this.createExecutionSession(\n      taskTitleStr(reportOptions?.type || 'Act', promptDisplay),\n      {\n        onTaskEvent: async (event) => {\n          await activeActionReporter?.(event);\n        },\n      },\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    if (replanningCycleLimit === undefined) {\n      throw new Error(\n        'replanningCycleLimit is required for TaskExecutor.action',\n      );\n    }\n\n    await this.emitAiActProgress('start', {\n      prompt: promptDisplay,\n      planLimit: replanningCycleLimit,\n    });\n\n    const appendFailedPlan = async (\n      errorMsg: string,\n      planIndex?: number,\n    ): Promise<{\n      output: undefined;\n      runner: TaskRunner;\n    }> => {\n      await this.emitAiActProgress('failed', {\n        ...(planIndex ? { planIndex } : {}),\n        planLimit: replanningCycleLimit,\n        error: errorMsg,\n      });\n      return session.appendErrorPlan(errorMsg);\n    };\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n    let latestPlanResult: PlanningAIResponse | undefined;\n    let latestPlanIndex = 0;\n\n    if (abortSignal?.aborted) {\n      return appendFailedPlan(\n        `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n      );\n    }\n    const referenceImageMessages = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(userPrompt),\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      const planIndex = replanCount + 1;\n      latestPlanIndex = planIndex;\n\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return appendFailedPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n          planIndex,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            ...(reportOptions?.prompt\n              ? { userInstructionDisplay: reportOptions.prompt }\n              : {}),\n            replanningCycleLimit,\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const planningUiContext = uiContext as UIContext;\n            const timing = executorContext.task.timing;\n            await this.emitAiActProgress('plan_thinking', {\n              planIndex,\n              planLimit: replanningCycleLimit,\n              screenshot: planningUiContext.screenshot,\n            });\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : genericXmlPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: planningUiContext,\n                actionContext: param.aiActContext,\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                referenceImageMessages,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                  rawChoiceMessage: planError.rawChoiceMessage,\n                };\n              }\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessageForAiAct(planError),\n              });\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              rawChoiceMessage,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n              rawChoiceMessage,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = planningUiContext;\n\n            // Forward the raw in-progress reasoning the model produced; the\n            // consumer decides which field to surface and how to format it.\n            // `finalizeMessage` is intentionally left for the `complete` event.\n            if (log || thought) {\n              await this.emitAiActProgress('plan_planned', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                ...(log ? { log } : {}),\n                ...(thought ? { thought } : {}),\n              });\n            }\n\n            if (error) {\n              const errorMessage = `Failed to continue: ${error}\\n${log || ''}`;\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessage,\n              });\n            }\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              const errorMessage = `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`;\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessage,\n              });\n              assert(false, errorMessage);\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n      latestPlanResult = planResult;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return appendFailedPlan(\n          `Error converting plans to executable tasks: ${errorMessageForAiAct(error)}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n          planIndex,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Capture the time context for the next planning call before running.\n      const initialTimeString = await this.getTimeString();\n\n      const taskCountBeforeRun = runner.tasks.length;\n      // Scope a reporter to this replanning round so native task events from the\n      // runner are mapped to aiAct progress with the right plan context; cleared\n      // in `finally` so events fired between batches are ignored.\n      activeActionReporter = createAiActActionReporter(\n        planIndex,\n        replanningCycleLimit,\n        (phase, data) => this.emitAiActProgress(phase, data),\n      );\n      try {\n        await session.appendAndRun(executables.tasks);\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          initialTimeString,\n          this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n        );\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          timeString,\n          `Error executing running tasks: ${error?.message || String(error)}`,\n        );\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      } finally {\n        activeActionReporter = undefined;\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return appendFailedPlan(\n          'Too many errors in one planning loop',\n          planIndex,\n        );\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return appendFailedPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n          planIndex,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // We are about to replan, which means the batch we just ran did not finish\n      // the task. Any locate task in that batch that was served from cache\n      // produced an element that failed to complete the step (the action threw,\n      // or it clicked the wrong element and the goal was not reached). Mark those\n      // cache entries stale so the re-locate of the same prompt replaces them in\n      // place instead of appending a poisoning duplicate that would be matched\n      // first on the next run (#2529).\n      this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return appendFailedPlan(errorMsg, planIndex);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    // Carry the raw final text; the consumer formats it. `outputString` is the\n    // model's finalize message, with the latest plan's log/thought as fallback\n    // context for consumers that want to show why the run completed.\n    await this.emitAiActProgress('complete', {\n      planIndex: latestPlanIndex,\n      planLimit: replanningCycleLimit,\n      ...((outputString ?? latestPlanResult?.output)\n        ? { output: outputString ?? latestPlanResult?.output }\n        : {}),\n      ...(latestPlanResult?.log ? { log: latestPlanResult.log } : {}),\n      ...(latestPlanResult?.thought\n        ? { thought: latestPlanResult.thought }\n        : {}),\n    });\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n            executionOptions,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        } finally {\n          // Surface the captured screenshot sequence in the report, then release\n          // it from the UIContext so its base64 is not retained twice.\n          recordAndReleaseScreenshotSequence(task, uiContext);\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n      uiContext?: UIContext;\n    },\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      executionOptions?.uiContext\n        ? { uiContext: executionOptions.uiContext }\n        : undefined,\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelRuntime,\n      opt,\n      multimodalPrompt,\n      executionOptions,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\n/**\n * Surface a captured screenshot sequence in the report timeline, then release\n * it from the UIContext.\n *\n * When a UIObserver assertion runs, the observed frames live on\n * `uiContext.screenshotSequence` only as a transient model input. This attaches\n * them to the task recorder so the report renders the full sequence the model\n * saw (the report timeline builds one screenshot per recorder item), then drops\n * the field from the UIContext so its base64 is not retained twice for the\n * lifetime of the dump.\n *\n * The last frame is the representative `uiContext.screenshot`, already shown in\n * the report, so only the earlier frames are recorded to avoid duplication.\n * Observed frames are PREPENDED to `task.recorder` (rather than appended) so\n * array order matches chronological order — they were captured before the\n * assertion's before/after screenshots.\n */\nexport function recordAndReleaseScreenshotSequence(\n  task: ExecutionTask,\n  uiContext: UIContext | undefined,\n): void {\n  const frames = uiContext?.screenshotSequence;\n  if (frames && frames.length > 1) {\n    const recorderItems: ExecutionRecorderItem[] = [];\n    for (let i = 0; i < frames.length - 1; i++) {\n      const frame = frames[i];\n      recorderItems.push({\n        type: 'screenshot',\n        ts: frame.capturedAt,\n        screenshot: frame,\n        description: `Observed frame ${i + 1}/${frames.length}`,\n        timing: 'observed-frame',\n      });\n    }\n    // Prepend observed frames so they appear before the task's own\n    // before/after screenshots in array order. Observed frames have earlier\n    // timestamps than the assertion task itself, so this keeps array order\n    // aligned with chronological order for consumers that iterate in-place.\n    task.recorder = task.recorder\n      ? [...recorderItems, ...task.recorder]\n      : recorderItems;\n  }\n  if (uiContext?.screenshotSequence) {\n    uiContext.screenshotSequence = undefined;\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = await getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","TaskExecutor","title","options","ExecutionSession","Promise","runner","error","scope","phase","data","aiActProgressScope","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","undefined","format","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","task","param","executorContext","uiContext","assert","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","fromIndex","i","prompt","ConversationHistory","promptDisplay","activeActionReporter","event","replanCount","yamlFlow","replanningCycleLimit","Error","appendFailedPlan","errorMsg","planIndex","errorCountInOnePlanningLoop","outputString","latestPlanResult","latestPlanIndex","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","subGoalStatus","memoriesStatus","planningUiContext","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","errorMessageForAiAct","JSON","actions","thought","log","memory","usage","rawResponse","rawChoiceMessage","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","errorMessage","executables","initialTimeString","taskCountBeforeRun","createAiActActionReporter","String","type","demand","modelRuntime","opt","multimodalPrompt","executionOptions","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","recordAndReleaseScreenshotSequence","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","frames","recorderItems","frame","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AA2EA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAM9C,MAAMC,4BAA4B;AAElC,SAASC,yBAAyBC,QAAgB;IAChD,IAAIA,SAAS,MAAM,IAAIF,2BACrB,OAAOE;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGF,2BAA2B;eAC1C,EAAEE,SAAS,MAAM,GAAGF,0BAA0B,iBAAiB,CAAC;AAC/E;AAIO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAQC,EACD;QACA,OAAO,IAAIC,iBACTF,OACA,IACEC,SAAS,YACLE,QAAQ,OAAO,CAACF,QAAQ,SAAS,IACjCE,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,kBAAkB,OAAOG,QAAQC;gBAC/B,MAAM,IAAI,CAAC,KAAK,EAAE,mBAAmBD,QAAQC;gBAC7C,MAAMJ,SAAS,mBAAmBG,QAAQC;YAC5C;YACA,GAAIJ,SAAS,cAAc;gBAAE,aAAaA,QAAQ,WAAW;YAAC,IAAI,CAAC,CAAC;QACtE;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IAQA,MAAc,aACZK,KAAa,EACbC,KAAa,EACbC,IAAa,EACE;QACf,MAAM,IAAI,CAAC,KAAK,EAAE,aAAaF,OAAOC,OAAOC;IAC/C;IAOQ,kBACND,KAAyB,EACzBC,IAAuB,EACR;QACf,OAAO,IAAI,CAAC,YAAY,CAACC,oBAAoBF,OAAOC;IACtD;IASQ,0BACNE,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CAAC,CAAC,EAAEE,gBAAgB,EAAE,GAC1DA,mBAAmB;gBAAClB,yBAAyBkB;aAAkB,GAAG,EAAE;QAEtE,OAAOD,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBE;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOZ,OAAO;YACdX,QACE,CAAC,gEAAgE,EAAEW,OAAO;QAE9E;aAEAX,QACE;QAKN,OAAOwB,sBAAsBD;IAC/B;IAEA,MAAa,wBACXE,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BpB,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACkB,OAAOC,eAAeC,cAAcpB;IACpE;IAEA,MAAM,uBACJqB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,mBAAmBL;QAIhD,MAAMM,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLN;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMnB,SAASqB,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACG;QAE3B,OAAO;YACLxB;QACF;IACF;IAEA,MAAM,SACJJ,KAAa,EACbmB,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BpB,OAAmC,EACT;QAC1B,MAAMwB,UAAU,IAAI,CAAC,sBAAsB,CAACzB,OAAOC;QACnD,MAAM,EAAEY,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDM,OACAC,eACAC;QAEF,MAAMjB,SAASqB,QAAQ,SAAS;QAChC,MAAMQ,SAAS,MAAMR,QAAQ,YAAY,CAACZ;QAC1C,MAAM,EAAEqB,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACA9B;QACF;IACF;IAEA,MAAM,OACJ+B,UAAuB,EACvBf,aAA2B,EAC3BC,YAA0B,EAC1Be,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EACzBpB,aAAmC,EASnC;QACA,OAAOqB,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAf,eACAC,cACAe,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC,aACApB;IAGN;IAcQ,gCACNpB,MAAkB,EAClB0C,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAI3C,OAAO,KAAK,CAAC,MAAM,EAAE2C,IAAK;YACpD,MAAMnB,OAAOxB,OAAO,KAAK,CAAC2C,EAAE;YAC5B,IACEnB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMoB,SAAUpB,KAAK,KAAK,EAAsC;gBAChE,IAAIoB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZb,UAAuB,EACvBf,aAA2B,EAC3BC,YAA0B,EAC1Be,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EACzBpB,aAAmC,EASnC;QACA,MAAMd,sBAAsB,IAAIuC;QAChC,MAAMC,gBACJ1B,eAAe,UAAUG,mBAAmBQ;QAM9C,IAAIgB;QAIJ,MAAM1B,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAaF,eAAe,QAAQ,OAAO0B,gBAC3C;YACE,aAAa,OAAOE;gBAClB,MAAMD,uBAAuBC;YAC/B;QACF;QAEF,MAAMhD,SAASqB,QAAQ,SAAS;QAEhC,IAAI4B,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJhB,gCAAgC,IAAI,CAAC,oBAAoB;QAC3D,IAAIgB,AAAyBvC,WAAzBuC,sBACF,MAAM,IAAIC,MACR;QAIJ,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS;YACpC,QAAQN;YACR,WAAWK;QACb;QAEA,MAAME,mBAAmB,OACvBC,UACAC;YAKA,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU;gBACrC,GAAIA,YAAY;oBAAEA;gBAAU,IAAI,CAAC,CAAC;gBAClC,WAAWJ;gBACX,OAAOG;YACT;YACA,OAAOjC,QAAQ,eAAe,CAACiC;QACjC;QAEA,IAAIE,8BAA8B;QAClC,IAAIC;QACJ,IAAIC;QACJ,IAAIC,kBAAkB;QAEtB,IAAInB,aAAa,SACf,OAAOa,iBACL,CAAC,cAAc,EAAEb,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMoB,yBAAyB,MAAMC,+BACnCC,6BAA6B/B;QAI/B,MAAO,KAAM;YACX,MAAMwB,YAAYN,cAAc;YAChCU,kBAAkBJ;YAGlB,IAAIf,aAAa,SACf,OAAOa,iBACL,CAAC,cAAc,EAAEb,YAAY,MAAM,IAAI,yBAAyB,EAChEe;YAKJ,MAAMQ,gBAAgBzD,oBAAoB,cAAc,MAAMM;YAG9D,MAAMoD,iBAAiB1D,oBAAoB,cAAc,MAAMM;YAE/D,MAAMiB,SAAS,MAAMR,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBU;oBACjB,GAAIX,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACN+B;oBACAlB;oBACAG;oBACAC;oBACA,GAAI0B,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAOvC,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAMsC,oBAAoBtC;oBAC1B,MAAMuC,SAASxC,gBAAgB,IAAI,CAAC,MAAM;oBAC1C,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB;wBAC5C6B;wBACA,WAAWJ;wBACX,YAAYc,kBAAkB,UAAU;oBAC1C;oBAEA,MAAME,cAAc,IAAI,CAAC,cAAc;oBACvC/E,MACE,sCACA+E,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhDxC,OAAOyC,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJvD,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrCwD;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAAS9C,MAAM,eAAe,EAAE;4BACjD,SAASwC;4BACT,eAAexC,MAAM,YAAY;4BACjC0C;4BACA,cAAcnD;4BACdV;4BACA0B;4BACAI;4BACAC;4BACAuB;4BACApB;wBACF;oBACF,EAAE,OAAOmC,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7ClD,gBAAgB,IAAI,CAAC,KAAK,GAAGmD,gBAC3BF,UAAU,KAAK,EACf;4BAEFjD,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAaiD,UAAU,WAAW;gCAClC,kBAAkBA,UAAU,gBAAgB;4BAC9C;wBACF;wBACA,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1CpB;4BACA,WAAWJ;4BACX,OAAO2B,qBAAqBH;wBAC9B;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBR,QAAQ;oBAC7B;oBACA9E,MAAM,cAAc2F,KAAK,SAAS,CAACN,YAAY,MAAM;oBAErD,MAAM,EACJO,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACNlF,KAAK,EACLmF,KAAK,EACLC,WAAW,EACXC,gBAAgB,EAChBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGlB;oBACJhB,eAAegC;oBAEf/D,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClC2D;wBACAC;oBACF;oBACA5D,gBAAgB,IAAI,CAAC,KAAK,GAAGmD,gBAAgBO,OAAO;oBACpD1D,gBAAgB,IAAI,CAAC,iBAAiB,GAAG6D;oBACzC7D,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAASsD,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUV,WAAW,QAAQ;wBAC7B,QAAQgB;wBACR,wBAAwBhB,WAAW,sBAAsB;wBACzDiB;wBACAC;oBACF;oBACAjE,gBAAgB,SAAS,GAAGuC;oBAK5B,IAAIiB,OAAOD,SACT,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB;wBAC3C1B;wBACA,WAAWJ;wBACX,GAAI+B,MAAM;4BAAEA;wBAAI,IAAI,CAAC,CAAC;wBACtB,GAAID,UAAU;4BAAEA;wBAAQ,IAAI,CAAC,CAAC;oBAChC;oBAGF,IAAIhF,OAAO;wBACT,MAAM2F,eAAe,CAAC,oBAAoB,EAAE3F,MAAM,EAAE,EAAEiF,OAAO,IAAI;wBACjE,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1C3B;4BACA,WAAWJ;4BACX,OAAOyC;wBACT;oBACF;oBAEAhE,OAAO,CAAC3B,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEiF,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBAA2B;wBAC7B,MAAMI,eAAe,CAAC,aAAa,EAAEH,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;wBACnG,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1C3B;4BACA,WAAWJ;4BACX,OAAOyC;wBACT;wBACAhE,OAAO,OAAOgE;oBAChB;oBAEA,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMnB,aAAa5C,QAAQ;YAC3B6B,mBAAmBe;YAGnB,MAAM1D,QAAQ0D,YAAY,WAAW,EAAE;YACvCvB,SAAS,IAAI,IAAKuB,YAAY,YAAY,EAAE;YAE5C,IAAIoB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9C9E,OACAC,eACAC,cACA;oBACEiB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAOvC,OAAO;gBACd,OAAOoD,iBACL,CAAC,4CAA4C,EAAEyB,qBAAqB7E,OAAO,SAAS,EAAE8E,KAAK,SAAS,CAClGhE,QACC,EACHwC;YAEJ;YACA,IAAIjD,oBAAoB,sBAAsB,EAC5CgE,QAAQ,IAAI,CACV,8FACAhE,oBAAoB,sBAAsB;YAK9C,MAAMwF,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqB/F,OAAO,KAAK,CAAC,MAAM;YAI9C+C,uBAAuBiD,0BACrBzC,WACAJ,sBACA,CAAChD,OAAOC,OAAS,IAAI,CAAC,iBAAiB,CAACD,OAAOC;YAEjD,IAAI;gBACF,MAAMiB,QAAQ,YAAY,CAACwE,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5BvF,qBACAwF,mBACA,IAAI,CAAC,uBAAuB,CAAC9F,OAAO,KAAK,CAAC,KAAK,CAAC+F;YAEpD,EAAE,OAAO9F,OAAY;gBAEnBuD;gBACA,MAAMjD,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEN,OAAO,WAAWgG,OAAOhG,QAAQ;gBAErEb,MACE,yFACAa,iBAAiBmD,QAAQnD,MAAM,OAAO,GAAGgG,OAAOhG,QAChD,6CACAuD;YAEJ,SAAU;gBACRT,uBAAuBnC;YACzB;YAEA,IAAI4C,8BAA8BjE,uCAChC,OAAO8D,iBACL,wCACAE;YAKJ,IAAIf,aAAa,SACf,OAAOa,iBACL,CAAC,cAAc,EAAEb,YAAY,MAAM,IAAI,yBAAyB,EAChEe;YAKJ,IAAI,CAACkB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAACzE,QAAQ+F;YAG7C,EAAE9C;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMG,WAAW,CAAC,UAAU,EAAEH,qBAAqB,4JAA4J,CAAC;gBAChN,OAAOE,iBAAiBC,UAAUC;YACpC;YAEA,IAAI,CAACjD,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAKA,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY;YACvC,WAAWoD;YACX,WAAWR;YACX,GAAKM,gBAAgBC,kBAAkB,SACnC;gBAAE,QAAQD,gBAAgBC,kBAAkB;YAAO,IACnD,CAAC,CAAC;YACN,GAAIA,kBAAkB,MAAM;gBAAE,KAAKA,iBAAiB,GAAG;YAAC,IAAI,CAAC,CAAC;YAC9D,GAAIA,kBAAkB,UAClB;gBAAE,SAASA,iBAAiB,OAAO;YAAC,IACpC,CAAC,CAAC;QACR;QAEA,OAAO;YACL,QAAQ;gBACNR;gBACA,QAAQO;YACV;YACAzD;QACF;IACF;IAEQ,oBACNkG,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAEC,EACD;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASN;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAO1E,OAAOgF;gBACtB,MAAM,EAAEjF,IAAI,EAAE,GAAGiF;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZpF,KAAK,GAAG,GAAG;wBACToF;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACApF,KAAK,KAAK,GAAGqD,gBAAgB+B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjBpF,KAAK,iBAAiB,GAAGoF,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMjF,YAAY8E,YAAY,SAAS;gBACvC7E,OAAOD,WAAW;gBAElB,MAAMkF,mBAAmBX,AAAS,YAATA;gBACzB,IAAIY,cAAcX;gBAClB,IAAIY,cAAc;gBAClB,IAAIF,oBAAqBX,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEa,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF,OAAO,IAAIU,kBAAkB;oBAC3BE,cAAcb;oBACdY,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF;gBAEA,IAAIc;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIb,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DjH,MAAM;oBACN,MAAM+H,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAd,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAV,cACAC,KACAa,sBACAZ,kBACA3E,WACA4E;gBAEJ,EAAE,OAAOtG,OAAO;oBACd,IAAIA,iBAAiBoH,cACnBV,UAAU1G,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBAGRqH,mCAAmC9F,MAAMG;gBAC3C;gBAEA,MAAM,EAAEvB,IAAI,EAAE6E,OAAO,EAAE2B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAenH;gBACnB,IAAIyG,kBAEF,IAAI,AAAgB,YAAhB,OAAOzG,MACTmH,eAAenH;qBACV,IAAI8F,AAAS,cAATA,MAEPqB,eADEnH,QAAAA,OACa,QAECA,IAAY,CAAC2G,YAAY;qBAEtC,IAAI3G,QAAAA,MACTmH,eAAe;qBAGf,IAAInH,MAAM,CAAC2G,YAAY,KAAKnG,QAC1B2G,eAAgBnH,IAAY,CAAC2G,YAAY;qBACpC,IAAI3G,MAAM,WAAWQ,QAC1B2G,eAAgBnH,KAAa,MAAM;qBAEnCwB,OAAO,OAAO;gBAKpB,IAAIsE,AAAS,aAATA,QAAqB,CAACqB,cAAc;oBACtC/F,KAAK,OAAO,GAAGyD;oBACf,MAAM,IAAI7B,MAAM,CAAC,kBAAkB,EAAE6B,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQsC;oBACR,KAAKb;oBACLzB;gBACF;YACF;QACF;QAEA,OAAOuB;IACT;IACA,MAAM,yBACJN,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAGC,EAC4B;QAC7B,MAAMlF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACE4E,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB,UAEvDI,kBAAkB,YACd;YAAE,WAAWA,iBAAiB,SAAS;QAAC,IACxC3F;QAGN,MAAM4F,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CN,MACAC,QACAC,cACAC,KACAC,kBACAC;QAGF,MAAMvG,SAASqB,QAAQ,SAAS;QAChC,MAAMQ,SAAS,MAAMR,QAAQ,YAAY,CAACmF;QAE1C,IAAI,CAAC3E,QACH,MAAM,IAAIuB,MACR;QAIJ,MAAM,EAAEtB,MAAM,EAAEmD,OAAO,EAAE,GAAGpD;QAE5B,OAAO;YACLC;YACAmD;YACAjF;QACF;IACF;IAEA,MAAM,QACJwH,SAAsB,EACtBnB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEqB,UAAU,EAAEnB,gBAAgB,EAAE,GAAGoB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMpG,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAWqG;QAE1B,MAAM3H,SAASqB,QAAQ,SAAS;QAChC,MAAM,EACJuG,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG3B;QACJ,MAAM4B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEApG,OAAO4F,WAAW;QAClB5F,OAAOgG,WAAW;QAClBhG,OAAOiG,iBAAiB;QAExBjG,OACEiG,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACArB,cACA6B,mBACA3B;YAGF,MAAMzE,SAAU,MAAMR,QAAQ,YAAY,CAACmF;YAO3C,IAAI3E,QAAQ,QACV,OAAO;gBACL,QAAQjB;gBACRZ;YACF;YAGFqI,eACExG,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAE4F,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMvD,UAAU,CAAC,kBAAkB,EAAE4C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGxD;oBAAQ;iBAAE,EAC9DmB,cACAA;gBAEF,IAAIsC,UAAU,CAAC,EAAE,EACf,MAAMrH,QAAQ,YAAY,CAACqH,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAOrH,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEgH,cAAc;IACnE;IA99BA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAk8BF;AAmBO,SAASvB,mCACd9F,IAAmB,EACnBG,SAAgC;IAEhC,MAAMoH,SAASpH,WAAW;IAC1B,IAAIoH,UAAUA,OAAO,MAAM,GAAG,GAAG;QAC/B,MAAMC,gBAAyC,EAAE;QACjD,IAAK,IAAIrG,IAAI,GAAGA,IAAIoG,OAAO,MAAM,GAAG,GAAGpG,IAAK;YAC1C,MAAMsG,QAAQF,MAAM,CAACpG,EAAE;YACvBqG,cAAc,IAAI,CAAC;gBACjB,MAAM;gBACN,IAAIC,MAAM,UAAU;gBACpB,YAAYA;gBACZ,aAAa,CAAC,eAAe,EAAEtG,IAAI,EAAE,CAAC,EAAEoG,OAAO,MAAM,EAAE;gBACvD,QAAQ;YACV;QACF;QAKAvH,KAAK,QAAQ,GAAGA,KAAK,QAAQ,GACzB;eAAIwH;eAAkBxH,KAAK,QAAQ;SAAC,GACpCwH;IACN;IACA,IAAIrH,WAAW,oBACbA,UAAU,kBAAkB,GAAGf;AAEnC;AAEO,eAAe6B,gBACpBkG,iBAAoC,EACpCrG,iBAAuC,EACvC8B,MAAwB;IAExB,IAAI,CAAC9B,mBAAmB,QACtB,OAAO8B;IAGT,IAAI,CAACuE,kBAAkB,2BAA2B,EAChD,MAAM,IAAIvF,MACR,CAAC,gCAAgC,EAAEuF,kBAAkB,aAAa,EAAE;IAIxE,MAAMO,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC7G;IACvB;IAEA,MAAM,EAAE8G,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMV,kBAAkB,2BAA2B,CAACO;IACtD,IAAI;QACF,MAAMrH,SAAS,MAAMuC;QAErB,MAAMnE,QAAQ,MAAMoJ;QACpB,IAAIpJ,OACF,MAAMA;QAER,OAAO4B;IACT,SAAU;QACRuH;IACF;AACF"}
+{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner, TaskRunnerEvent } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  AiActProgressData,\n  AiActProgressPhase,\n  DeviceAction,\n  ExecutionRecorderItem,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  PlanningLocateParam,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  UIContext,\n} from '@/types';\nimport { ServiceError, aiActProgressScope } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { withFileChooser } from './file-chooser';\nimport {\n  type AgentProgressPublisher,\n  createAiActActionReporter,\n  errorMessageForAiAct,\n} from './progress';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { type TaskTitleType, taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onSnapshotChange?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n  // Publish onto the agent's progress bus. The bus owns sequencing, so the\n  // executor is a pure producer: it names the scope/phase and hands over data.\n  onProgress?: AgentProgressPublisher;\n}\n\nexport type ActionReportOptions = {\n  type?: TaskTitleType;\n  prompt?: string;\n};\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\n// Cap each task's planning feedback so a large action output (e.g. a long adb\n// shell stdout) cannot blow up the next planning request's context. This is the\n// single place that truncates feedback before it is sent to the model; action\n// implementations should hand over the untruncated value.\nconst maxPlanningFeedbackLength = 500;\n\nfunction truncatePlanningFeedback(feedback: string): string {\n  if (feedback.length <= maxPlanningFeedbackLength) {\n    return feedback;\n  }\n\n  return `${feedback.slice(0, maxPlanningFeedbackLength)}\n...[truncated, ${feedback.length - maxPlanningFeedbackLength} more characters]`;\n}\n\nexport { TaskExecutionError };\nexport { withFileChooser } from './file-chooser';\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: {\n      tasks?: ExecutionTaskApply[];\n      uiContext?: UIContext;\n      onSnapshotChange?: (\n        runner: TaskRunner,\n        error?: TaskExecutionError,\n      ) => Promise<void> | void;\n      onTaskEvent?: (event: TaskRunnerEvent) => Promise<void> | void;\n    },\n  ) {\n    return new ExecutionSession(\n      title,\n      () =>\n        options?.uiContext\n          ? Promise.resolve(options.uiContext)\n          : Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onSnapshotChange: async (runner, error) => {\n          await this.hooks?.onSnapshotChange?.(runner, error);\n          await options?.onSnapshotChange?.(runner, error);\n        },\n        ...(options?.onTaskEvent ? { onTaskEvent: options.onTaskEvent } : {}),\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Publish one event onto the agent's progress bus. The task layer is a pure\n   * producer here: it names the scope/phase and forwards the structured\n   * payload; the bus stamps the sequence and isolates listener errors. It has\n   * no knowledge of how the event is rendered or consumed.\n   */\n  private async emitProgress(\n    scope: string,\n    phase: string,\n    data: unknown,\n  ): Promise<void> {\n    await this.hooks?.onProgress?.(scope, phase, data);\n  }\n\n  /**\n   * aiAct-flavored convenience over {@link emitProgress}: aiAct is the first\n   * producer (\"pilot\") on the generic bus, so its events are simply tagged with\n   * the `aiAct` scope.\n   */\n  private emitAiActProgress(\n    phase: AiActProgressPhase,\n    data: AiActProgressData,\n  ): Promise<void> {\n    return this.emitProgress(aiActProgressScope, phase, data);\n  }\n\n  /**\n   * Set the pending feedback message consumed by the next planning round.\n   * The message is always prefixed with the current time. When a body is\n   * provided it is appended after the timestamp; otherwise only the time\n   * context is recorded. This is the single entry point for writing\n   * `pendingFeedbackMessage` so the time prefix stays consistent.\n   */\n  private setPendingFeedbackMessage(\n    conversationHistory: ConversationHistory,\n    timeString: string,\n    body?: string,\n  ) {\n    conversationHistory.pendingFeedbackMessage = body\n      ? `Time: ${timeString}, ${body}`\n      : `Current time: ${timeString}`;\n  }\n\n  /**\n   * Collect feedback produced by executed tasks for the next planning round.\n   * Returns undefined when no task reported feedback.\n   */\n  private collectPlanningFeedback(tasks: ExecutionTask[]): string | undefined {\n    const feedbackMessages = tasks.flatMap(({ planningFeedback }) =>\n      planningFeedback ? [truncatePlanningFeedback(planningFeedback)] : [],\n    );\n    return feedbackMessages.length > 0\n      ? feedbackMessages.join('\\n\\n')\n      : undefined;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(\n    userInstruction: TUserPrompt,\n    yamlString: string,\n    reportOptions?: ActionReportOptions,\n  ) {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        reportOptions?.type || 'Act',\n        reportOptions?.prompt || userPromptToString(userInstruction),\n      ),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n        ...(reportOptions?.prompt\n          ? { userInstructionDisplay: reportOptions.prompt }\n          : {}),\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: { uiContext?: UIContext },\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title, options);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      planningModel,\n      defaultModel,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        includeLocateInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n        reportOptions,\n      );\n    });\n  }\n\n  /**\n   * Called when the task is about to replan. Marks every cache-hit locate task\n   * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n   * did not finish the task, so the element each cache hit produced is suspect.\n   * The upcoming re-locate of the same prompt then replaces the bad entry in\n   * place instead of appending a duplicate that would re-poison the cache on the\n   * next run (#2529).\n   *\n   * Marking a locate that was actually fine is harmless: the step is only ever\n   * replaced if the same prompt is located again (i.e. the step is redone),\n   * which does not happen for a locate that already succeeded.\n   */\n  private invalidateFailedCacheHitLocates(\n    runner: TaskRunner,\n    fromIndex: number,\n  ) {\n    if (!this.taskCache) {\n      return;\n    }\n    for (let i = fromIndex; i < runner.tasks.length; i++) {\n      const task = runner.tasks[i];\n      if (\n        task.type === 'Planning' &&\n        task.subType === 'Locate' &&\n        task.hitBy?.from === 'Cache'\n      ) {\n        const prompt = (task.param as PlanningLocateParam | undefined)?.prompt;\n        if (prompt) {\n          this.taskCache.markLocateCacheStale(prompt);\n        }\n      }\n    }\n  }\n\n  private async runAction(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n    reportOptions?: ActionReportOptions,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    const conversationHistory = new ConversationHistory();\n    const promptDisplay =\n      reportOptions?.prompt || userPromptToString(userPrompt);\n\n    // Per-call reporter that maps the runner's native task events to aiAct\n    // action progress for the action batch currently running. Kept local (not\n    // on the instance) so concurrent aiAct() calls on the same executor stay\n    // isolated; it is set while a planned batch runs and cleared afterwards.\n    let activeActionReporter:\n      | ((event: TaskRunnerEvent) => Promise<void>)\n      | undefined;\n\n    const session = this.createExecutionSession(\n      taskTitleStr(reportOptions?.type || 'Act', promptDisplay),\n      {\n        onTaskEvent: async (event) => {\n          await activeActionReporter?.(event);\n        },\n      },\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    if (replanningCycleLimit === undefined) {\n      throw new Error(\n        'replanningCycleLimit is required for TaskExecutor.action',\n      );\n    }\n\n    await this.emitAiActProgress('start', {\n      prompt: promptDisplay,\n      planLimit: replanningCycleLimit,\n    });\n\n    const appendFailedPlan = async (\n      errorMsg: string,\n      planIndex?: number,\n    ): Promise<{\n      output: undefined;\n      runner: TaskRunner;\n    }> => {\n      await this.emitAiActProgress('failed', {\n        ...(planIndex ? { planIndex } : {}),\n        planLimit: replanningCycleLimit,\n        error: errorMsg,\n      });\n      return session.appendErrorPlan(errorMsg);\n    };\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n    let latestPlanResult: PlanningAIResponse | undefined;\n    let latestPlanIndex = 0;\n\n    if (abortSignal?.aborted) {\n      return appendFailedPlan(\n        `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n      );\n    }\n    const referenceImageMessages = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(userPrompt),\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      const planIndex = replanCount + 1;\n      latestPlanIndex = planIndex;\n\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return appendFailedPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n          planIndex,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            ...(reportOptions?.prompt\n              ? { userInstructionDisplay: reportOptions.prompt }\n              : {}),\n            replanningCycleLimit,\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const planningUiContext = uiContext as UIContext;\n            const timing = executorContext.task.timing;\n            await this.emitAiActProgress('plan_thinking', {\n              planIndex,\n              planLimit: replanningCycleLimit,\n              screenshot: planningUiContext.screenshot,\n            });\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : genericXmlPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: planningUiContext,\n                actionContext: param.aiActContext,\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                referenceImageMessages,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                  rawChoiceMessage: planError.rawChoiceMessage,\n                };\n              }\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessageForAiAct(planError),\n              });\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              rawChoiceMessage,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n              rawChoiceMessage,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = planningUiContext;\n\n            // Forward the raw in-progress reasoning the model produced; the\n            // consumer decides which field to surface and how to format it.\n            // `finalizeMessage` is intentionally left for the `complete` event.\n            if (log || thought) {\n              await this.emitAiActProgress('plan_planned', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                ...(log ? { log } : {}),\n                ...(thought ? { thought } : {}),\n              });\n            }\n\n            if (error) {\n              const errorMessage = `Failed to continue: ${error}\\n${log || ''}`;\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessage,\n              });\n            }\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              const errorMessage = `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`;\n              await this.emitAiActProgress('plan_failed', {\n                planIndex,\n                planLimit: replanningCycleLimit,\n                error: errorMessage,\n              });\n              assert(false, errorMessage);\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n      latestPlanResult = planResult;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return appendFailedPlan(\n          `Error converting plans to executable tasks: ${errorMessageForAiAct(error)}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n          planIndex,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Capture the time context for the next planning call before running.\n      const initialTimeString = await this.getTimeString();\n\n      const taskCountBeforeRun = runner.tasks.length;\n      // Scope a reporter to this replanning round so native task events from the\n      // runner are mapped to aiAct progress with the right plan context; cleared\n      // in `finally` so events fired between batches are ignored.\n      activeActionReporter = createAiActActionReporter(\n        planIndex,\n        replanningCycleLimit,\n        (phase, data) => this.emitAiActProgress(phase, data),\n      );\n      try {\n        await session.appendAndRun(executables.tasks);\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          initialTimeString,\n          this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)),\n        );\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        this.setPendingFeedbackMessage(\n          conversationHistory,\n          timeString,\n          `Error executing running tasks: ${error?.message || String(error)}`,\n        );\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      } finally {\n        activeActionReporter = undefined;\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return appendFailedPlan(\n          'Too many errors in one planning loop',\n          planIndex,\n        );\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return appendFailedPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n          planIndex,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // We are about to replan, which means the batch we just ran did not finish\n      // the task. Any locate task in that batch that was served from cache\n      // produced an element that failed to complete the step (the action threw,\n      // or it clicked the wrong element and the goal was not reached). Mark those\n      // cache entries stale so the re-locate of the same prompt replaces them in\n      // place instead of appending a poisoning duplicate that would be matched\n      // first on the next run (#2529).\n      this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return appendFailedPlan(errorMsg, planIndex);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    // Carry the raw final text; the consumer formats it. `outputString` is the\n    // model's finalize message, with the latest plan's log/thought as fallback\n    // context for consumers that want to show why the run completed.\n    await this.emitAiActProgress('complete', {\n      planIndex: latestPlanIndex,\n      planLimit: replanningCycleLimit,\n      ...((outputString ?? latestPlanResult?.output)\n        ? { output: outputString ?? latestPlanResult?.output }\n        : {}),\n      ...(latestPlanResult?.log ? { log: latestPlanResult.log } : {}),\n      ...(latestPlanResult?.thought\n        ? { thought: latestPlanResult.thought }\n        : {}),\n    });\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        ...(opt?.context !== undefined ? { context: opt.context } : {}),\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n            executionOptions,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        } finally {\n          // Surface the captured screenshot sequence in the report, then release\n          // it from the UIContext so its base64 is not retained twice.\n          recordAndReleaseScreenshotSequence(task, uiContext);\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n    executionOptions?: {\n      abortSignal?: AbortSignal;\n      uiContext?: UIContext;\n    },\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n      executionOptions?.uiContext\n        ? { uiContext: executionOptions.uiContext }\n        : undefined,\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelRuntime,\n      opt,\n      multimodalPrompt,\n      executionOptions,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n/**\n * Surface a captured screenshot sequence in the report timeline, then release\n * it from the UIContext.\n *\n * When a UIObserver assertion runs, the observed frames live on\n * `uiContext.screenshotSequence` only as a transient model input. This attaches\n * them to the task recorder so the report renders the full sequence the model\n * saw (the report timeline builds one screenshot per recorder item), then drops\n * the field from the UIContext so its base64 is not retained twice for the\n * lifetime of the dump.\n *\n * The last frame is the representative `uiContext.screenshot`, already shown in\n * the report, so only the earlier frames are recorded to avoid duplication.\n * Observed frames are PREPENDED to `task.recorder` (rather than appended) so\n * array order matches chronological order — they were captured before the\n * assertion's before/after screenshots.\n */\nexport function recordAndReleaseScreenshotSequence(\n  task: ExecutionTask,\n  uiContext: UIContext | undefined,\n): void {\n  const frames = uiContext?.screenshotSequence;\n  if (frames && frames.length > 1) {\n    const recorderItems: ExecutionRecorderItem[] = [];\n    for (let i = 0; i < frames.length - 1; i++) {\n      const frame = frames[i];\n      recorderItems.push({\n        type: 'screenshot',\n        ts: frame.capturedAt,\n        screenshot: frame,\n        description: `Observed frame ${i + 1}/${frames.length}`,\n        timing: 'observed-frame',\n      });\n    }\n    // Prepend observed frames so they appear before the task's own\n    // before/after screenshots in array order. Observed frames have earlier\n    // timestamps than the assertion task itself, so this keeps array order\n    // aligned with chronological order for consumers that iterate in-place.\n    task.recorder = task.recorder\n      ? [...recorderItems, ...task.recorder]\n      : recorderItems;\n  }\n  if (uiContext?.screenshotSequence) {\n    uiContext.screenshotSequence = undefined;\n  }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","maxPlanningFeedbackLength","truncatePlanningFeedback","feedback","TaskExecutor","title","options","ExecutionSession","Promise","runner","error","scope","phase","data","aiActProgressScope","conversationHistory","timeString","body","tasks","feedbackMessages","planningFeedback","undefined","format","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","reportOptions","session","taskTitleStr","userPromptToString","task","param","executorContext","uiContext","assert","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","fromIndex","i","prompt","ConversationHistory","promptDisplay","activeActionReporter","event","replanCount","yamlFlow","replanningCycleLimit","Error","appendFailedPlan","errorMsg","planIndex","errorCountInOnePlanningLoop","outputString","latestPlanResult","latestPlanIndex","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","subGoalStatus","memoriesStatus","planningUiContext","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","errorMessageForAiAct","JSON","actions","thought","log","memory","usage","rawResponse","rawChoiceMessage","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","errorMessage","executables","initialTimeString","taskCountBeforeRun","createAiActActionReporter","String","type","demand","modelRuntime","opt","multimodalPrompt","executionOptions","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","recordAndReleaseScreenshotSequence","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","frames","recorderItems","frame"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AA4EA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAM9C,MAAMC,4BAA4B;AAElC,SAASC,yBAAyBC,QAAgB;IAChD,IAAIA,SAAS,MAAM,IAAIF,2BACrB,OAAOE;IAGT,OAAO,GAAGA,SAAS,KAAK,CAAC,GAAGF,2BAA2B;eAC1C,EAAEE,SAAS,MAAM,GAAGF,0BAA0B,iBAAiB,CAAC;AAC/E;AAKO,MAAMG;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAQC,EACD;QACA,OAAO,IAAIC,iBACTF,OACA,IACEC,SAAS,YACLE,QAAQ,OAAO,CAACF,QAAQ,SAAS,IACjCE,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,kBAAkB,OAAOG,QAAQC;gBAC/B,MAAM,IAAI,CAAC,KAAK,EAAE,mBAAmBD,QAAQC;gBAC7C,MAAMJ,SAAS,mBAAmBG,QAAQC;YAC5C;YACA,GAAIJ,SAAS,cAAc;gBAAE,aAAaA,QAAQ,WAAW;YAAC,IAAI,CAAC,CAAC;QACtE;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IAQA,MAAc,aACZK,KAAa,EACbC,KAAa,EACbC,IAAa,EACE;QACf,MAAM,IAAI,CAAC,KAAK,EAAE,aAAaF,OAAOC,OAAOC;IAC/C;IAOQ,kBACND,KAAyB,EACzBC,IAAuB,EACR;QACf,OAAO,IAAI,CAAC,YAAY,CAACC,oBAAoBF,OAAOC;IACtD;IASQ,0BACNE,mBAAwC,EACxCC,UAAkB,EAClBC,IAAa,EACb;QACAF,oBAAoB,sBAAsB,GAAGE,OACzC,CAAC,MAAM,EAAED,WAAW,EAAE,EAAEC,MAAM,GAC9B,CAAC,cAAc,EAAED,YAAY;IACnC;IAMQ,wBAAwBE,KAAsB,EAAsB;QAC1E,MAAMC,mBAAmBD,MAAM,OAAO,CAAC,CAAC,EAAEE,gBAAgB,EAAE,GAC1DA,mBAAmB;gBAAClB,yBAAyBkB;aAAkB,GAAG,EAAE;QAEtE,OAAOD,iBAAiB,MAAM,GAAG,IAC7BA,iBAAiB,IAAI,CAAC,UACtBE;IACN;IASA,MAAc,cAAcC,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOZ,OAAO;YACdX,QACE,CAAC,gEAAgE,EAAEW,OAAO;QAE9E;aAEAX,QACE;QAKN,OAAOwB,sBAAsBD;IAC/B;IAEA,MAAa,wBACXE,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BpB,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACkB,OAAOC,eAAeC,cAAcpB;IACpE;IAEA,MAAM,uBACJqB,eAA4B,EAC5BC,UAAkB,EAClBC,aAAmC,EACnC;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEF,eAAe,QAAQ,OACvBA,eAAe,UAAUG,mBAAmBL;QAIhD,MAAMM,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLN;gBACA,GAAIE,eAAe,SACf;oBAAE,wBAAwBA,cAAc,MAAM;gBAAC,IAC/C,CAAC,CAAC;YACR;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLR;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMnB,SAASqB,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACG;QAE3B,OAAO;YACLxB;QACF;IACF;IAEA,MAAM,SACJJ,KAAa,EACbmB,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BpB,OAAmC,EACT;QAC1B,MAAMwB,UAAU,IAAI,CAAC,sBAAsB,CAACzB,OAAOC;QACnD,MAAM,EAAEY,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDM,OACAC,eACAC;QAEF,MAAMjB,SAASqB,QAAQ,SAAS;QAChC,MAAMQ,SAAS,MAAMR,QAAQ,YAAY,CAACZ;QAC1C,MAAM,EAAEqB,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACA9B;QACF;IACF;IAEA,MAAM,OACJ+B,UAAuB,EACvBf,aAA2B,EAC3BC,YAA0B,EAC1Be,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EACzBpB,aAAmC,EASnC;QACA,OAAOqB,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAf,eACAC,cACAe,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC,aACApB;IAGN;IAcQ,gCACNpB,MAAkB,EAClB0C,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAI3C,OAAO,KAAK,CAAC,MAAM,EAAE2C,IAAK;YACpD,MAAMnB,OAAOxB,OAAO,KAAK,CAAC2C,EAAE;YAC5B,IACEnB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMoB,SAAUpB,KAAK,KAAK,EAAsC;gBAChE,IAAIoB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZb,UAAuB,EACvBf,aAA2B,EAC3BC,YAA0B,EAC1Be,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EACzBpB,aAAmC,EASnC;QACA,MAAMd,sBAAsB,IAAIuC;QAChC,MAAMC,gBACJ1B,eAAe,UAAUG,mBAAmBQ;QAM9C,IAAIgB;QAIJ,MAAM1B,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAaF,eAAe,QAAQ,OAAO0B,gBAC3C;YACE,aAAa,OAAOE;gBAClB,MAAMD,uBAAuBC;YAC/B;QACF;QAEF,MAAMhD,SAASqB,QAAQ,SAAS;QAEhC,IAAI4B,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJhB,gCAAgC,IAAI,CAAC,oBAAoB;QAC3D,IAAIgB,AAAyBvC,WAAzBuC,sBACF,MAAM,IAAIC,MACR;QAIJ,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS;YACpC,QAAQN;YACR,WAAWK;QACb;QAEA,MAAME,mBAAmB,OACvBC,UACAC;YAKA,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU;gBACrC,GAAIA,YAAY;oBAAEA;gBAAU,IAAI,CAAC,CAAC;gBAClC,WAAWJ;gBACX,OAAOG;YACT;YACA,OAAOjC,QAAQ,eAAe,CAACiC;QACjC;QAEA,IAAIE,8BAA8B;QAClC,IAAIC;QACJ,IAAIC;QACJ,IAAIC,kBAAkB;QAEtB,IAAInB,aAAa,SACf,OAAOa,iBACL,CAAC,cAAc,EAAEb,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMoB,yBAAyB,MAAMC,+BACnCC,6BAA6B/B;QAI/B,MAAO,KAAM;YACX,MAAMwB,YAAYN,cAAc;YAChCU,kBAAkBJ;YAGlB,IAAIf,aAAa,SACf,OAAOa,iBACL,CAAC,cAAc,EAAEb,YAAY,MAAM,IAAI,yBAAyB,EAChEe;YAKJ,MAAMQ,gBAAgBzD,oBAAoB,cAAc,MAAMM;YAG9D,MAAMoD,iBAAiB1D,oBAAoB,cAAc,MAAMM;YAE/D,MAAMiB,SAAS,MAAMR,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBU;oBACjB,GAAIX,eAAe,SACf;wBAAE,wBAAwBA,cAAc,MAAM;oBAAC,IAC/C,CAAC,CAAC;oBACN+B;oBACAlB;oBACAG;oBACAC;oBACA,GAAI0B,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAOvC,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAMsC,oBAAoBtC;oBAC1B,MAAMuC,SAASxC,gBAAgB,IAAI,CAAC,MAAM;oBAC1C,MAAM,IAAI,CAAC,iBAAiB,CAAC,iBAAiB;wBAC5C6B;wBACA,WAAWJ;wBACX,YAAYc,kBAAkB,UAAU;oBAC1C;oBAEA,MAAME,cAAc,IAAI,CAAC,cAAc;oBACvC/E,MACE,sCACA+E,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhDxC,OAAOyC,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJvD,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrCwD;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAAS9C,MAAM,eAAe,EAAE;4BACjD,SAASwC;4BACT,eAAexC,MAAM,YAAY;4BACjC0C;4BACA,cAAcnD;4BACdV;4BACA0B;4BACAI;4BACAC;4BACAuB;4BACApB;wBACF;oBACF,EAAE,OAAOmC,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7ClD,gBAAgB,IAAI,CAAC,KAAK,GAAGmD,gBAC3BF,UAAU,KAAK,EACf;4BAEFjD,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAaiD,UAAU,WAAW;gCAClC,kBAAkBA,UAAU,gBAAgB;4BAC9C;wBACF;wBACA,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1CpB;4BACA,WAAWJ;4BACX,OAAO2B,qBAAqBH;wBAC9B;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBR,QAAQ;oBAC7B;oBACA9E,MAAM,cAAc2F,KAAK,SAAS,CAACN,YAAY,MAAM;oBAErD,MAAM,EACJO,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACNlF,KAAK,EACLmF,KAAK,EACLC,WAAW,EACXC,gBAAgB,EAChBC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGlB;oBACJhB,eAAegC;oBAEf/D,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClC2D;wBACAC;oBACF;oBACA5D,gBAAgB,IAAI,CAAC,KAAK,GAAGmD,gBAAgBO,OAAO;oBACpD1D,gBAAgB,IAAI,CAAC,iBAAiB,GAAG6D;oBACzC7D,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAASsD,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUV,WAAW,QAAQ;wBAC7B,QAAQgB;wBACR,wBAAwBhB,WAAW,sBAAsB;wBACzDiB;wBACAC;oBACF;oBACAjE,gBAAgB,SAAS,GAAGuC;oBAK5B,IAAIiB,OAAOD,SACT,MAAM,IAAI,CAAC,iBAAiB,CAAC,gBAAgB;wBAC3C1B;wBACA,WAAWJ;wBACX,GAAI+B,MAAM;4BAAEA;wBAAI,IAAI,CAAC,CAAC;wBACtB,GAAID,UAAU;4BAAEA;wBAAQ,IAAI,CAAC,CAAC;oBAChC;oBAGF,IAAIhF,OAAO;wBACT,MAAM2F,eAAe,CAAC,oBAAoB,EAAE3F,MAAM,EAAE,EAAEiF,OAAO,IAAI;wBACjE,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1C3B;4BACA,WAAWJ;4BACX,OAAOyC;wBACT;oBACF;oBAEAhE,OAAO,CAAC3B,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEiF,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBAA2B;wBAC7B,MAAMI,eAAe,CAAC,aAAa,EAAEH,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;wBACnG,MAAM,IAAI,CAAC,iBAAiB,CAAC,eAAe;4BAC1C3B;4BACA,WAAWJ;4BACX,OAAOyC;wBACT;wBACAhE,OAAO,OAAOgE;oBAChB;oBAEA,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMnB,aAAa5C,QAAQ;YAC3B6B,mBAAmBe;YAGnB,MAAM1D,QAAQ0D,YAAY,WAAW,EAAE;YACvCvB,SAAS,IAAI,IAAKuB,YAAY,YAAY,EAAE;YAE5C,IAAIoB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9C9E,OACAC,eACAC,cACA;oBACEiB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAOvC,OAAO;gBACd,OAAOoD,iBACL,CAAC,4CAA4C,EAAEyB,qBAAqB7E,OAAO,SAAS,EAAE8E,KAAK,SAAS,CAClGhE,QACC,EACHwC;YAEJ;YACA,IAAIjD,oBAAoB,sBAAsB,EAC5CgE,QAAQ,IAAI,CACV,8FACAhE,oBAAoB,sBAAsB;YAK9C,MAAMwF,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAElD,MAAMC,qBAAqB/F,OAAO,KAAK,CAAC,MAAM;YAI9C+C,uBAAuBiD,0BACrBzC,WACAJ,sBACA,CAAChD,OAAOC,OAAS,IAAI,CAAC,iBAAiB,CAACD,OAAOC;YAEjD,IAAI;gBACF,MAAMiB,QAAQ,YAAY,CAACwE,YAAY,KAAK;gBAC5C,IAAI,CAAC,yBAAyB,CAC5BvF,qBACAwF,mBACA,IAAI,CAAC,uBAAuB,CAAC9F,OAAO,KAAK,CAAC,KAAK,CAAC+F;YAEpD,EAAE,OAAO9F,OAAY;gBAEnBuD;gBACA,MAAMjD,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3C,IAAI,CAAC,yBAAyB,CAC5BD,qBACAC,YACA,CAAC,+BAA+B,EAAEN,OAAO,WAAWgG,OAAOhG,QAAQ;gBAErEb,MACE,yFACAa,iBAAiBmD,QAAQnD,MAAM,OAAO,GAAGgG,OAAOhG,QAChD,6CACAuD;YAEJ,SAAU;gBACRT,uBAAuBnC;YACzB;YAEA,IAAI4C,8BAA8BjE,uCAChC,OAAO8D,iBACL,wCACAE;YAKJ,IAAIf,aAAa,SACf,OAAOa,iBACL,CAAC,cAAc,EAAEb,YAAY,MAAM,IAAI,yBAAyB,EAChEe;YAKJ,IAAI,CAACkB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAACzE,QAAQ+F;YAG7C,EAAE9C;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMG,WAAW,CAAC,UAAU,EAAEH,qBAAqB,4JAA4J,CAAC;gBAChN,OAAOE,iBAAiBC,UAAUC;YACpC;YAEA,IAAI,CAACjD,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CD,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAKA,MAAM,IAAI,CAAC,iBAAiB,CAAC,YAAY;YACvC,WAAWoD;YACX,WAAWR;YACX,GAAKM,gBAAgBC,kBAAkB,SACnC;gBAAE,QAAQD,gBAAgBC,kBAAkB;YAAO,IACnD,CAAC,CAAC;YACN,GAAIA,kBAAkB,MAAM;gBAAE,KAAKA,iBAAiB,GAAG;YAAC,IAAI,CAAC,CAAC;YAC9D,GAAIA,kBAAkB,UAClB;gBAAE,SAASA,iBAAiB,OAAO;YAAC,IACpC,CAAC,CAAC;QACR;QAEA,OAAO;YACL,QAAQ;gBACNR;gBACA,QAAQO;YACV;YACAzD;QACF;IACF;IAEQ,oBACNkG,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAEC,EACD;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASN;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,GAAIA,KAAK,YAAYzF,SAAY;oBAAE,SAASyF,IAAI,OAAO;gBAAC,IAAI,CAAC,CAAC;gBAC9D,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAO1E,OAAOgF;gBACtB,MAAM,EAAEjF,IAAI,EAAE,GAAGiF;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZpF,KAAK,GAAG,GAAG;wBACToF;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACApF,KAAK,KAAK,GAAGqD,gBAAgB+B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjBpF,KAAK,iBAAiB,GAAGoF,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMjF,YAAY8E,YAAY,SAAS;gBACvC7E,OAAOD,WAAW;gBAElB,MAAMkF,mBAAmBX,AAAS,YAATA;gBACzB,IAAIY,cAAcX;gBAClB,IAAIY,cAAc;gBAClB,IAAIF,oBAAqBX,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEa,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF,OAAO,IAAIU,kBAAkB;oBAC3BE,cAAcb;oBACdY,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bd,MAAMC;oBACjD;gBACF;gBAEA,IAAIc;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIb,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DjH,MAAM;oBACN,MAAM+H,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAd,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAV,cACAC,KACAa,sBACAZ,kBACA3E,WACA4E;gBAEJ,EAAE,OAAOtG,OAAO;oBACd,IAAIA,iBAAiBoH,cACnBV,UAAU1G,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBAGRqH,mCAAmC9F,MAAMG;gBAC3C;gBAEA,MAAM,EAAEvB,IAAI,EAAE6E,OAAO,EAAE2B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAenH;gBACnB,IAAIyG,kBAEF,IAAI,AAAgB,YAAhB,OAAOzG,MACTmH,eAAenH;qBACV,IAAI8F,AAAS,cAATA,MAEPqB,eADEnH,QAAAA,OACa,QAECA,IAAY,CAAC2G,YAAY;qBAEtC,IAAI3G,QAAAA,MACTmH,eAAe;qBAGf,IAAInH,MAAM,CAAC2G,YAAY,KAAKnG,QAC1B2G,eAAgBnH,IAAY,CAAC2G,YAAY;qBACpC,IAAI3G,MAAM,WAAWQ,QAC1B2G,eAAgBnH,KAAa,MAAM;qBAEnCwB,OAAO,OAAO;gBAKpB,IAAIsE,AAAS,aAATA,QAAqB,CAACqB,cAAc;oBACtC/F,KAAK,OAAO,GAAGyD;oBACf,MAAM,IAAI7B,MAAM,CAAC,kBAAkB,EAAE6B,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQsC;oBACR,KAAKb;oBACLzB;gBACF;YACF;QACF;QAEA,OAAOuB;IACT;IACA,MAAM,yBACJN,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpCC,gBAGC,EAC4B;QAC7B,MAAMlF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACE4E,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB,UAEvDI,kBAAkB,YACd;YAAE,WAAWA,iBAAiB,SAAS;QAAC,IACxC3F;QAGN,MAAM4F,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CN,MACAC,QACAC,cACAC,KACAC,kBACAC;QAGF,MAAMvG,SAASqB,QAAQ,SAAS;QAChC,MAAMQ,SAAS,MAAMR,QAAQ,YAAY,CAACmF;QAE1C,IAAI,CAAC3E,QACH,MAAM,IAAIuB,MACR;QAIJ,MAAM,EAAEtB,MAAM,EAAEmD,OAAO,EAAE,GAAGpD;QAE5B,OAAO;YACLC;YACAmD;YACAjF;QACF;IACF;IAEA,MAAM,QACJwH,SAAsB,EACtBnB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEqB,UAAU,EAAEnB,gBAAgB,EAAE,GAAGoB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMpG,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAWqG;QAE1B,MAAM3H,SAASqB,QAAQ,SAAS;QAChC,MAAM,EACJuG,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG3B;QACJ,MAAM4B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEApG,OAAO4F,WAAW;QAClB5F,OAAOgG,WAAW;QAClBhG,OAAOiG,iBAAiB;QAExBjG,OACEiG,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACArB,cACA6B,mBACA3B;YAGF,MAAMzE,SAAU,MAAMR,QAAQ,YAAY,CAACmF;YAO3C,IAAI3E,QAAQ,QACV,OAAO;gBACL,QAAQjB;gBACRZ;YACF;YAGFqI,eACExG,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAE4F,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMvD,UAAU,CAAC,kBAAkB,EAAE4C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGxD;oBAAQ;iBAAE,EAC9DmB,cACAA;gBAEF,IAAIsC,UAAU,CAAC,EAAE,EACf,MAAMrH,QAAQ,YAAY,CAACqH,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAOrH,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEgH,cAAc;IACnE;IA/9BA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAm8BF;AAkBO,SAASvB,mCACd9F,IAAmB,EACnBG,SAAgC;IAEhC,MAAMoH,SAASpH,WAAW;IAC1B,IAAIoH,UAAUA,OAAO,MAAM,GAAG,GAAG;QAC/B,MAAMC,gBAAyC,EAAE;QACjD,IAAK,IAAIrG,IAAI,GAAGA,IAAIoG,OAAO,MAAM,GAAG,GAAGpG,IAAK;YAC1C,MAAMsG,QAAQF,MAAM,CAACpG,EAAE;YACvBqG,cAAc,IAAI,CAAC;gBACjB,MAAM;gBACN,IAAIC,MAAM,UAAU;gBACpB,YAAYA;gBACZ,aAAa,CAAC,eAAe,EAAEtG,IAAI,EAAE,CAAC,EAAEoG,OAAO,MAAM,EAAE;gBACvD,QAAQ;YACV;QACF;QAKAvH,KAAK,QAAQ,GAAGA,KAAK,QAAQ,GACzB;eAAIwH;eAAkBxH,KAAK,QAAQ;SAAC,GACpCwH;IACN;IACA,IAAIrH,WAAW,oBACbA,UAAU,kBAAkB,GAAGf;AAEnC"}

package/dist/es/agent/ui-utils.mjs

@@ -6,6 +6,7 @@ function locateParamStr(locate) {
     if (!locate) return '';
     if ('string' == typeof locate) return locate;
     if ('object' == typeof locate) {
+        if ('string' == typeof locate.promptDisplay) return locate.promptDisplay;
         if ('object' == typeof locate.prompt && null !== locate.prompt && locate.prompt.prompt) {
             const prompt = locate.prompt.prompt;
             return prompt;

package/dist/es/agent/ui-utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.mjs","sources":["../../../src/agent/ui-utils.ts"],"sourcesContent":["import type { TUserPrompt } from '@/common';\nimport { userPromptToString } from '@/common';\nimport type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport type TaskTitleType =\n  | 'Tap'\n  | 'Hover'\n  | 'Input'\n  | 'RightClick'\n  | 'KeyboardPress'\n  | 'Scroll'\n  | 'Act'\n  | 'Query'\n  | 'Assert'\n  | 'WaitFor'\n  | 'Locate'\n  | 'Markdown'\n  | 'Boolean'\n  | 'Number'\n  | 'String';\n\nexport function taskTitleStr(type: TaskTitleType, prompt: string) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      // Prefer AI-generated output.log over user input\n      const planTask = task as ExecutionTaskPlanning;\n      value =\n        planTask.output?.log ||\n        planTask.param?.userInstructionDisplay ||\n        (planTask.param?.userInstruction\n          ? userPromptToString(planTask.param.userInstruction as TUserPrompt)\n          : undefined);\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'PullGesture'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    // Flatten `{key: \"raw value\"}` into `key: raw value` instead of emitting\n    // a pretty-printed JSON string. JSON.stringify would escape every inner\n    // quote as `\\\"`, and the UI renders the result as plain text — so a\n    // command like `grep -E \"version\"` ends up shown with the literal\n    // backslashes, which is both noisy and confusing for users.\n    const entries = Object.entries(value as Record<string, unknown>);\n    if (entries.length === 0) {\n      return '';\n    }\n    const formatValue = (v: unknown): string => {\n      if (typeof v === 'string') return v;\n      if (v === null || v === undefined) return String(v);\n      if (typeof v === 'object') return JSON.stringify(v);\n      return String(v);\n    };\n    if (entries.length === 1) {\n      const [key, v] = entries[0];\n      return `${key}: ${formatValue(v)}`;\n    }\n    return entries.map(([key, v]) => `${key}: ${formatValue(v)}`).join(', ');\n  }\n\n  return String(value);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","planTask","userPromptToString","locateStr","Object","entries","formatValue","v","String","key"],"mappings":";AAcO,SAASA,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAE9B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAmBO,SAASC,aAAaC,IAAmB,EAAEjB,MAAc;IAC9D,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,MAAsC;SACzD;QAEL,MAAMsB,WAAWtB;QACjBgB,QACEM,SAAS,MAAM,EAAE,OACjBA,SAAS,KAAK,EAAE,0BACfA,CAAAA,SAAS,KAAK,EAAE,kBACbC,mBAAmBD,SAAS,KAAK,CAAC,eAAe,IACjDR,MAAQ;IAChB;IAGF,IAAId,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCyB,OAAO,IAAI,CAAEzB,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIwB,WAEAR,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMQ,YAAYvB,eAAee;QACjC,IAAIQ,WACF,OAAOA;QAOT,MAAME,UAAUD,OAAO,OAAO,CAACT;QAC/B,IAAIU,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAO;QAET,MAAMC,cAAc,CAACC;YACnB,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOA;YAClC,IAAIA,QAAAA,GAA+B,OAAOC,OAAOD;YACjD,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOX,KAAK,SAAS,CAACW;YACjD,OAAOC,OAAOD;QAChB;QACA,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;YACxB,MAAM,CAACI,KAAKF,EAAE,GAAGF,OAAO,CAAC,EAAE;YAC3B,OAAO,GAAGI,IAAI,EAAE,EAAEH,YAAYC,IAAI;QACpC;QACA,OAAOF,QAAQ,GAAG,CAAC,CAAC,CAACI,KAAKF,EAAE,GAAK,GAAGE,IAAI,EAAE,EAAEH,YAAYC,IAAI,EAAE,IAAI,CAAC;IACrE;IAEA,OAAOC,OAAOb;AAChB"}
+{"version":3,"file":"agent/ui-utils.mjs","sources":["../../../src/agent/ui-utils.ts"],"sourcesContent":["import type { TUserPrompt } from '@/common';\nimport { userPromptToString } from '@/common';\nimport type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.promptDisplay === 'string') {\n      return locate.promptDisplay;\n    }\n\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport type TaskTitleType =\n  | 'Tap'\n  | 'Hover'\n  | 'Input'\n  | 'RightClick'\n  | 'KeyboardPress'\n  | 'Scroll'\n  | 'Act'\n  | 'Query'\n  | 'Assert'\n  | 'WaitFor'\n  | 'Locate'\n  | 'Markdown'\n  | 'Boolean'\n  | 'Number'\n  | 'String';\n\nexport function taskTitleStr(type: TaskTitleType, prompt: string) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      // Prefer AI-generated output.log over user input\n      const planTask = task as ExecutionTaskPlanning;\n      value =\n        planTask.output?.log ||\n        planTask.param?.userInstructionDisplay ||\n        (planTask.param?.userInstruction\n          ? userPromptToString(planTask.param.userInstruction as TUserPrompt)\n          : undefined);\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'PullGesture'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    // Flatten `{key: \"raw value\"}` into `key: raw value` instead of emitting\n    // a pretty-printed JSON string. JSON.stringify would escape every inner\n    // quote as `\\\"`, and the UI renders the result as plain text — so a\n    // command like `grep -E \"version\"` ends up shown with the literal\n    // backslashes, which is both noisy and confusing for users.\n    const entries = Object.entries(value as Record<string, unknown>);\n    if (entries.length === 0) {\n      return '';\n    }\n    const formatValue = (v: unknown): string => {\n      if (typeof v === 'string') return v;\n      if (v === null || v === undefined) return String(v);\n      if (typeof v === 'object') return JSON.stringify(v);\n      return String(v);\n    };\n    if (entries.length === 1) {\n      const [key, v] = entries[0];\n      return `${key}: ${formatValue(v)}`;\n    }\n    return entries.map(([key, v]) => `${key}: ${formatValue(v)}`).join(', ');\n  }\n\n  return String(value);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","planTask","userPromptToString","locateStr","Object","entries","formatValue","v","String","key"],"mappings":";AAcO,SAASA,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAgC,YAAhC,OAAOA,OAAO,aAAa,EAC7B,OAAOA,OAAO,aAAa;QAI7B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAmBO,SAASC,aAAaC,IAAmB,EAAEjB,MAAc;IAC9D,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,MAAsC;SACzD;QAEL,MAAMsB,WAAWtB;QACjBgB,QACEM,SAAS,MAAM,EAAE,OACjBA,SAAS,KAAK,EAAE,0BACfA,CAAAA,SAAS,KAAK,EAAE,kBACbC,mBAAmBD,SAAS,KAAK,CAAC,eAAe,IACjDR,MAAQ;IAChB;IAGF,IAAId,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCyB,OAAO,IAAI,CAAEzB,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIwB,WAEAR,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMQ,YAAYvB,eAAee;QACjC,IAAIQ,WACF,OAAOA;QAOT,MAAME,UAAUD,OAAO,OAAO,CAACT;QAC/B,IAAIU,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAO;QAET,MAAMC,cAAc,CAACC;YACnB,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOA;YAClC,IAAIA,QAAAA,GAA+B,OAAOC,OAAOD;YACjD,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOX,KAAK,SAAS,CAACW;YACjD,OAAOC,OAAOD;QAChB;QACA,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;YACxB,MAAM,CAACI,KAAKF,EAAE,GAAGF,OAAO,CAAC,EAAE;YAC3B,OAAO,GAAGI,IAAI,EAAE,EAAEH,YAAYC,IAAI;QACpC;QACA,OAAOF,QAAQ,GAAG,CAAC,CAAC,CAACI,KAAKF,EAAE,GAAK,GAAGE,IAAI,EAAE,EAAEH,YAAYC,IAAI,EAAE,IAAI,CAAC;IACrE;IAEA,OAAOC,OAAOb;AAChB"}

package/dist/es/agent/utils.mjs

@@ -181,7 +181,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.10.8-beta-20260724035136.0";
+const getMidsceneVersion = ()=>"1.10.8";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/inspect.mjs

@@ -346,7 +346,7 @@ async function AiExtractElementInfo(options) {
         referenceImagesIncluded: !!multimodalPrompt?.images?.length
     });
     const screenshotBase64 = context.screenshot.base64;
-    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery);
+    const extractDataPromptText = extractDataQueryPrompt(options.pageDescription || '', dataQuery, extractOption?.context);
     const userContent = [];
     if (extractOption?.screenshotIncluded !== false) {
         const screenshotSequence = context.screenshotSequence;

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\ntype SectionLocateObjectResponse = Awaited<\n  ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n>;\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const referenceImageMessages =\n    typeof targetElementDescription === 'string'\n      ? undefined\n      : await multimodalPromptToChatMessages(\n          userPromptToMultimodalPrompt(targetElementDescription),\n        );\n  const locateRequest: LocateRequestContext = {\n    elementDescriptionText: userPromptToString(targetElementDescription),\n    locateImage,\n    referenceImageMessages,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(\n    targetElementDescription,\n    locateOptions,\n    locateRequest,\n  );\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugInspect('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          locateRequest.elementDescriptionText,\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  _elementDescription: TUserPrompt,\n  options: LocateOptions,\n  locateRequest: LocateRequestContext,\n): Promise<LocateModelResponse> {\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const userInstructionPrompt = findElementPrompt(\n    locateRequest.elementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateRequest.locateImage.imageBase64,\n    width: locateRequest.locateImage.width,\n    height: locateRequest.locateImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (locateRequest.referenceImageMessages) {\n    msgs.push(...locateRequest.referenceImageMessages);\n  }\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AIElementLocateResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'locate',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (response): LocateModelResponse => {\n        const rawResponse = response.contentString;\n        const errors: string[] | undefined =\n          'errors' in response.content ? response.content.errors : [];\n        if (\n          !hasLocateResult(response.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: errors as string[],\n          };\n        }\n\n        const locatedPixelBbox =\n          resultAdapter.adaptElementLocateResultToPixelBbox(response.content, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n        return {\n          locatedPixelBbox,\n          rawResponse,\n          rawChoiceMessage: response.rawChoiceMessage,\n          usage: response.usage,\n          reasoningContent: response.reasoning_content,\n          errors: errors as string[],\n        };\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        const modelErrors =\n          'errors' in response.content ? response.content.errors : undefined;\n        const message =\n          modelErrors && modelErrors.length > 0\n            ? `${modelErrors.join('\\n')} (${parseErrorMessage})`\n            : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          response.contentString,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugInspect(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let parsedResult:\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AISectionLocatorResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'section-locator',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (result) => {\n        const sectionError = result.content.error;\n        if (\n          !hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return { result, sectionError };\n        }\n\n        const adaptedResult =\n          resultAdapter.adaptSectionLocateResultToPixelBboxGroup(\n            result.content,\n            {\n              preparedSize: preparedImage.preparedSize,\n              contentSize: preparedImage.contentSize,\n            },\n          );\n        const mergedRect = mergePixelBboxesToRect([\n          adaptedResult.target,\n          ...(adaptedResult.references ?? []),\n        ]);\n        debugSection('mergedRect %j', mergedRect);\n        return { result, sectionError, mergedRect };\n      },\n      toParseError: (error, result) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        const message = result.content.error\n          ? `${result.content.error} (${parseErrorMessage})`\n          : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          result.contentString,\n          result.usage,\n          result.rawChoiceMessage,\n          result.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(msgs, modelRuntime, {\n        abortSignal: options.abortSignal,\n      }),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseXMLExtractionResponse<T>(rawResponse);\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInspect(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","assert","locateImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","locateRequest","userPromptToString","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","error","msg","Error","_elementDescription","modelRuntime","adapter","resultAdapter","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","preparedImage","prepareModelImage","imagePayload","msgs","callAiAndParseWithRetry","callAIWithObjectResponse","response","parseErrorMessage","modelErrors","message","AIResponseParseError","String","callError","errorMessage","AiLocateSection","sectionDescription","screenshotBase64","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","addOns","parsedResult","result","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","searchAreaConfig","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","callAI","reasoning_content","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;;AAkEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAMO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvDa,OACEF,0BACA;IAGF,MAAM,EAAEV,OAAO,EAAE,GAAGW;IACpB,MAAME,cAAcF,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaX,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMc,yBACJ,AAAoC,YAApC,OAAOJ,2BACHb,SACA,MAAMkB,+BACJC,6BAA6BN;IAErC,MAAMO,gBAAsC;QAC1C,wBAAwBC,mBAAmBR;QAC3CG;QACAC;QACA,SAASH;IACX;IAEA,MAAMQ,gBAAgBpB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMqB,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAC3BV,0BACAC,eACAM;IAEF,MAAM,EACJM,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,SAASA;YACT+B;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAMC,OAAOC,gBACXC,0CACET,kBACAZ,cAAc,YAAY,EAAE;QAGhCvB,aAAa,WAAW0C;QAExB,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,sBACPH,MACAb,cAAc,sBAAsB;gBAEtC,QAAQ,EAAE;YACZ;YACA,GAAGY,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMrC;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ+B,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBgB,mBAAgC,EAChCtC,OAAsB,EACtBkB,aAAmC;IAEnC,MAAMqB,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAME,wBAAwBC,kBAC5BzB,cAAc,sBAAsB;IAEtC,MAAM0B,eAAeC,4BACnBL,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMM,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAa7B,cAAc,WAAW,CAAC,WAAW;QAClD,OAAOA,cAAc,WAAW,CAAC,KAAK;QACtC,QAAQA,cAAc,WAAW,CAAC,MAAM;QACxC,QAAQsB,QAAQ,eAAe;IACjC;IAEA,MAAMQ,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMN;gBACR;aACD;QACH;KACD;IAED,IAAIxB,cAAc,sBAAsB,EACtC+B,KAAK,IAAI,IAAI/B,cAAc,sBAAsB;IAGnD,IAAI;QACF,OAAO,MAAMgC,wBAAwB;YACnC,QAAQ,IACNC,yBAAkDF,MAAMV,cAAc;oBACpE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAACa;gBACd,MAAM3B,cAAc2B,SAAS,aAAa;gBAC1C,MAAMvB,SACJ,YAAYuB,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAG,EAAE;gBAC7D,IACE,CAAC5D,gBAAgB4D,SAAS,OAAO,EAAEX,cAAc,UAAU,CAAC,SAAS,GAErE,OAAO;oBACLhB;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;gBAGF,MAAML,mBACJiB,cAAc,mCAAmC,CAACW,SAAS,OAAO,EAAE;oBAClE,cAAcN,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBACF,OAAO;oBACLtB;oBACAC;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;YACF;YACA,cAAc,CAACM,OAAOiB;gBACpB,MAAMC,oBACJlB,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,MAAMmB,cACJ,YAAYF,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAGtD;gBAC3D,MAAMyD,UACJD,eAAeA,YAAY,MAAM,GAAG,IAChC,GAAGA,YAAY,IAAI,CAAC,MAAM,EAAE,EAAED,kBAAkB,CAAC,CAAC,GAClDA;gBACN,OAAO,IAAIG,qBACTD,SACAH,SAAS,aAAa,EACtBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb9C,aACE,uDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,sBACvB,OAAO;YACL,aAAaE,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,aAAaC;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF;AAEO,eAAeC,gBAAgB5D,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAE4D,kBAAkB,EAAE,GAAG7D;IACxC,MAAMuC,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAMsB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM6C,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAae;QACb,OAAO7D,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQuC,QAAQ,eAAe;IACjC;IAEA,MAAMI,eAAemB,4BACnBvB,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMwB,gCAAgCC,0BACpC9C,mBAAmB0C;IAErB,MAAMZ,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMkB;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOH,oBAAiC;QAC1C,MAAMK,SAAS,MAAMlD,+BACnBC,6BAA6B4C;QAE/BZ,KAAK,IAAI,IAAIiB;IACf;IAEA,IAAIC;IAYJ,IAAI;QACFA,eAAe,MAAMjB,wBAAwB;YAC3C,QAAQ,IACNC,yBAAmDF,MAAMV,cAAc;oBACrE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAAC6B;gBACd,MAAMC,eAAeD,OAAO,OAAO,CAAC,KAAK;gBACzC,IACE,CAAC5E,gBAAgB4E,OAAO,OAAO,EAAE3B,cAAc,UAAU,CAAC,SAAS,GAEnE,OAAO;oBAAE2B;oBAAQC;gBAAa;gBAGhC,MAAMC,gBACJ7B,cAAc,wCAAwC,CACpD2B,OAAO,OAAO,EACd;oBACE,cAActB,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBAEJ,MAAMyB,aAAaC,uBAAuB;oBACxCF,cAAc,MAAM;uBAChBA,cAAc,UAAU,IAAI,EAAE;iBACnC;gBACD/E,aAAa,iBAAiBgF;gBAC9B,OAAO;oBAAEH;oBAAQC;oBAAcE;gBAAW;YAC5C;YACA,cAAc,CAACpC,OAAOiC;gBACpB,MAAMf,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,MAAMoB,UAAUa,OAAO,OAAO,CAAC,KAAK,GAChC,GAAGA,OAAO,OAAO,CAAC,KAAK,CAAC,EAAE,EAAEf,kBAAkB,CAAC,CAAC,GAChDA;gBACJ,OAAO,IAAIG,qBACTD,SACAa,OAAO,aAAa,EACpBA,OAAO,KAAK,EACZA,OAAO,gBAAgB,EACvBA,OAAO,iBAAiB;YAE5B;YACA,iBAAiB7B,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb5C,aACE,+DACA4C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,sBACvB,OAAO;YACL,kBAAkB1D;YAClB,OAAO4D,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,kBAAkB5D;YAClB,OAAO,CAAC,eAAe,EAAE6D,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAES,MAAM,EAAEC,YAAY,EAAEE,UAAU,EAAE,GAAGJ;IAC7C,IAAI,CAACI,YACH,OAAO;QACL,kBAAkBzE;QAClB,OAAOuE;QACP,aAAaD,OAAO,aAAa;QACjC,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMK,eAAepE,iBAAiBkE,YAAYtE,QAAQ,QAAQ;QAClE,MAAMyE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1ClF,aAAa,2BAA2BkF;QAExC,MAAMG,mBAAmB,MAAM7E,sBAAsB;YACnDE;YACA,UAAUsE;QACZ;QAEAhF,aACE,uDACAmF,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOP;YACP,aAAaD,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOjC,OAAO;QACd,MAAMkB,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMwB,eAAeU,eACjB,GAAGA,aAAa,EAAE,EAAEhB,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBvD;YAClB,OAAO6D;YACP,aAAaS,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF;AAEO,eAAeS,qBAAwB7E,OAQ7C;IACC,MAAM,EAAE8E,SAAS,EAAE7E,OAAO,EAAE8E,aAAa,EAAEC,gBAAgB,EAAEzC,YAAY,EAAE,GACzEvC;IACF,MAAM4C,eAAeqC,sBAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMlB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMiF,wBAAwBC,uBAC5BnF,QAAQ,eAAe,IAAI,IAC3B8E;IAGF,MAAMM,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/C,MAAMM,qBAAqBpF,QAAQ,kBAAkB;QACrD,IAAIoF,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKtB;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAsB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMjC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASwC;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAMd,SAAS,MAAMlD,+BAA+BgE;QACpD/B,KAAK,IAAI,IAAIiB;IACf;IAEA,OAAOhB,wBAAwB;QAC7B,QAAQ,IACNsC,OAAOvC,MAAMV,cAAc;gBACzB,aAAavC,QAAQ,WAAW;YAClC;QACF,eAAe,CAACoD;YACd,MAAM,EACJ,SAAS3B,WAAW,EACpBE,KAAK,EACL8D,iBAAiB,EACjB/D,gBAAgB,EACjB,GAAG0B;YACJ,MAAMsC,cAAcC,2BAA8BlE;YAClD,OAAO;gBACLiE;gBACAjE;gBACAC;gBACAC;gBACA8D;YACF;QACF;QACA,cAAc,CAACG,YAAYxC;YACzB,MAAMO,eACJiC,sBAAsBvD,QAAQuD,WAAW,OAAO,GAAGnC,OAAOmC;YAC5D,OAAO,IAAIpC,qBACT,CAAC,iBAAiB,EAAEG,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAavC,QAAQ,WAAW;QAChC,cAAc,CAACmC;YACb9C,aACE,iDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;QAEpD;IACF;AACF;AAEO,eAAe0D,sBACpBC,WAAmB,EACnBvD,YAA0B;IAK1B,MAAMK,eAAemD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAM7C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASoD;QACX;KACD;IAED3G,aAAa,yCAAyCyG;IAEtD,MAAM1B,SAAS,MAAMjB,yBACnBF,MACAV,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB6B,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { callAiAndParseWithRetry } from './service-caller/semantic-retry';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\ntype SectionLocateObjectResponse = Awaited<\n  ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n>;\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const referenceImageMessages =\n    typeof targetElementDescription === 'string'\n      ? undefined\n      : await multimodalPromptToChatMessages(\n          userPromptToMultimodalPrompt(targetElementDescription),\n        );\n  const locateRequest: LocateRequestContext = {\n    elementDescriptionText: userPromptToString(targetElementDescription),\n    locateImage,\n    referenceImageMessages,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(\n    targetElementDescription,\n    locateOptions,\n    locateRequest,\n  );\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugInspect('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          locateRequest.elementDescriptionText,\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  _elementDescription: TUserPrompt,\n  options: LocateOptions,\n  locateRequest: LocateRequestContext,\n): Promise<LocateModelResponse> {\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const userInstructionPrompt = findElementPrompt(\n    locateRequest.elementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateRequest.locateImage.imageBase64,\n    width: locateRequest.locateImage.width,\n    height: locateRequest.locateImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (locateRequest.referenceImageMessages) {\n    msgs.push(...locateRequest.referenceImageMessages);\n  }\n\n  try {\n    return await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AIElementLocateResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'locate',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (response): LocateModelResponse => {\n        const rawResponse = response.contentString;\n        const errors: string[] | undefined =\n          'errors' in response.content ? response.content.errors : [];\n        if (\n          !hasLocateResult(response.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return {\n            rawResponse,\n            rawChoiceMessage: response.rawChoiceMessage,\n            usage: response.usage,\n            reasoningContent: response.reasoning_content,\n            errors: errors as string[],\n          };\n        }\n\n        const locatedPixelBbox =\n          resultAdapter.adaptElementLocateResultToPixelBbox(response.content, {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          });\n        return {\n          locatedPixelBbox,\n          rawResponse,\n          rawChoiceMessage: response.rawChoiceMessage,\n          usage: response.usage,\n          reasoningContent: response.reasoning_content,\n          errors: errors as string[],\n        };\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse locate result: ${error.message}`\n            : 'unknown error in locate result';\n        const modelErrors =\n          'errors' in response.content ? response.content.errors : undefined;\n        const message =\n          modelErrors && modelErrors.length > 0\n            ? `${modelErrors.join('\\n')} (${parseErrorMessage})`\n            : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          response.contentString,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugInspect(\n          'retrying locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n        reasoningContent: callError.reasoningContent,\n        errors: [callError.message],\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      rawResponse: errorMessage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const resultAdapter = adapter.locate.resultAdapter;\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let parsedResult:\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateObjectResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: () =>\n        callAIWithObjectResponse<AISectionLocatorResponse>(msgs, modelRuntime, {\n          abortSignal: options.abortSignal,\n          jsonParserSource: 'section-locator',\n          retryTimes: modelRuntime.config.retryCount,\n          retryInterval: modelRuntime.config.retryInterval,\n        }),\n      parseResponse: (result) => {\n        const sectionError = result.content.error;\n        if (\n          !hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)\n        ) {\n          return { result, sectionError };\n        }\n\n        const adaptedResult =\n          resultAdapter.adaptSectionLocateResultToPixelBboxGroup(\n            result.content,\n            {\n              preparedSize: preparedImage.preparedSize,\n              contentSize: preparedImage.contentSize,\n            },\n          );\n        const mergedRect = mergePixelBboxesToRect([\n          adaptedResult.target,\n          ...(adaptedResult.references ?? []),\n        ]);\n        debugSection('mergedRect %j', mergedRect);\n        return { result, sectionError, mergedRect };\n      },\n      toParseError: (error, result) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        const message = result.content.error\n          ? `${result.content.error} (${parseErrorMessage})`\n          : parseErrorMessage;\n        return new AIResponseParseError(\n          message,\n          result.contentString,\n          result.usage,\n          result.rawChoiceMessage,\n          result.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n    extractOption?.context,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    const screenshotSequence = context.screenshotSequence;\n    if (screenshotSequence && screenshotSequence.length > 1) {\n      userContent.push({\n        type: 'text',\n        text: `The following ${screenshotSequence.length} images are consecutive screenshots captured over a time window, ordered from earliest to latest (Frame 1 is first, Frame ${screenshotSequence.length} is last). They record what appeared on screen during that window. Some UI elements such as toasts, banners, or transitions may appear only in certain frames and be gone by later ones. Interpret the temporal scope from the statement or question itself: if it asks whether something appeared at any point, inspect the whole sequence; if it asks about the final or current state, use the relevant later frame; if it asks about a change or sequence, compare frames in order. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on these screenshots and their contents.`,\n      });\n\n      screenshotSequence.forEach((frame, index) => {\n        userContent.push({\n          type: 'text',\n          text: `Frame ${index + 1}/${screenshotSequence.length}`,\n        });\n        userContent.push({\n          type: 'image_url',\n          image_url: {\n            url: frame.base64,\n            detail: 'high',\n          },\n        });\n      });\n    } else {\n      userContent.push({\n        type: 'text',\n        text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n      });\n\n      userContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshotBase64,\n          detail: 'high',\n        },\n      });\n    }\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  return callAiAndParseWithRetry({\n    callAi: () =>\n      callAI(msgs, modelRuntime, {\n        abortSignal: options.abortSignal,\n      }),\n    parseResponse: (response) => {\n      const {\n        content: rawResponse,\n        usage,\n        reasoning_content,\n        rawChoiceMessage,\n      } = response;\n      const parseResult = parseXMLExtractionResponse<T>(rawResponse);\n      return {\n        parseResult,\n        rawResponse,\n        rawChoiceMessage,\n        usage,\n        reasoning_content,\n      };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal: options.abortSignal,\n    onParseRetry: (error) => {\n      debugInspect(\n        'retrying insight after XML parsing failed: %s',\n        error instanceof Error ? error.message : String(error),\n      );\n    },\n  });\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","assert","locateImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","locateRequest","userPromptToString","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","error","msg","Error","_elementDescription","modelRuntime","adapter","resultAdapter","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","preparedImage","prepareModelImage","imagePayload","msgs","callAiAndParseWithRetry","callAIWithObjectResponse","response","parseErrorMessage","modelErrors","message","AIResponseParseError","String","callError","errorMessage","AiLocateSection","sectionDescription","screenshotBase64","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","addOns","parsedResult","result","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","searchAreaConfig","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","screenshotSequence","frame","index","callAI","reasoning_content","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;;AAkEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAMO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvDa,OACEF,0BACA;IAGF,MAAM,EAAEV,OAAO,EAAE,GAAGW;IACpB,MAAME,cAAcF,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaX,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMc,yBACJ,AAAoC,YAApC,OAAOJ,2BACHb,SACA,MAAMkB,+BACJC,6BAA6BN;IAErC,MAAMO,gBAAsC;QAC1C,wBAAwBC,mBAAmBR;QAC3CG;QACAC;QACA,SAASH;IACX;IAEA,MAAMQ,gBAAgBpB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMqB,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAC3BV,0BACAC,eACAM;IAEF,MAAM,EACJM,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,SAASA;YACT+B;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAMC,OAAOC,gBACXC,0CACET,kBACAZ,cAAc,YAAY,EAAE;QAGhCvB,aAAa,WAAW0C;QAExB,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,sBACPH,MACAb,cAAc,sBAAsB;gBAEtC,QAAQ,EAAE;YACZ;YACA,GAAGY,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMrC;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ+B,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBgB,mBAAgC,EAChCtC,OAAsB,EACtBkB,aAAmC;IAEnC,MAAMqB,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAME,wBAAwBC,kBAC5BzB,cAAc,sBAAsB;IAEtC,MAAM0B,eAAeC,4BACnBL,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMM,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAa7B,cAAc,WAAW,CAAC,WAAW;QAClD,OAAOA,cAAc,WAAW,CAAC,KAAK;QACtC,QAAQA,cAAc,WAAW,CAAC,MAAM;QACxC,QAAQsB,QAAQ,eAAe;IACjC;IAEA,MAAMQ,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMN;gBACR;aACD;QACH;KACD;IAED,IAAIxB,cAAc,sBAAsB,EACtC+B,KAAK,IAAI,IAAI/B,cAAc,sBAAsB;IAGnD,IAAI;QACF,OAAO,MAAMgC,wBAAwB;YACnC,QAAQ,IACNC,yBAAkDF,MAAMV,cAAc;oBACpE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAACa;gBACd,MAAM3B,cAAc2B,SAAS,aAAa;gBAC1C,MAAMvB,SACJ,YAAYuB,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAG,EAAE;gBAC7D,IACE,CAAC5D,gBAAgB4D,SAAS,OAAO,EAAEX,cAAc,UAAU,CAAC,SAAS,GAErE,OAAO;oBACLhB;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;gBAGF,MAAML,mBACJiB,cAAc,mCAAmC,CAACW,SAAS,OAAO,EAAE;oBAClE,cAAcN,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBACF,OAAO;oBACLtB;oBACAC;oBACA,kBAAkB2B,SAAS,gBAAgB;oBAC3C,OAAOA,SAAS,KAAK;oBACrB,kBAAkBA,SAAS,iBAAiB;oBAC5C,QAAQvB;gBACV;YACF;YACA,cAAc,CAACM,OAAOiB;gBACpB,MAAMC,oBACJlB,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;gBACN,MAAMmB,cACJ,YAAYF,SAAS,OAAO,GAAGA,SAAS,OAAO,CAAC,MAAM,GAAGtD;gBAC3D,MAAMyD,UACJD,eAAeA,YAAY,MAAM,GAAG,IAChC,GAAGA,YAAY,IAAI,CAAC,MAAM,EAAE,EAAED,kBAAkB,CAAC,CAAC,GAClDA;gBACN,OAAO,IAAIG,qBACTD,SACAH,SAAS,aAAa,EACtBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb9C,aACE,uDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,sBACvB,OAAO;YACL,aAAaE,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;YACtB,kBAAkBA,UAAU,gBAAgB;YAC5C,QAAQ;gBAACA,UAAU,OAAO;aAAC;QAC7B;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,aAAaC;YACb,QAAQ;gBAAC,CAAC,eAAe,EAAEA,cAAc;aAAC;QAC5C;IACF;AACF;AAEO,eAAeC,gBAAgB5D,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAE4D,kBAAkB,EAAE,GAAG7D;IACxC,MAAMuC,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,gBAAgBD,QAAQ,MAAM,CAAC,aAAa;IAClD,MAAMsB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM6C,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAae;QACb,OAAO7D,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQuC,QAAQ,eAAe;IACjC;IAEA,MAAMI,eAAemB,4BACnBvB,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMwB,gCAAgCC,0BACpC9C,mBAAmB0C;IAErB,MAAMZ,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMkB;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOH,oBAAiC;QAC1C,MAAMK,SAAS,MAAMlD,+BACnBC,6BAA6B4C;QAE/BZ,KAAK,IAAI,IAAIiB;IACf;IAEA,IAAIC;IAYJ,IAAI;QACFA,eAAe,MAAMjB,wBAAwB;YAC3C,QAAQ,IACNC,yBAAmDF,MAAMV,cAAc;oBACrE,aAAavC,QAAQ,WAAW;oBAChC,kBAAkB;oBAClB,YAAYuC,aAAa,MAAM,CAAC,UAAU;oBAC1C,eAAeA,aAAa,MAAM,CAAC,aAAa;gBAClD;YACF,eAAe,CAAC6B;gBACd,MAAMC,eAAeD,OAAO,OAAO,CAAC,KAAK;gBACzC,IACE,CAAC5E,gBAAgB4E,OAAO,OAAO,EAAE3B,cAAc,UAAU,CAAC,SAAS,GAEnE,OAAO;oBAAE2B;oBAAQC;gBAAa;gBAGhC,MAAMC,gBACJ7B,cAAc,wCAAwC,CACpD2B,OAAO,OAAO,EACd;oBACE,cAActB,cAAc,YAAY;oBACxC,aAAaA,cAAc,WAAW;gBACxC;gBAEJ,MAAMyB,aAAaC,uBAAuB;oBACxCF,cAAc,MAAM;uBAChBA,cAAc,UAAU,IAAI,EAAE;iBACnC;gBACD/E,aAAa,iBAAiBgF;gBAC9B,OAAO;oBAAEH;oBAAQC;oBAAcE;gBAAW;YAC5C;YACA,cAAc,CAACpC,OAAOiC;gBACpB,MAAMf,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,MAAMoB,UAAUa,OAAO,OAAO,CAAC,KAAK,GAChC,GAAGA,OAAO,OAAO,CAAC,KAAK,CAAC,EAAE,EAAEf,kBAAkB,CAAC,CAAC,GAChDA;gBACJ,OAAO,IAAIG,qBACTD,SACAa,OAAO,aAAa,EACpBA,OAAO,KAAK,EACZA,OAAO,gBAAgB,EACvBA,OAAO,iBAAiB;YAE5B;YACA,iBAAiB7B,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAavC,QAAQ,WAAW;YAChC,cAAc,CAACmC;gBACb5C,aACE,+DACA4C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;YAEpD;QACF;IACF,EAAE,OAAOuB,WAAW;QAClB,IAAIA,qBAAqBF,sBACvB,OAAO;YACL,kBAAkB1D;YAClB,OAAO4D,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMC,eACJD,qBAAqBrB,QAAQqB,UAAU,OAAO,GAAGD,OAAOC;QAC1D,OAAO;YACL,kBAAkB5D;YAClB,OAAO,CAAC,eAAe,EAAE6D,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAES,MAAM,EAAEC,YAAY,EAAEE,UAAU,EAAE,GAAGJ;IAC7C,IAAI,CAACI,YACH,OAAO;QACL,kBAAkBzE;QAClB,OAAOuE;QACP,aAAaD,OAAO,aAAa;QACjC,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMK,eAAepE,iBAAiBkE,YAAYtE,QAAQ,QAAQ;QAClE,MAAMyE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1ClF,aAAa,2BAA2BkF;QAExC,MAAMG,mBAAmB,MAAM7E,sBAAsB;YACnDE;YACA,UAAUsE;QACZ;QAEAhF,aACE,uDACAmF,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOP;YACP,aAAaD,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOjC,OAAO;QACd,MAAMkB,oBACJlB,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMwB,eAAeU,eACjB,GAAGA,aAAa,EAAE,EAAEhB,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBvD;YAClB,OAAO6D;YACP,aAAaS,OAAO,aAAa;YACjC,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF;AAEO,eAAeS,qBAAwB7E,OAQ7C;IACC,MAAM,EAAE8E,SAAS,EAAE7E,OAAO,EAAE8E,aAAa,EAAEC,gBAAgB,EAAEzC,YAAY,EAAE,GACzEvC;IACF,MAAM4C,eAAeqC,sBAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMlB,mBAAmB7D,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMiF,wBAAwBC,uBAC5BnF,QAAQ,eAAe,IAAI,IAC3B8E,WACAC,eAAe;IAGjB,MAAMK,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/C,MAAMM,qBAAqBpF,QAAQ,kBAAkB;QACrD,IAAIoF,sBAAsBA,mBAAmB,MAAM,GAAG,GAAG;YACvDD,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM,CAAC,cAAc,EAAEC,mBAAmB,MAAM,CAAC,0HAA0H,EAAEA,mBAAmB,MAAM,CAAC,2mBAA2mB,CAAC;YACrzB;YAEAA,mBAAmB,OAAO,CAAC,CAACC,OAAOC;gBACjCH,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,MAAM,CAAC,MAAM,EAAEG,QAAQ,EAAE,CAAC,EAAEF,mBAAmB,MAAM,EAAE;gBACzD;gBACAD,YAAY,IAAI,CAAC;oBACf,MAAM;oBACN,WAAW;wBACT,KAAKE,MAAM,MAAM;wBACjB,QAAQ;oBACV;gBACF;YACF;QACF,OAAO;YACLF,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,MAAM;YACR;YAEAA,YAAY,IAAI,CAAC;gBACf,MAAM;gBACN,WAAW;oBACT,KAAKtB;oBACL,QAAQ;gBACV;YACF;QACF;IACF;IAEAsB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMjC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASwC;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAMd,SAAS,MAAMlD,+BAA+BgE;QACpD/B,KAAK,IAAI,IAAIiB;IACf;IAEA,OAAOhB,wBAAwB;QAC7B,QAAQ,IACNsC,OAAOvC,MAAMV,cAAc;gBACzB,aAAavC,QAAQ,WAAW;YAClC;QACF,eAAe,CAACoD;YACd,MAAM,EACJ,SAAS3B,WAAW,EACpBE,KAAK,EACL8D,iBAAiB,EACjB/D,gBAAgB,EACjB,GAAG0B;YACJ,MAAMsC,cAAcC,2BAA8BlE;YAClD,OAAO;gBACLiE;gBACAjE;gBACAC;gBACAC;gBACA8D;YACF;QACF;QACA,cAAc,CAACG,YAAYxC;YACzB,MAAMO,eACJiC,sBAAsBvD,QAAQuD,WAAW,OAAO,GAAGnC,OAAOmC;YAC5D,OAAO,IAAIpC,qBACT,CAAC,iBAAiB,EAAEG,cAAc,EAClCP,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB;QAE7B;QACA,iBAAiBb,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrD,aAAavC,QAAQ,WAAW;QAChC,cAAc,CAACmC;YACb9C,aACE,iDACA8C,iBAAiBE,QAAQF,MAAM,OAAO,GAAGsB,OAAOtB;QAEpD;IACF;AACF;AAEO,eAAe0D,sBACpBC,WAAmB,EACnBvD,YAA0B;IAK1B,MAAMK,eAAemD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAM7C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASoD;QACX;KACD;IAED3G,aAAa,yCAAyCyG;IAEtD,MAAM1B,SAAS,MAAMjB,yBACnBF,MACAV,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB6B,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/llm-planning.mjs

@@ -245,7 +245,8 @@ async function plan(userInstruction, opts) {
         if (planFromAI.log) conversationHistory.appendSubGoalLog(planFromAI.log);
     } else if (planFromAI.log) conversationHistory.appendHistoricalLog(planFromAI.log);
     if (planFromAI.memory) conversationHistory.appendMemory(planFromAI.memory);
-    conversationHistory.append({
+    if (modelRuntime.adapter.chatCompletion.replayRawAssistantMessage && rawChoiceMessage) conversationHistory.append(rawChoiceMessage);
+    else conversationHistory.append({
         role: 'assistant',
         content: [
             {