@midscene/core 1.9.1 → 1.9.2-beta-20260605084246.0

package/dist/es/agent/tasks.mjs

@@ -1,7 +1,7 @@
 import { AIResponseParseError, ConversationHistory } from "../ai-model/index.mjs";
 import { buildTypeQueryDemandValue } from "../ai-model/prompt/extraction.mjs";
 import { genericXmlPlan } from "../ai-model/workflows/planning/index.mjs";
-import { getReadableTimeString } from "../common.mjs";
+import { getReadableTimeString, multimodalPromptToChatMessages, userPromptToMultimodalPrompt, userPromptToString } from "../common.mjs";
 import { TaskExecutionError } from "../task-runner.mjs";
 import { ServiceError } from "../types.mjs";
 import { getDebug } from "@midscene/shared/logger";
@@ -55,7 +55,7 @@ class TaskExecutor {
         return this.taskBuilder.build(plans, planningModel, defaultModel, options);
     }
     async loadYamlFlowAsPlanning(userInstruction, yamlString) {
-        const session = this.createExecutionSession(taskTitleStr('Act', userInstruction));
+        const session = this.createExecutionSession(taskTitleStr('Act', userPromptToString(userInstruction)));
         const task = {
             type: 'Planning',
             subType: 'LoadYaml',
@@ -104,13 +104,23 @@ class TaskExecutor {
     async action(userPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, fileChooserAccept, deepLocate, abortSignal) {
         return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal));
     }
+    invalidateFailedCacheHitLocates(runner, fromIndex) {
+        if (!this.taskCache) return;
+        for(let i = fromIndex; i < runner.tasks.length; i++){
+            const task = runner.tasks[i];
+            if ('Planning' === task.type && 'Locate' === task.subType && task.hitBy?.from === 'Cache') {
+                const prompt = task.param?.prompt;
+                if (prompt) this.taskCache.markLocateCacheStale(prompt);
+            }
+        }
+    }
     async runAction(userPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal) {
         if (deepLocate && !planningModel.adapter.planning.supportsActionDeepLocate) {
             warnLog(`The "deepLocate" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`);
             deepLocate = false;
         }
         const conversationHistory = new ConversationHistory();
-        const session = this.createExecutionSession(taskTitleStr('Act', userPrompt));
+        const session = this.createExecutionSession(taskTitleStr('Act', userPromptToString(userPrompt)));
         const runner = session.getRunner();
         let replanCount = 0;
         const yamlFlow = [];
@@ -118,6 +128,8 @@ class TaskExecutor {
         assert(void 0 !== replanningCycleLimit, 'replanningCycleLimit is required for TaskExecutor.action');
         let errorCountInOnePlanningLoop = 0;
         let outputString;
+        if (abortSignal?.aborted) return session.appendErrorPlan(`Task aborted: ${abortSignal.reason || 'abort signal received'}`);
+        const referenceImageMessages = await multimodalPromptToChatMessages(userPromptToMultimodalPrompt(userPrompt));
         while(true){
             if (abortSignal?.aborted) return session.appendErrorPlan(`Task aborted: ${abortSignal.reason || 'abort signal received'}`);
             const subGoalStatus = conversationHistory.subGoalsToText() || void 0;
@@ -158,6 +170,7 @@ class TaskExecutor {
                             includeLocateInPlanning,
                             imagesIncludeCount,
                             deepThink,
+                            referenceImageMessages,
                             abortSignal
                         });
                     } catch (planError) {
@@ -220,6 +233,7 @@ class TaskExecutor {
             if (conversationHistory.pendingFeedbackMessage) console.warn('unconsumed pending feedback message detected, this may lead to unexpected planning result:', conversationHistory.pendingFeedbackMessage);
             const initialTimeString = await this.getTimeString();
             conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;
+            const taskCountBeforeRun = runner.tasks.length;
             try {
                 await session.appendAndRun(executables.tasks);
             } catch (error) {
@@ -231,6 +245,7 @@ class TaskExecutor {
             if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) return session.appendErrorPlan('Too many errors in one planning loop');
             if (abortSignal?.aborted) return session.appendErrorPlan(`Task aborted: ${abortSignal.reason || 'abort signal received'}`);
             if (!planResult?.shouldContinuePlanning) break;
+            this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);
             ++replanCount;
             if (replanCount > replanningCycleLimit) {
                 const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeviceAction,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      planningModel,\n      defaultModel,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        includeLocateInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n      );\n    });\n  }\n\n  private async runAction(\n    userPrompt: string,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    if (\n      deepLocate &&\n      !planningModel.adapter.planning.supportsActionDeepLocate\n    ) {\n      warnLog(\n        `The \"deepLocate\" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`,\n      );\n      deepLocate = false;\n    }\n\n    const conversationHistory = new ConversationHistory();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const timing = executorContext.task.timing;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : genericXmlPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: uiContext,\n                actionContext: param.aiActContext,\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                };\n              }\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = uiContext;\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              assert(\n                false,\n                `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`,\n              );\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Set initial time context for the first planning call\n      const initialTimeString = await this.getTimeString();\n      conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;\n\n      try {\n        await session.appendAndRun(executables.tasks);\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message || String(error)}`;\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelRuntime,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","format","error","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","conversationHistory","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","outputString","subGoalStatus","memoriesStatus","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","timeString","String","Error","errorMsg","type","demand","modelRuntime","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAqDA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAIvC,MAAMC;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASA,MAAc,cAAcG,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdR,QACE,CAAC,gEAAgE,EAAEQ,OAAO;QAE9E;aAEAR,QACE;QAKN,OAAOS,sBAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BR,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACM,OAAOC,eAAeC,cAAcR;IACpE;IAEA,MAAM,uBAAuBS,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOH;QAGtB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJnB,KAAa,EACbO,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EACA;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACZ;QAC5C,MAAM,EAAEoB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,eACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,aAA2B,EAC3BC,YAA0B,EAC1Be,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EASzB;QACA,OAAOC,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAf,eACAC,cACAe,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC;IAGN;IAEA,MAAc,UACZT,UAAkB,EAClBf,aAA2B,EAC3BC,YAA0B,EAC1Be,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EASzB;QACA,IACED,cACA,CAACvB,cAAc,OAAO,CAAC,QAAQ,CAAC,wBAAwB,EACxD;YACAX,QACE,CAAC,mGAAmG,EAAEW,cAAc,MAAM,CAAC,WAAW,IAAI,UAAU,sBAAsB,CAAC;YAE7KuB,aAAa;QACf;QAEA,MAAMG,sBAAsB,IAAIC;QAEhC,MAAMvB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOU;QAEtB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIwB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJX,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DT,OACEoB,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAClC,IAAIC;QAGJ,MAAO,KAAM;YAEX,IAAIT,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMU,gBAAgBR,oBAAoB,cAAc,MAAMK;YAG9D,MAAMI,iBAAiBT,oBAAoB,cAAc,MAAMK;YAE/D,MAAMlB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjBE;oBACAG;oBACAC;oBACA,GAAIa,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO5B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM2B,SAAS5B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAM6B,cAAc,IAAI,CAAC,cAAc;oBACvClD,MACE,sCACAkD,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD5B,OAAO6B,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJzC,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrC0C;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAASlC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjC8B;4BACA,cAAcrC;4BACd0B;4BACAV;4BACAI;4BACAC;4BACAG;wBACF;oBACF,EAAE,OAAOqB,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7CtC,gBAAgB,IAAI,CAAC,KAAK,GAAGuC,gBAC3BF,UAAU,KAAK,EACf;4BAEFrC,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAaqC,UAAU,WAAW;4BACpC;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBR,QAAQ;oBAC7B;oBACAjD,MAAM,cAAc6D,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACNvD,KAAK,EACLwD,KAAK,EACLC,WAAW,EACXC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGhB;oBACJV,eAAewB;oBAEfjD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClC8C;oBACF;oBACA9C,gBAAgB,IAAI,CAAC,KAAK,GAAGuC,gBAAgBM,OAAO;oBACpD7C,gBAAgB,IAAI,CAAC,iBAAiB,GAAG+C;oBACzC/C,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAASyC,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQc;wBACR,wBAAwBd,WAAW,sBAAsB;wBACzDe;wBACAC;oBACF;oBACAnD,gBAAgB,SAAS,GAAGC;oBAE5BC,OAAO,CAACb,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEsD,OAAO,IAAI;oBAG3D,IAAIK,AAAoB,UAApBA,iBACF9C,OACE,OACA,CAAC,aAAa,EAAE+C,mBAAmB,4BAA4B,EAAE,EAAEN,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAa9B,QAAQ;YAG3B,MAAMd,QAAQ4C,YAAY,WAAW,EAAE;YACvCd,SAAS,IAAI,IAAKc,YAAY,YAAY,EAAE;YAE5C,IAAIiB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9C7D,OACAC,eACAC,cACA;oBACEiB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO3B,OAAO;gBACd,OAAOO,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEP,MAAM,SAAS,EAAEmD,KAAK,SAAS,CAC5EjD,QACC;YAEP;YACA,IAAI2B,oBAAoB,sBAAsB,EAC5Cc,QAAQ,IAAI,CACV,8FACAd,oBAAoB,sBAAsB;YAK9C,MAAMmC,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAClDnC,oBAAoB,sBAAsB,IAAI,CAAC,cAAc,EAAEmC,mBAAmB;YAElF,IAAI;gBACF,MAAMzD,QAAQ,YAAY,CAACwD,YAAY,KAAK;YAC9C,EAAE,OAAO/D,OAAY;gBAEnBmC;gBACA,MAAM8B,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CpC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEoC,WAAW,iCAAiC,EAAEjE,OAAO,WAAWkE,OAAOlE,QAAQ;gBACrIV,MACE,yFACAU,iBAAiBmE,QAAQnE,MAAM,OAAO,GAAGkE,OAAOlE,QAChD,6CACAmC;YAEJ;YAEA,IAAIA,8BAA8B1C,uCAChC,OAAOc,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACmB,YAAY,wBACf;YAIF,EAAEf;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMmC,WAAW,CAAC,UAAU,EAAEnC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO1B,QAAQ,eAAe,CAAC6D;YACjC;YAEA,IAAI,CAACvC,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMoC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CpC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEoC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACNjC;gBACA,QAAQI;YACV;YACAtB;QACF;IACF;IAEQ,oBACNuD,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAO5D,OAAOiE;gBACtB,MAAM,EAAElE,IAAI,EAAE,GAAGkE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZrE,KAAK,GAAG,GAAG;wBACTqE;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACArE,KAAK,KAAK,GAAGyC,gBAAgB4B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjBrE,KAAK,iBAAiB,GAAGqE,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMlE,YAAY+D,YAAY,SAAS;gBACvC9D,OAAOD,WAAW;gBAElB,MAAMmE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF,OAAO,IAAIS,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DlF,MAAM;oBACN,MAAM+F,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,cACAC,KACAY,sBACAX,kBACA7D;gBAEJ,EAAE,OAAOZ,OAAO;oBACd,IAAIA,iBAAiBuF,cACnBV,UAAU7E,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEwF,IAAI,EAAEnC,OAAO,EAAEyB,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAK/C,QAC1BuD,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAWtD,QAC1BuD,eAAgBD,KAAa,MAAM;qBAEnC3E,OAAO,OAAO;gBAKpB,IAAIwD,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtChF,KAAK,OAAO,GAAG4C;oBACf,MAAM,IAAIc,MAAM,CAAC,kBAAkB,EAAEd,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQoC;oBACR,KAAKb;oBACLvB;gBACF;YACF;QACF;QAEA,OAAOqB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMlE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACE6D,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASnB,KAAK,SAAS,CAACmB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,cACAC,KACAC;QAGF,MAAM3D,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACmE;QAE1C,IAAI,CAAC1D,QACH,MAAM,IAAImD,MACR;QAIJ,MAAM,EAAElD,MAAM,EAAEoC,OAAO,EAAE,GAAGrC;QAE5B,OAAO;YACLC;YACAoC;YACAvC;QACF;IACF;IAEA,MAAM,QACJ4E,SAAsB,EACtBlB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMpF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAWqF;QAE1B,MAAM/E,SAASP,QAAQ,SAAS;QAChC,MAAM,EACJuF,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEArF,OAAO6E,WAAW;QAClB7E,OAAOiF,WAAW;QAClBjF,OAAOkF,iBAAiB;QAExBlF,OACEkF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,cACA4B,mBACA1B;YAGF,MAAMzD,SAAU,MAAMT,QAAQ,YAAY,CAACmE;YAO3C,IAAI1D,QAAQ,QACV,OAAO;gBACL,QAAQkB;gBACRpB;YACF;YAGFyF,eACEvF,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAE2E,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMrD,UAAU,CAAC,kBAAkB,EAAE0C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGtD;oBAAQ;iBAAE,EAC9DkB,cACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAMrG,QAAQ,YAAY,CAACqG,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAOrG,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEgG,cAAc;IACnE;IA/sBA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAmrBF;AAEO,eAAenF,gBACpBiF,iBAAoC,EACpCpF,iBAAuC,EACvCgB,MAAwB;IAExB,IAAI,CAAChB,mBAAmB,QACtB,OAAOgB;IAGT,IAAI,CAACoE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAACzF;IACvB;IAEA,MAAM,EAAE0F,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMjG,SAAS,MAAMyB;QAErB,MAAMzC,QAAQoH;QACd,IAAIpH,OACF,MAAMA;QAER,OAAOgB;IACT,SAAU;QACRmG;IACF;AACF"}
+{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import { AIResponseParseError, ConversationHistory } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport { genericXmlPlan } from '@/ai-model/workflows/planning';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeviceAction,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  PlanningLocateParam,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTime?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTime?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTime = opts.useDeviceTime;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Get a readable time string. When device time is enabled, use the\n   * device-formatted wall-clock time directly so host timezone formatting does\n   * not reinterpret a device timestamp.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    if (this.useDeviceTime) {\n      if (this.interface.getDeviceLocalTimeString) {\n        try {\n          return await this.interface.getDeviceLocalTimeString(format);\n        } catch (error) {\n          warnLog(\n            `Failed to get device time string, falling back to runtime time: ${error}`,\n          );\n        }\n      } else {\n        warnLog(\n          'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n        );\n      }\n    }\n\n    return getReadableTimeString(format);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(plans, planningModel, defaultModel, options);\n  }\n\n  async loadYamlFlowAsPlanning(\n    userInstruction: TUserPrompt,\n    yamlString: string,\n  ) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userPromptToString(userInstruction)),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      planningModel,\n      defaultModel,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        planningModel,\n        defaultModel,\n        includeLocateInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n      );\n    });\n  }\n\n  /**\n   * Called when the task is about to replan. Marks every cache-hit locate task\n   * in the just-run batch (tasks at index >= fromIndex) as stale: that batch\n   * did not finish the task, so the element each cache hit produced is suspect.\n   * The upcoming re-locate of the same prompt then replaces the bad entry in\n   * place instead of appending a duplicate that would re-poison the cache on the\n   * next run (#2529).\n   *\n   * Marking a locate that was actually fine is harmless: the step is only ever\n   * replaced if the same prompt is located again (i.e. the step is redone),\n   * which does not happen for a locate that already succeeded.\n   */\n  private invalidateFailedCacheHitLocates(\n    runner: TaskRunner,\n    fromIndex: number,\n  ) {\n    if (!this.taskCache) {\n      return;\n    }\n    for (let i = fromIndex; i < runner.tasks.length; i++) {\n      const task = runner.tasks[i];\n      if (\n        task.type === 'Planning' &&\n        task.subType === 'Locate' &&\n        task.hitBy?.from === 'Cache'\n      ) {\n        const prompt = (task.param as PlanningLocateParam | undefined)?.prompt;\n        if (prompt) {\n          this.taskCache.markLocateCacheStale(prompt);\n        }\n      }\n    }\n  }\n\n  private async runAction(\n    userPrompt: TUserPrompt,\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    includeLocateInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: boolean,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    if (\n      deepLocate &&\n      !planningModel.adapter.planning.supportsActionDeepLocate\n    ) {\n      warnLog(\n        `The \"deepLocate\" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`,\n      );\n      deepLocate = false;\n    }\n\n    const conversationHistory = new ConversationHistory();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userPromptToString(userPrompt)),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n\n    if (abortSignal?.aborted) {\n      return session.appendErrorPlan(\n        `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n      );\n    }\n    const referenceImageMessages = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(userPrompt),\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const timing = executorContext.task.timing;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl =\n              planningModel.adapter.planning.kind === 'custom'\n                ? planningModel.adapter.planning.planFn\n                : genericXmlPlan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: uiContext,\n                actionContext: param.aiActContext,\n                actionSpace,\n                modelRuntime: planningModel,\n                conversationHistory,\n                includeLocateInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                referenceImageMessages,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = withUsageIntent(\n                  planError.usage,\n                  'planning',\n                );\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                };\n              }\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n            };\n            executorContext.task.usage = withUsageIntent(usage, 'planning');\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = uiContext;\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              assert(\n                false,\n                `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`,\n              );\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          planningModel,\n          defaultModel,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Set initial time context for the first planning call\n      const initialTimeString = await this.getTimeString();\n      conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;\n\n      const taskCountBeforeRun = runner.tasks.length;\n      try {\n        await session.appendAndRun(executables.tasks);\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message || String(error)}`;\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // We are about to replan, which means the batch we just ran did not finish\n      // the task. Any locate task in that batch that was served from cache\n      // produced an element that failed to complete the step (the action threw,\n      // or it clicked the wrong element and the goal was not reached). Mark those\n      // cache entries stale so the re-locate of the same prompt replaces them in\n      // place instead of appending a poisoning duplicate that would be matched\n      // first on the next run (#2529).\n      this.invalidateFailedCacheHitLocates(runner, taskCountBeforeRun);\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelRuntime,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelRuntime: ModelRuntime,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelRuntime,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelRuntime: ModelRuntime,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelRuntime,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelRuntime,\n          modelRuntime,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","format","error","getReadableTimeString","plans","planningModel","defaultModel","userInstruction","yamlString","session","taskTitleStr","userPromptToString","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeLocateInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","fromIndex","i","prompt","conversationHistory","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","outputString","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","subGoalStatus","memoriesStatus","timing","actionSpace","action","Array","console","planImpl","genericXmlPlan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","taskCountBeforeRun","timeString","String","Error","errorMsg","type","demand","modelRuntime","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AAyDA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAIvC,MAAMC;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASA,MAAc,cAAcG,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdR,QACE,CAAC,gEAAgE,EAAEQ,OAAO;QAE9E;aAEAR,QACE;QAKN,OAAOS,sBAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BR,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACM,OAAOC,eAAeC,cAAcR;IACpE;IAEA,MAAM,uBACJS,eAA4B,EAC5BC,UAAkB,EAClB;QACA,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOC,mBAAmBJ;QAGzC,MAAMK,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLL;YACF;YACA,UAAU,OAAOM,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLP;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMS,SAASR,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACG;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJpB,KAAa,EACbO,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EACA;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACZ;QAC5C,MAAM,EAAEqB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDd,OACAC,eACAC;QAEF,MAAMW,SAASR,QAAQ,SAAS;QAChC,MAAMU,SAAS,MAAMV,QAAQ,YAAY,CAACS;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EASzB;QACA,OAAOC,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAhB,eACAC,cACAgB,yBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC;IAGN;IAcQ,gCACNb,MAAkB,EAClBe,SAAiB,EACjB;QACA,IAAI,CAAC,IAAI,CAAC,SAAS,EACjB;QAEF,IAAK,IAAIC,IAAID,WAAWC,IAAIhB,OAAO,KAAK,CAAC,MAAM,EAAEgB,IAAK;YACpD,MAAMrB,OAAOK,OAAO,KAAK,CAACgB,EAAE;YAC5B,IACErB,AAAc,eAAdA,KAAK,IAAI,IACTA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,KAAK,KAAK,EAAE,SAAS,SACrB;gBACA,MAAMsB,SAAUtB,KAAK,KAAK,EAAsC;gBAChE,IAAIsB,QACF,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAACA;YAExC;QACF;IACF;IAEA,MAAc,UACZb,UAAuB,EACvBhB,aAA2B,EAC3BC,YAA0B,EAC1BgB,uBAAgC,EAChCC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EASzB;QACA,IACED,cACA,CAACxB,cAAc,OAAO,CAAC,QAAQ,CAAC,wBAAwB,EACxD;YACAX,QACE,CAAC,mGAAmG,EAAEW,cAAc,MAAM,CAAC,WAAW,IAAI,UAAU,sBAAsB,CAAC;YAE7KwB,aAAa;QACf;QAEA,MAAMM,sBAAsB,IAAIC;QAEhC,MAAM3B,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOC,mBAAmBU;QAEzC,MAAMJ,SAASR,QAAQ,SAAS;QAEhC,IAAI4B,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJd,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DT,OACEuB,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAClC,IAAIC;QAEJ,IAAIZ,aAAa,SACf,OAAOrB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEqB,YAAY,MAAM,IAAI,yBAAyB;QAGpE,MAAMa,yBAAyB,MAAMC,+BACnCC,6BAA6BxB;QAI/B,MAAO,KAAM;YAEX,IAAIS,aAAa,SACf,OAAOrB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEqB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMgB,gBAAgBX,oBAAoB,cAAc,MAAMK;YAG9D,MAAMO,iBAAiBZ,oBAAoB,cAAc,MAAMK;YAE/D,MAAMrB,SAAS,MAAMV,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBY;oBACjBE;oBACAG;oBACAC;oBACA,GAAImB,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAOlC,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAMiC,SAASlC,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAMmC,cAAc,IAAI,CAAC,cAAc;oBACvCzD,MACE,sCACAyD,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhDlC,OAAOmC,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WACJhD,AAAwC,aAAxCA,cAAc,OAAO,CAAC,QAAQ,CAAC,IAAI,GAC/BA,cAAc,OAAO,CAAC,QAAQ,CAAC,MAAM,GACrCiD;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBR,QAAQ;wBAC3BO,aAAa,MAAMF,SAASxC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjCoC;4BACA,cAAc5C;4BACd8B;4BACAb;4BACAI;4BACAC;4BACAgB;4BACAb;wBACF;oBACF,EAAE,OAAO2B,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7C5C,gBAAgB,IAAI,CAAC,KAAK,GAAG6C,gBAC3BF,UAAU,KAAK,EACf;4BAEF3C,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAa2C,UAAU,WAAW;4BACpC;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBR,QAAQ;oBAC7B;oBACAxD,MAAM,cAAcoE,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN9D,KAAK,EACL+D,KAAK,EACLC,WAAW,EACXC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGhB;oBACJb,eAAe2B;oBAEfvD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCoD;oBACF;oBACApD,gBAAgB,IAAI,CAAC,KAAK,GAAG6C,gBAAgBM,OAAO;oBACpDnD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGqD;oBACzCrD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS+C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQc;wBACR,wBAAwBd,WAAW,sBAAsB;wBACzDe;wBACAC;oBACF;oBACAzD,gBAAgB,SAAS,GAAGC;oBAE5BC,OAAO,CAACd,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAE6D,OAAO,IAAI;oBAG3D,IAAIK,AAAoB,UAApBA,iBACFpD,OACE,OACA,CAAC,aAAa,EAAEqD,mBAAmB,4BAA4B,EAAE,EAAEN,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAapC,QAAQ;YAG3B,MAAMf,QAAQmD,YAAY,WAAW,EAAE;YACvCjB,SAAS,IAAI,IAAKiB,YAAY,YAAY,EAAE;YAE5C,IAAIiB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CpE,OACAC,eACAC,cACA;oBACEkB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO5B,OAAO;gBACd,OAAOO,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEP,MAAM,SAAS,EAAE0D,KAAK,SAAS,CAC5ExD,QACC;YAEP;YACA,IAAI+B,oBAAoB,sBAAsB,EAC5CiB,QAAQ,IAAI,CACV,8FACAjB,oBAAoB,sBAAsB;YAK9C,MAAMsC,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAClDtC,oBAAoB,sBAAsB,IAAI,CAAC,cAAc,EAAEsC,mBAAmB;YAElF,MAAMC,qBAAqBzD,OAAO,KAAK,CAAC,MAAM;YAC9C,IAAI;gBACF,MAAMR,QAAQ,YAAY,CAAC+D,YAAY,KAAK;YAC9C,EAAE,OAAOtE,OAAY;gBAEnBuC;gBACA,MAAMkC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CxC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEwC,WAAW,iCAAiC,EAAEzE,OAAO,WAAW0E,OAAO1E,QAAQ;gBACrIV,MACE,yFACAU,iBAAiB2E,QAAQ3E,MAAM,OAAO,GAAG0E,OAAO1E,QAChD,6CACAuC;YAEJ;YAEA,IAAIA,8BAA8B9C,uCAChC,OAAOc,QAAQ,eAAe,CAAC;YAIjC,IAAIqB,aAAa,SACf,OAAOrB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEqB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACyB,YAAY,wBACf;YAUF,IAAI,CAAC,+BAA+B,CAACtC,QAAQyD;YAG7C,EAAErC;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMuC,WAAW,CAAC,UAAU,EAAEvC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO9B,QAAQ,eAAe,CAACqE;YACjC;YAEA,IAAI,CAAC3C,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMwC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CxC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEwC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACNrC;gBACA,QAAQI;YACV;YACAzB;QACF;IACF;IAEQ,oBACN8D,IAAsE,EACtEC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOnE,OAAOwE;gBACtB,MAAM,EAAEzE,IAAI,EAAE,GAAGyE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ5E,KAAK,GAAG,GAAG;wBACT4E;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA5E,KAAK,KAAK,GAAG+C,gBAAgB6B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjB5E,KAAK,iBAAiB,GAAG4E,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMzE,YAAYsE,YAAY,SAAS;gBACvCrE,OAAOD,WAAW;gBAElB,MAAM0E,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF,OAAO,IAAIS,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1D1F,MAAM;oBACN,MAAMuG,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,cACAC,KACAY,sBACAX,kBACApE;gBAEJ,EAAE,OAAOb,OAAO;oBACd,IAAIA,iBAAiB+F,cACnBV,UAAUrF,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEgG,IAAI,EAAEpC,OAAO,EAAE0B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAKnD,QAC1B2D,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAW1D,QAC1B2D,eAAgBD,KAAa,MAAM;qBAEnClF,OAAO,OAAO;gBAKpB,IAAI+D,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCvF,KAAK,OAAO,GAAGkD;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQqC;oBACR,KAAKb;oBACLxB;gBACF;YACF;QACF;QAEA,OAAOsB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,YAA0B,EAC1BC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAM1E,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEqE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,cACAC,KACAC;QAGF,MAAMlE,SAASR,QAAQ,SAAS;QAChC,MAAMU,SAAS,MAAMV,QAAQ,YAAY,CAAC2E;QAE1C,IAAI,CAACjE,QACH,MAAM,IAAI0D,MACR;QAIJ,MAAM,EAAEzD,MAAM,EAAE0C,OAAO,EAAE,GAAG3C;QAE5B,OAAO;YACLC;YACA0C;YACA7C;QACF;IACF;IAEA,MAAM,QACJmF,SAAsB,EACtBlB,GAA+B,EAC/BD,YAA0B,EACM;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM5F,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW6F;QAE1B,MAAMtF,SAASR,QAAQ,SAAS;QAChC,MAAM,EACJ+F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA5F,OAAOoF,WAAW;QAClBpF,OAAOwF,WAAW;QAClBxF,OAAOyF,iBAAiB;QAExBzF,OACEyF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,cACA4B,mBACA1B;YAGF,MAAMhE,SAAU,MAAMV,QAAQ,YAAY,CAAC2E;YAO3C,IAAIjE,QAAQ,QACV,OAAO;gBACL,QAAQqB;gBACRvB;YACF;YAGFgG,eACE9F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEkF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMtD,UAAU,CAAC,kBAAkB,EAAE2C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGvD;oBAAQ;iBAAE,EAC9DmB,cACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAM7G,QAAQ,YAAY,CAAC6G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO7G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEwG,cAAc;IACnE;IAxwBA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AA4uBF;AAEO,eAAe1F,gBACpBwF,iBAAoC,EACpC3F,iBAAuC,EACvCsB,MAAwB;IAExB,IAAI,CAACtB,mBAAmB,QACtB,OAAOsB;IAGT,IAAI,CAACqE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAChG;IACvB;IAEA,MAAM,EAAEiG,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMxG,SAAS,MAAM+B;QAErB,MAAMhD,QAAQ4H;QACd,IAAI5H,OACF,MAAMA;QAER,OAAOiB;IACT,SAAU;QACR0G;IACF;AACF"}

package/dist/es/agent/ui-utils.mjs

@@ -1,3 +1,4 @@
+import { userPromptToString } from "../common.mjs";
 function typeStr(task) {
     return task.subType || task.type;
 }
@@ -63,7 +64,7 @@ function paramStr(task) {
     if ('Planning' === task.type) if ('Locate' === task.subType) value = locateParamStr(task?.param);
     else {
         const planTask = task;
-        value = planTask.output?.log || planTask.param?.userInstruction;
+        value = planTask.output?.log || (planTask.param?.userInstruction ? userPromptToString(planTask.param.userInstruction) : void 0);
     }
     if ('Insight' === task.type) value = extractInsightParam(task?.param).content;
     if ('Action Space' === task.type) {

package/dist/es/agent/ui-utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.mjs","sources":["../../../src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Act'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      // Prefer AI-generated output.log over user input\n      const planTask = task as ExecutionTaskPlanning;\n      value = planTask.output?.log || planTask.param?.userInstruction;\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'PullGesture'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    // Flatten `{key: \"raw value\"}` into `key: raw value` instead of emitting\n    // a pretty-printed JSON string. JSON.stringify would escape every inner\n    // quote as `\\\"`, and the UI renders the result as plain text — so a\n    // command like `grep -E \"version\"` ends up shown with the literal\n    // backslashes, which is both noisy and confusing for users.\n    const entries = Object.entries(value as Record<string, unknown>);\n    if (entries.length === 0) {\n      return '';\n    }\n    const formatValue = (v: unknown): string => {\n      if (typeof v === 'string') return v;\n      if (v === null || v === undefined) return String(v);\n      if (typeof v === 'object') return JSON.stringify(v);\n      return String(v);\n    };\n    if (entries.length === 1) {\n      const [key, v] = entries[0];\n      return `${key}: ${formatValue(v)}`;\n    }\n    return entries.map(([key, v]) => `${key}: ${formatValue(v)}`).join(', ');\n  }\n\n  return String(value);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","planTask","locateStr","Object","entries","formatValue","v","String","key"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAE9B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAEO,SAASC,aACdC,IAcY,EACZjB,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,MAAsC;SACzD;QAEL,MAAMsB,WAAWtB;QACjBgB,QAAQM,SAAS,MAAM,EAAE,OAAOA,SAAS,KAAK,EAAE;IAClD;IAGF,IAAItB,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMuB,YAAYrB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCwB,OAAO,IAAI,CAAExB,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIuB,WAEAP,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGO,UAAU,GAAG,EAAEP,OAAO,GAEzBO;IAGd;IAEA,IAAI,AAAiB,WAAVP,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMO,YAAYtB,eAAee;QACjC,IAAIO,WACF,OAAOA;QAOT,MAAME,UAAUD,OAAO,OAAO,CAACR;QAC/B,IAAIS,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAO;QAET,MAAMC,cAAc,CAACC;YACnB,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOA;YAClC,IAAIA,QAAAA,GAA+B,OAAOC,OAAOD;YACjD,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOV,KAAK,SAAS,CAACU;YACjD,OAAOC,OAAOD;QAChB;QACA,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;YACxB,MAAM,CAACI,KAAKF,EAAE,GAAGF,OAAO,CAAC,EAAE;YAC3B,OAAO,GAAGI,IAAI,EAAE,EAAEH,YAAYC,IAAI;QACpC;QACA,OAAOF,QAAQ,GAAG,CAAC,CAAC,CAACI,KAAKF,EAAE,GAAK,GAAGE,IAAI,EAAE,EAAEH,YAAYC,IAAI,EAAE,IAAI,CAAC;IACrE;IAEA,OAAOC,OAAOZ;AAChB"}
+{"version":3,"file":"agent/ui-utils.mjs","sources":["../../../src/agent/ui-utils.ts"],"sourcesContent":["import type { TUserPrompt } from '@/common';\nimport { userPromptToString } from '@/common';\nimport type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Act'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      // Prefer AI-generated output.log over user input\n      const planTask = task as ExecutionTaskPlanning;\n      value =\n        planTask.output?.log ||\n        (planTask.param?.userInstruction\n          ? userPromptToString(planTask.param.userInstruction as TUserPrompt)\n          : undefined);\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'PullGesture'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    // Flatten `{key: \"raw value\"}` into `key: raw value` instead of emitting\n    // a pretty-printed JSON string. JSON.stringify would escape every inner\n    // quote as `\\\"`, and the UI renders the result as plain text — so a\n    // command like `grep -E \"version\"` ends up shown with the literal\n    // backslashes, which is both noisy and confusing for users.\n    const entries = Object.entries(value as Record<string, unknown>);\n    if (entries.length === 0) {\n      return '';\n    }\n    const formatValue = (v: unknown): string => {\n      if (typeof v === 'string') return v;\n      if (v === null || v === undefined) return String(v);\n      if (typeof v === 'object') return JSON.stringify(v);\n      return String(v);\n    };\n    if (entries.length === 1) {\n      const [key, v] = entries[0];\n      return `${key}: ${formatValue(v)}`;\n    }\n    return entries.map(([key, v]) => `${key}: ${formatValue(v)}`).join(', ');\n  }\n\n  return String(value);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","planTask","userPromptToString","locateStr","Object","entries","formatValue","v","String","key"],"mappings":";AAcO,SAASA,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAE9B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAEO,SAASC,aACdC,IAcY,EACZjB,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,MAAsC;SACzD;QAEL,MAAMsB,WAAWtB;QACjBgB,QACEM,SAAS,MAAM,EAAE,OAChBA,CAAAA,SAAS,KAAK,EAAE,kBACbC,mBAAmBD,SAAS,KAAK,CAAC,eAAe,IACjDR,MAAQ;IAChB;IAGF,IAAId,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCyB,OAAO,IAAI,CAAEzB,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIwB,WAEAR,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMQ,YAAYvB,eAAee;QACjC,IAAIQ,WACF,OAAOA;QAOT,MAAME,UAAUD,OAAO,OAAO,CAACT;QAC/B,IAAIU,AAAmB,MAAnBA,QAAQ,MAAM,EAChB,OAAO;QAET,MAAMC,cAAc,CAACC;YACnB,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOA;YAClC,IAAIA,QAAAA,GAA+B,OAAOC,OAAOD;YACjD,IAAI,AAAa,YAAb,OAAOA,GAAgB,OAAOX,KAAK,SAAS,CAACW;YACjD,OAAOC,OAAOD;QAChB;QACA,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ;YACxB,MAAM,CAACI,KAAKF,EAAE,GAAGF,OAAO,CAAC,EAAE;YAC3B,OAAO,GAAGI,IAAI,EAAE,EAAEH,YAAYC,IAAI;QACpC;QACA,OAAOF,QAAQ,GAAG,CAAC,CAAC,CAACI,KAAKF,EAAE,GAAK,GAAGE,IAAI,EAAE,EAAEH,YAAYC,IAAI,EAAE,IAAI,CAAC;IACrE;IAEA,OAAOC,OAAOb;AAChB"}

package/dist/es/agent/utils.mjs

@@ -120,7 +120,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.9.1";
+const getMidsceneVersion = ()=>"1.9.2-beta-20260605084246.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/inspect.mjs

@@ -1,8 +1,8 @@
 import { generateElementByRect } from "@midscene/shared/extractor";
-import { cropByRect, preProcessImageUrl, scaleImage } from "@midscene/shared/img";
+import { cropByRect, scaleImage } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
-import { expandSearchArea } from "../common.mjs";
+import { expandSearchArea, multimodalPromptToChatMessages, userPromptToMultimodalPrompt, userPromptToString } from "../common.mjs";
 import { extractDataQueryPrompt, parseXMLExtractionResponse, systemPromptToExtract } from "./prompt/extraction.mjs";
 import { findElementPrompt, systemPromptToLocateElement } from "./prompt/llm-locator.mjs";
 import { sectionLocatorInstruction, systemPromptToLocateSection } from "./prompt/llm-section-locator.mjs";
@@ -41,49 +41,6 @@ async function buildSearchAreaConfig(options) {
         }
     };
 }
-const extraTextFromUserPrompt = (prompt)=>{
-    if ('string' == typeof prompt) return prompt;
-    return prompt.prompt;
-};
-const promptsToChatParam = async (multimodalPrompt)=>{
-    const msgs = [];
-    if (multimodalPrompt?.images?.length) {
-        msgs.push({
-            role: 'user',
-            content: [
-                {
-                    type: 'text',
-                    text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.'
-                }
-            ]
-        });
-        for (const item of multimodalPrompt.images){
-            const base64 = await preProcessImageUrl(item.url, !!multimodalPrompt.convertHttpImage2Base64);
-            msgs.push({
-                role: 'user',
-                content: [
-                    {
-                        type: 'text',
-                        text: `this is the reference image named '${item.name}'. It is a reference image, not the current screenshot:`
-                    }
-                ]
-            });
-            msgs.push({
-                role: 'user',
-                content: [
-                    {
-                        type: 'image_url',
-                        image_url: {
-                            url: base64,
-                            detail: 'high'
-                        }
-                    }
-                ]
-            });
-        }
-    }
-    return msgs;
-};
 async function AiLocateElement(options) {
     const { targetElementDescription, ...locateOptions } = options;
     const locateAdapter = options.modelRuntime.adapter.locate;
@@ -97,7 +54,7 @@ async function genericLocate(elementDescription, options) {
     assert('standard' === adapter.locate.kind, 'generic locate requires a standard locate adapter');
     const screenshotBase64 = context.screenshot.base64;
     assert(elementDescription, "cannot find the target element description");
-    const elementDescriptionText = extraTextFromUserPrompt(elementDescription);
+    const elementDescriptionText = userPromptToString(elementDescription);
     const userInstructionPrompt = findElementPrompt(elementDescriptionText);
     const systemPrompt = systemPromptToLocateElement(adapter.locate.resultAdapter.promptSpec);
     const modelImage = options.searchConfig?.image ?? {
@@ -135,10 +92,7 @@ async function genericLocate(elementDescription, options) {
         }
     ];
     if ('string' != typeof elementDescription) {
-        const addOns = await promptsToChatParam({
-            images: elementDescription.images,
-            convertHttpImage2Base64: elementDescription.convertHttpImage2Base64
-        });
+        const addOns = await multimodalPromptToChatMessages(userPromptToMultimodalPrompt(elementDescription));
         msgs.push(...addOns);
     }
     let res;
@@ -221,7 +175,7 @@ async function AiLocateSection(options) {
         policy: adapter.imagePreprocess
     });
     const systemPrompt = systemPromptToLocateSection(adapter.locate.resultAdapter.promptSpec);
-    const sectionLocatorInstructionText = sectionLocatorInstruction(extraTextFromUserPrompt(sectionDescription));
+    const sectionLocatorInstructionText = sectionLocatorInstruction(userPromptToString(sectionDescription));
     const msgs = [
         {
             role: 'system',
@@ -245,10 +199,7 @@ async function AiLocateSection(options) {
         }
     ];
     if ('string' != typeof sectionDescription) {
-        const addOns = await promptsToChatParam({
-            images: sectionDescription.images,
-            convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64
-        });
+        const addOns = await multimodalPromptToChatMessages(userPromptToMultimodalPrompt(sectionDescription));
         msgs.push(...addOns);
     }
     let result;
@@ -344,10 +295,7 @@ async function AiExtractElementInfo(options) {
         }
     ];
     if (multimodalPrompt) {
-        const addOns = await promptsToChatParam({
-            images: multimodalPrompt.images,
-            convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64
-        });
+        const addOns = await multimodalPromptToChatMessages(multimodalPrompt);
         msgs.push(...addOns);
     }
     const { content: rawResponse, usage, reasoning_content } = await callAI(msgs, modelRuntime);
@@ -387,6 +335,6 @@ async function AiJudgeOrderSensitive(description, modelRuntime) {
         usage: result.usage
     };
 }
-export { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, buildSearchAreaConfig, extraTextFromUserPrompt, genericLocate, promptsToChatParam };
+export { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, buildSearchAreaConfig, userPromptToString as extraTextFromUserPrompt, genericLocate, multimodalPromptToChatMessages as promptsToChatParam };
 
 //# sourceMappingURL=inspect.mjs.map

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport {\n  cropByRect,\n  preProcessImageUrl,\n  scaleImage,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { expandSearchArea } from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateOptions,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport const extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  }\n  return prompt.prompt;\n};\n\nexport const promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images. These reference images are supporting context only, not the current screenshot being evaluated, unless the task explicitly asks for comparison or matching.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}'. It is a reference image, not the current screenshot:`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  if (locateAdapter.kind === 'custom') {\n    return locateAdapter.locateFn(targetElementDescription, locateOptions);\n  }\n  return genericLocate(targetElementDescription, locateOptions);\n}\n\nexport async function genericLocate(\n  elementDescription: TUserPrompt,\n  options: LocateOptions,\n): Promise<LocateResult> {\n  const { context } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const screenshotBase64 = context.screenshot.base64;\n\n  assert(elementDescription, 'cannot find the target element description');\n  const elementDescriptionText = extraTextFromUserPrompt(elementDescription);\n  const userInstructionPrompt = findElementPrompt(elementDescriptionText);\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const modelImage = options.searchConfig?.image ?? {\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const preparedImage = await prepareModelImage({\n    imageBase64: modelImage.imageBase64,\n    width: modelImage.width,\n    height: modelImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof elementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: elementDescription.images,\n      convertHttpImage2Base64: elementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  let res: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AIElementLocateResponse>>\n  >;\n  try {\n    res = await callAIWithObjectResponse<AIElementLocateResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'locate',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: [`AI call error: ${errorMessage}`],\n      },\n      rawResponse,\n      usage,\n      reasoning_content: undefined,\n    };\n  }\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElement: LocateResultElement | undefined;\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(res.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors as string[],\n      },\n      rawResponse,\n      usage: res.usage,\n      reasoning_content: res.reasoning_content,\n    };\n  }\n\n  try {\n    const mapping = options.searchConfig?.mapping;\n    const targetPixelBbox = resultAdapter.adaptElementLocateResultToPixelBbox(\n      res.content,\n      {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    );\n    resRect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(targetPixelBbox, mapping),\n    );\n\n    debugInspect('resRect', resRect);\n\n    const element: LocateResultElement = generateElementByRect(\n      resRect,\n      elementDescriptionText as string,\n    );\n    errors = [];\n\n    if (element) {\n      matchedElement = element;\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse locate result: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      element: matchedElement,\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  let result: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n  >;\n  try {\n    result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'section-locator',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse,\n      usage,\n    };\n  }\n\n  let searchAreaConfig:\n    | Awaited<ReturnType<typeof buildSearchAreaConfig>>\n    | undefined;\n  let sectionError = result.content.error;\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: JSON.stringify(result.content),\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const adaptedResult =\n      resultAdapter.adaptSectionLocateResultToPixelBboxGroup(result.content, {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      });\n    const mergedRect = mergePixelBboxesToRect([\n      adaptedResult.target,\n      ...(adaptedResult.references ?? []),\n    ]);\n    debugSection('mergedRect %j', mergedRect);\n\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    sectionError = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n  }\n\n  return {\n    searchAreaConfig,\n    error: sectionError,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'text',\n      text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n    });\n\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n  } = await callAI(msgs, modelRuntime);\n\n  let parseResult: AIDataExtractionResponse<T>;\n  try {\n    parseResult = parseXMLExtractionResponse<T>(rawResponse);\n  } catch (parseError) {\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n    );\n  }\n\n  return {\n    parseResult,\n    rawResponse,\n    usage,\n    reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","msgs","item","base64","preProcessImageUrl","AiLocateElement","targetElementDescription","locateOptions","locateAdapter","genericLocate","elementDescription","modelRuntime","adapter","assert","screenshotBase64","elementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","modelImage","preparedImage","prepareModelImage","imagePayload","addOns","res","callAIWithObjectResponse","callError","errorMessage","Error","String","rawResponse","AIResponseParseError","usage","JSON","resRect","matchedElement","errors","resultAdapter","mapping","targetPixelBbox","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","element","generateElementByRect","e","msg","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","searchAreaConfig","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","error","parseErrorMessage","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","reasoning_content","callAI","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;AAgEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAEO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,MAAMO,0BAA0B,CAACC;IACtC,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAET,OAAOA,OAAO,MAAM;AACtB;AAEO,MAAMC,qBAAqB,OAChCC;IAEA,MAAMC,OAAyC,EAAE;IACjD,IAAID,kBAAkB,QAAQ,QAAQ;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQF,iBAAiB,MAAM,CAAE;YAC1C,MAAMG,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACF,iBAAiB,uBAAuB;YAG5CC,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,uDAAuD,CAAC;oBAChH;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBACpBlB,OAAkE;IAElE,MAAM,EAAEmB,wBAAwB,EAAE,GAAGC,eAAe,GAAGpB;IACvD,MAAMqB,gBAAgBrB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,IAAIqB,AAAuB,aAAvBA,cAAc,IAAI,EACpB,OAAOA,cAAc,QAAQ,CAACF,0BAA0BC;IAE1D,OAAOE,cAAcH,0BAA0BC;AACjD;AAEO,eAAeE,cACpBC,kBAA+B,EAC/BvB,OAAsB;IAEtB,MAAM,EAAEC,OAAO,EAAE,GAAGD;IACpB,MAAMwB,eAAexB,QAAQ,YAAY;IACzC,MAAM,EAAEyB,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,mBAAmB1B,QAAQ,UAAU,CAAC,MAAM;IAElDyB,OAAOH,oBAAoB;IAC3B,MAAMK,yBAAyBlB,wBAAwBa;IACvD,MAAMM,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,4BACnBP,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMQ,aAAajC,QAAQ,YAAY,EAAE,SAAS;QAChD,aAAa2B;QACb,OAAO1B,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMiC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaF,WAAW,WAAW;QACnC,OAAOA,WAAW,KAAK;QACvB,QAAQA,WAAW,MAAM;QACzB,QAAQR,QAAQ,eAAe;IACjC;IAEA,MAAMW,eAAeF,cAAc,WAAW;IAE9C,MAAMpB,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKK;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMP;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAON,oBAAiC;QAC1C,MAAMc,SAAS,MAAMzB,mBAAmB;YACtC,QAAQW,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACAT,KAAK,IAAI,IAAIuB;IACf;IAEA,IAAIC;IAGJ,IAAI;QACFA,MAAM,MAAMC,yBACVzB,MACAU,cACA;YACE,aAAaxB,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOwC,WAAW;QAClB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMK,QACJN,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAG1C;QAChE,OAAO;YACL,MAAMA;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ;oBAAC,CAAC,eAAe,EAAE2C,cAAc;iBAAC;YAC5C;YACAG;YACAE;YACA,mBAAmBhD;QACrB;IACF;IAEA,MAAM8C,cAAcG,KAAK,SAAS,CAACT,IAAI,OAAO;IAE9C,IAAIU;IACJ,IAAIC;IACJ,IAAIC,SACF,YAAYZ,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,MAAMa,gBAAgB1B,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAACjC,gBAAgB8C,IAAI,OAAO,EAAEa,cAAc,UAAU,CAAC,SAAS,GAClE,OAAO;QACL,MAAMrD;QACN,aAAa;YACX,SAASA;YACT,QAAQoD;QACV;QACAN;QACA,OAAON,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;IAGF,IAAI;QACF,MAAMc,UAAUpD,QAAQ,YAAY,EAAE;QACtC,MAAMqD,kBAAkBF,cAAc,mCAAmC,CACvEb,IAAI,OAAO,EACX;YACE,cAAcJ,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QAEFc,UAAUM,gBACRC,0CAA0CF,iBAAiBD;QAG7D/D,aAAa,WAAW2D;QAExB,MAAMQ,UAA+BC,sBACnCT,SACApB;QAEFsB,SAAS,EAAE;QAEX,IAAIM,SACFP,iBAAiBO;IAErB,EAAE,OAAOE,GAAG;QACV,MAAMC,MACJD,aAAahB,QACT,CAAC,+BAA+B,EAAEgB,EAAE,OAAO,EAAE,GAC7C;QACN,IAAI,AAACR,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAES,IAAI,CAAC,CAAC;aAFtBT,SAAS;YAACS;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMX;QACN,aAAa;YACX,SAASC;YACT,QAAQC;QACV;QACAN;QACA,OAAON,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAesB,gBAAgB5D,OAKrC;IAMC,MAAM,EAAEC,OAAO,EAAE4D,kBAAkB,EAAE,GAAG7D;IACxC,MAAMwB,eAAexB,QAAQ,YAAY;IACzC,MAAM,EAAEyB,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,mBAAmB1B,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAMiC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaR;QACb,OAAO1B,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQwB,QAAQ,eAAe;IACjC;IAEA,MAAMM,eAAe+B,4BACnBrC,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMsC,gCAAgCC,0BACpCtD,wBAAwBmD;IAE1B,MAAM/C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKG,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAM6B;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMxB,SAAS,MAAMzB,mBAAmB;YACtC,QAAQiD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA/C,KAAK,IAAI,IAAIuB;IACf;IAEA,IAAI4B;IAGJ,IAAI;QACFA,SAAS,MAAM1B,yBACbzB,MACAU,cACA;YACE,aAAaxB,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOwC,WAAW;QAClB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMK,QACJN,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAG1C;QAChE,OAAO;YACL,kBAAkBA;YAClB,OAAO,CAAC,eAAe,EAAE2C,cAAc;YACvCG;YACAE;QACF;IACF;IAEA,IAAIoB;IAGJ,IAAIC,eAAeF,OAAO,OAAO,CAAC,KAAK;IACvC,MAAMd,gBAAgB1B,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAACjC,gBAAgByE,OAAO,OAAO,EAAEd,cAAc,UAAU,CAAC,SAAS,GACrE,OAAO;QACL,kBAAkBrD;QAClB,OAAOqE;QACP,aAAapB,KAAK,SAAS,CAACkB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMG,gBACJjB,cAAc,wCAAwC,CAACc,OAAO,OAAO,EAAE;YACrE,cAAc/B,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACF,MAAMmC,aAAaC,uBAAuB;YACxCF,cAAc,MAAM;eAChBA,cAAc,UAAU,IAAI,EAAE;SACnC;QACD7E,aAAa,iBAAiB8E;QAE9B,MAAME,eAAelE,iBAAiBgE,YAAYpE,QAAQ,QAAQ;QAClE,MAAMuE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1ChF,aAAa,2BAA2BgF;QAExCL,mBAAmB,MAAMnE,sBAAsB;YAC7CE;YACA,UAAUoE;QACZ;QAEA9E,aACE,uDACAiF,eACAC,gBACAP,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;IAElC,EAAE,OAAOQ,OAAO;QACd,MAAMC,oBACJD,iBAAiBhC,QACb,CAAC,uCAAuC,EAAEgC,MAAM,OAAO,EAAE,GACzD;QACNP,eAAeA,eACX,GAAGA,aAAa,EAAE,EAAEQ,kBAAkB,CAAC,CAAC,GACxCA;IACN;IAEA,OAAO;QACLT;QACA,OAAOC;QACP,aAAapB,KAAK,SAAS,CAACkB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeW,qBAAwB5E,OAO7C;IACC,MAAM,EAAE6E,SAAS,EAAE5E,OAAO,EAAE6E,aAAa,EAAEjE,gBAAgB,EAAEW,YAAY,EAAE,GACzExB;IACF,MAAM+B,eAAegD,sBAAsB;QACzC,oBAAoBD,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACjE,kBAAkB,QAAQ;IACvD;IACA,MAAMc,mBAAmB1B,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM+E,wBAAwBC,uBAC5BjF,QAAQ,eAAe,IAAI,IAC3B6E;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,eAAe,uBAAuB,OAAO;QAC/CI,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QAEAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBACT,KAAKvD;gBACL,QAAQ;YACV;QACF;IACF;IAEAuD,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMlE,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAASmD;QACX;KACD;IAED,IAAIrE,kBAAkB;QACpB,MAAMwB,SAAS,MAAMzB,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAC,KAAK,IAAI,IAAIuB;IACf;IAEA,MAAM,EACJ,SAASO,WAAW,EACpBE,KAAK,EACLqC,iBAAiB,EAClB,GAAG,MAAMC,OAAOtE,MAAMU;IAEvB,IAAI6D;IACJ,IAAI;QACFA,cAAcC,2BAA8B1C;IAC9C,EAAE,OAAO2C,YAAY;QACnB,MAAM9C,eACJ8C,sBAAsB7C,QAAQ6C,WAAW,OAAO,GAAG5C,OAAO4C;QAC5D,MAAM,IAAI1C,qBACR,CAAC,iBAAiB,EAAEJ,cAAc,EAClCG,aACAE;IAEJ;IAEA,OAAO;QACLuC;QACAzC;QACAE;QACAqC;IACF;AACF;AAEO,eAAeK,sBACpBC,WAAmB,EACnBjE,YAA0B;IAK1B,MAAMO,eAAe2D;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAM3E,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASiB;QAAa;QACxC;YACE,MAAM;YACN,SAAS4D;QACX;KACD;IAEDtG,aAAa,yCAAyCoG;IAEtD,MAAMxB,SAAS,MAAM1B,yBACnBzB,MACAU,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkByC,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateOptions,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  if (locateAdapter.kind === 'custom') {\n    return locateAdapter.locateFn(targetElementDescription, locateOptions);\n  }\n  return genericLocate(targetElementDescription, locateOptions);\n}\n\nexport async function genericLocate(\n  elementDescription: TUserPrompt,\n  options: LocateOptions,\n): Promise<LocateResult> {\n  const { context } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const screenshotBase64 = context.screenshot.base64;\n\n  assert(elementDescription, 'cannot find the target element description');\n  const elementDescriptionText = userPromptToString(elementDescription);\n  const userInstructionPrompt = findElementPrompt(elementDescriptionText);\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const modelImage = options.searchConfig?.image ?? {\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const preparedImage = await prepareModelImage({\n    imageBase64: modelImage.imageBase64,\n    width: modelImage.width,\n    height: modelImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof elementDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(elementDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let res: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AIElementLocateResponse>>\n  >;\n  try {\n    res = await callAIWithObjectResponse<AIElementLocateResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'locate',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: [`AI call error: ${errorMessage}`],\n      },\n      rawResponse,\n      usage,\n      reasoning_content: undefined,\n    };\n  }\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElement: LocateResultElement | undefined;\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(res.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors as string[],\n      },\n      rawResponse,\n      usage: res.usage,\n      reasoning_content: res.reasoning_content,\n    };\n  }\n\n  try {\n    const mapping = options.searchConfig?.mapping;\n    const targetPixelBbox = resultAdapter.adaptElementLocateResultToPixelBbox(\n      res.content,\n      {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    );\n    resRect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(targetPixelBbox, mapping),\n    );\n\n    debugInspect('resRect', resRect);\n\n    const element: LocateResultElement = generateElementByRect(\n      resRect,\n      elementDescriptionText as string,\n    );\n    errors = [];\n\n    if (element) {\n      matchedElement = element;\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse locate result: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      element: matchedElement,\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let result: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n  >;\n  try {\n    result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'section-locator',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse,\n      usage,\n    };\n  }\n\n  let searchAreaConfig:\n    | Awaited<ReturnType<typeof buildSearchAreaConfig>>\n    | undefined;\n  let sectionError = result.content.error;\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: JSON.stringify(result.content),\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const adaptedResult =\n      resultAdapter.adaptSectionLocateResultToPixelBboxGroup(result.content, {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      });\n    const mergedRect = mergePixelBboxesToRect([\n      adaptedResult.target,\n      ...(adaptedResult.references ?? []),\n    ]);\n    debugSection('mergedRect %j', mergedRect);\n\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    sectionError = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n  }\n\n  return {\n    searchAreaConfig,\n    error: sectionError,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'text',\n      text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n    });\n\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  const {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n  } = await callAI(msgs, modelRuntime);\n\n  let parseResult: AIDataExtractionResponse<T>;\n  try {\n    parseResult = parseXMLExtractionResponse<T>(rawResponse);\n  } catch (parseError) {\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n    );\n  }\n\n  return {\n    parseResult,\n    rawResponse,\n    usage,\n    reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","locateAdapter","genericLocate","elementDescription","modelRuntime","adapter","assert","screenshotBase64","elementDescriptionText","userPromptToString","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","modelImage","preparedImage","prepareModelImage","imagePayload","msgs","addOns","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","res","callAIWithObjectResponse","callError","errorMessage","Error","String","rawResponse","AIResponseParseError","usage","JSON","resRect","matchedElement","errors","resultAdapter","mapping","targetPixelBbox","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","element","generateElementByRect","e","msg","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","searchAreaConfig","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","error","parseErrorMessage","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","reasoning_content","callAI","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;AAiEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAEO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvD,MAAMa,gBAAgBb,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,IAAIa,AAAuB,aAAvBA,cAAc,IAAI,EACpB,OAAOA,cAAc,QAAQ,CAACF,0BAA0BC;IAE1D,OAAOE,cAAcH,0BAA0BC;AACjD;AAEO,eAAeE,cACpBC,kBAA+B,EAC/Bf,OAAsB;IAEtB,MAAM,EAAEC,OAAO,EAAE,GAAGD;IACpB,MAAMgB,eAAehB,QAAQ,YAAY;IACzC,MAAM,EAAEiB,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,mBAAmBlB,QAAQ,UAAU,CAAC,MAAM;IAElDiB,OAAOH,oBAAoB;IAC3B,MAAMK,yBAAyBC,mBAAmBN;IAClD,MAAMO,wBAAwBC,kBAAkBH;IAChD,MAAMI,eAAeC,4BACnBR,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMS,aAAa1B,QAAQ,YAAY,EAAE,SAAS;QAChD,aAAamB;QACb,OAAOlB,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAM0B,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaF,WAAW,WAAW;QACnC,OAAOA,WAAW,KAAK;QACvB,QAAQA,WAAW,MAAM;QACzB,QAAQT,QAAQ,eAAe;IACjC;IAEA,MAAMY,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASN;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKK;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMP;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOP,oBAAiC;QAC1C,MAAMgB,SAAS,MAAMC,+BACnBC,6BAA6BlB;QAE/Be,KAAK,IAAI,IAAIC;IACf;IAEA,IAAIG;IAGJ,IAAI;QACFA,MAAM,MAAMC,yBACVL,MACAd,cACA;YACE,aAAahB,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOoC,WAAW;QAClB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMK,QACJN,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGtC;QAChE,OAAO;YACL,MAAMA;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ;oBAAC,CAAC,eAAe,EAAEuC,cAAc;iBAAC;YAC5C;YACAG;YACAE;YACA,mBAAmB5C;QACrB;IACF;IAEA,MAAM0C,cAAcG,KAAK,SAAS,CAACT,IAAI,OAAO;IAE9C,IAAIU;IACJ,IAAIC;IACJ,IAAIC,SACF,YAAYZ,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,MAAMa,gBAAgB9B,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAACzB,gBAAgB0C,IAAI,OAAO,EAAEa,cAAc,UAAU,CAAC,SAAS,GAClE,OAAO;QACL,MAAMjD;QACN,aAAa;YACX,SAASA;YACT,QAAQgD;QACV;QACAN;QACA,OAAON,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;IAGF,IAAI;QACF,MAAMc,UAAUhD,QAAQ,YAAY,EAAE;QACtC,MAAMiD,kBAAkBF,cAAc,mCAAmC,CACvEb,IAAI,OAAO,EACX;YACE,cAAcP,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QAEFiB,UAAUM,gBACRC,0CAA0CF,iBAAiBD;QAG7D3D,aAAa,WAAWuD;QAExB,MAAMQ,UAA+BC,sBACnCT,SACAxB;QAEF0B,SAAS,EAAE;QAEX,IAAIM,SACFP,iBAAiBO;IAErB,EAAE,OAAOE,GAAG;QACV,MAAMC,MACJD,aAAahB,QACT,CAAC,+BAA+B,EAAEgB,EAAE,OAAO,EAAE,GAC7C;QACN,IAAI,AAACR,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAES,IAAI,CAAC,CAAC;aAFtBT,SAAS;YAACS;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMX;QACN,aAAa;YACX,SAASC;YACT,QAAQC;QACV;QACAN;QACA,OAAON,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAesB,gBAAgBxD,OAKrC;IAMC,MAAM,EAAEC,OAAO,EAAEwD,kBAAkB,EAAE,GAAGzD;IACxC,MAAMgB,eAAehB,QAAQ,YAAY;IACzC,MAAM,EAAEiB,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,mBAAmBlB,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM0B,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaT;QACb,OAAOlB,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQgB,QAAQ,eAAe;IACjC;IAEA,MAAMO,eAAekC,4BACnBzC,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAM0C,gCAAgCC,0BACpCvC,mBAAmBoC;IAErB,MAAM3B,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASN;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKG,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMgC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM1B,SAAS,MAAMC,+BACnBC,6BAA6BwB;QAE/B3B,KAAK,IAAI,IAAIC;IACf;IAEA,IAAI8B;IAGJ,IAAI;QACFA,SAAS,MAAM1B,yBACbL,MACAd,cACA;YACE,aAAahB,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOoC,WAAW;QAClB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMK,QACJN,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGtC;QAChE,OAAO;YACL,kBAAkBA;YAClB,OAAO,CAAC,eAAe,EAAEuC,cAAc;YACvCG;YACAE;QACF;IACF;IAEA,IAAIoB;IAGJ,IAAIC,eAAeF,OAAO,OAAO,CAAC,KAAK;IACvC,MAAMd,gBAAgB9B,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAACzB,gBAAgBqE,OAAO,OAAO,EAAEd,cAAc,UAAU,CAAC,SAAS,GACrE,OAAO;QACL,kBAAkBjD;QAClB,OAAOiE;QACP,aAAapB,KAAK,SAAS,CAACkB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMG,gBACJjB,cAAc,wCAAwC,CAACc,OAAO,OAAO,EAAE;YACrE,cAAclC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACF,MAAMsC,aAAaC,uBAAuB;YACxCF,cAAc,MAAM;eAChBA,cAAc,UAAU,IAAI,EAAE;SACnC;QACDzE,aAAa,iBAAiB0E;QAE9B,MAAME,eAAe9D,iBAAiB4D,YAAYhE,QAAQ,QAAQ;QAClE,MAAMmE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1C5E,aAAa,2BAA2B4E;QAExCL,mBAAmB,MAAM/D,sBAAsB;YAC7CE;YACA,UAAUgE;QACZ;QAEA1E,aACE,uDACA6E,eACAC,gBACAP,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;IAElC,EAAE,OAAOQ,OAAO;QACd,MAAMC,oBACJD,iBAAiBhC,QACb,CAAC,uCAAuC,EAAEgC,MAAM,OAAO,EAAE,GACzD;QACNP,eAAeA,eACX,GAAGA,aAAa,EAAE,EAAEQ,kBAAkB,CAAC,CAAC,GACxCA;IACN;IAEA,OAAO;QACLT;QACA,OAAOC;QACP,aAAapB,KAAK,SAAS,CAACkB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeW,qBAAwBxE,OAO7C;IACC,MAAM,EAAEyE,SAAS,EAAExE,OAAO,EAAEyE,aAAa,EAAEC,gBAAgB,EAAE3D,YAAY,EAAE,GACzEhB;IACF,MAAMwB,eAAeoD,sBAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMxD,mBAAmBlB,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM4E,wBAAwBC,uBAC5B9E,QAAQ,eAAe,IAAI,IAC3ByE;IAGF,MAAMM,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/CK,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QAEAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBACT,KAAK5D;gBACL,QAAQ;YACV;QACF;IACF;IAEA4D,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAM/C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASN;QAAa;QACxC;YACE,MAAM;YACN,SAASuD;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAM5C,SAAS,MAAMC,+BAA+B2C;QACpD7C,KAAK,IAAI,IAAIC;IACf;IAEA,MAAM,EACJ,SAASS,WAAW,EACpBE,KAAK,EACLsC,iBAAiB,EAClB,GAAG,MAAMC,OAAOnD,MAAMd;IAEvB,IAAIkE;IACJ,IAAI;QACFA,cAAcC,2BAA8B3C;IAC9C,EAAE,OAAO4C,YAAY;QACnB,MAAM/C,eACJ+C,sBAAsB9C,QAAQ8C,WAAW,OAAO,GAAG7C,OAAO6C;QAC5D,MAAM,IAAI3C,qBACR,CAAC,iBAAiB,EAAEJ,cAAc,EAClCG,aACAE;IAEJ;IAEA,OAAO;QACLwC;QACA1C;QACAE;QACAsC;IACF;AACF;AAEO,eAAeK,sBACpBC,WAAmB,EACnBtE,YAA0B;IAK1B,MAAMQ,eAAe+D;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMxD,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASN;QAAa;QACxC;YACE,MAAM;YACN,SAASgE;QACX;KACD;IAEDnG,aAAa,yCAAyCiG;IAEtD,MAAMzB,SAAS,MAAM1B,yBACnBL,MACAd,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB6C,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}