@midscene/core 1.6.2 → 1.6.3-beta-20260403070857.0

package/dist/es/agent/tasks.mjs

@@ -297,10 +297,9 @@ class TaskExecutor {
                 if (ifTypeRestricted) if ('string' == typeof data) outputResult = data;
                 else if ('WaitFor' === type) outputResult = null == data ? false : data[keyOfResult];
                 else if (null == data) outputResult = null;
-                else {
-                    assert(data?.[keyOfResult] !== void 0, 'No result in query data');
-                    outputResult = data[keyOfResult];
-                }
+                else if (data?.[keyOfResult] !== void 0) outputResult = data[keyOfResult];
+                else if (data?.result !== void 0) outputResult = data.result;
+                else assert(false, 'No result in query data');
                 if ('Assert' === type && !outputResult) {
                     task.thought = thought;
                     throw new Error(`Assertion failed: ${thought}`);

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import {\n  AIResponseParseError,\n  ConversationHistory,\n  autoGLMPlanning,\n  plan,\n  uiTarsPlanning,\n} from '@/ai-model';\nimport { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeepThinkOption,\n  DeviceAction,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { type IModelConfig, getCurrentTime } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTimestamp?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTimestamp?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTimestamp = opts.useDeviceTimestamp;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Get a readable time string using device time when configured.\n   * This method respects the useDeviceTimestamp configuration.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    const timestamp = await getCurrentTime(\n      this.interface,\n      this.useDeviceTimestamp,\n    );\n    return getReadableTimeString(format, timestamp);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      options,\n    );\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: DeepThinkOption,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        modelConfigForPlanning,\n        modelConfigForDefaultIntent,\n        includeBboxInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n      );\n    });\n  }\n\n  private async runAction(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: DeepThinkOption,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    const conversationHistory = new ConversationHistory();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const { modelFamily } = modelConfigForPlanning;\n            const timing = executorContext.task.timing;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl = isUITars(modelFamily)\n              ? uiTarsPlanning\n              : isAutoGLM(modelFamily)\n                ? autoGLMPlanning\n                : plan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: uiContext,\n                actionContext: param.aiActContext,\n                interfaceType: this.interface.interfaceType as InterfaceType,\n                actionSpace,\n                modelConfig: modelConfigForPlanning,\n                conversationHistory,\n                includeBbox: includeBboxInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = planError.usage;\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                };\n              }\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n            };\n            executorContext.task.usage = usage;\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = uiContext;\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              assert(\n                false,\n                `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`,\n              );\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          modelConfigForPlanning,\n          modelConfigForDefaultIntent,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Set initial time context for the first planning call\n      const initialTimeString = await this.getTimeString();\n      conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;\n\n      try {\n        await session.appendAndRun(executables.tasks);\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message || String(error)}`;\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelConfig,\n          modelConfig,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["debug","getDebug","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","format","timestamp","getCurrentTime","getReadableTimeString","plans","modelConfigForPlanning","modelConfigForDefaultIntent","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeBboxInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","conversationHistory","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","outputString","subGoalStatus","memoriesStatus","modelFamily","timing","actionSpace","action","Array","console","planImpl","isUITars","uiTarsPlanning","isAutoGLM","autoGLMPlanning","plan","planResult","setTimingFieldOnce","planError","AIResponseParseError","JSON","actions","thought","log","memory","error","usage","rawResponse","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","timeString","String","Error","errorMsg","type","demand","modelConfig","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA2DA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,wCAAwC;AAIvC,MAAMC;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IAQA,MAAc,cAAcG,MAAe,EAAmB;QAC5D,MAAMC,YAAY,MAAMC,eACtB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,kBAAkB;QAEzB,OAAOC,sBAAsBH,QAAQC;IACvC;IAEA,MAAa,wBACXG,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCT,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAC3BO,OACAC,wBACAC,6BACAT;IAEJ;IAEA,MAAM,uBAAuBU,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOH;QAGtB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJpB,KAAa,EACbQ,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACf;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACb;QAC5C,MAAM,EAAEqB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,wBACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAA2B,EAC3BC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EASzB;QACA,OAAOC,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAf,wBACAC,6BACAe,uBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC;IAGN;IAEA,MAAc,UACZT,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAA2B,EAC3BE,UAAoB,EACpBC,WAAyB,EASzB;QACA,MAAME,sBAAsB,IAAIC;QAEhC,MAAMvB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOU;QAEtB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIwB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJX,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DT,OACEoB,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAClC,IAAIC;QAGJ,MAAO,KAAM;YAEX,IAAIT,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMU,gBAAgBR,oBAAoB,cAAc,MAAMK;YAG9D,MAAMI,iBAAiBT,oBAAoB,cAAc,MAAMK;YAE/D,MAAMlB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjBE;oBACAG;oBACAC;oBACA,GAAIa,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO5B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM,EAAE2B,WAAW,EAAE,GAAGpC;oBACxB,MAAMqC,SAAS7B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAM8B,cAAc,IAAI,CAAC,cAAc;oBACvCnD,MACE,sCACAmD,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD7B,OAAO8B,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WAAWC,SAASP,eACtBQ,iBACAC,UAAUT,eACRU,kBACAC;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBZ,QAAQ;wBAC3BW,aAAa,MAAMN,SAASnC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;4BAC3C+B;4BACA,aAAatC;4BACb0B;4BACA,aAAaV;4BACbI;4BACAC;4BACAG;wBACF;oBACF,EAAE,OAAO0B,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7C3C,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,UAAU,KAAK;4BAC5C1C,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAa0C,UAAU,WAAW;4BACpC;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBZ,QAAQ;oBAC7B;oBACAlD,MAAM,cAAciE,KAAK,SAAS,CAACJ,YAAY,MAAM;oBAErD,MAAM,EACJK,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACNC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGhB;oBACJf,eAAe6B;oBAEftD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCmD;oBACF;oBACAnD,gBAAgB,IAAI,CAAC,KAAK,GAAGkD;oBAC7BlD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGoD;oBACzCpD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS6C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUR,WAAW,QAAQ;wBAC7B,QAAQc;wBACR,wBAAwBd,WAAW,sBAAsB;wBACzDe;wBACAC;oBACF;oBACAxD,gBAAgB,SAAS,GAAGC;oBAE5BC,OAAO,CAAC+C,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEF,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBACFnD,OACE,OACA,CAAC,aAAa,EAAEoD,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMP,aAAanC,QAAQ;YAG3B,MAAMd,QAAQiD,YAAY,WAAW,EAAE;YACvCnB,SAAS,IAAI,IAAKmB,YAAY,YAAY,EAAE;YAE5C,IAAIiB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9ClE,OACAC,wBACAC,6BACA;oBACEiB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAOiC,OAAO;gBACd,OAAOrD,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEqD,MAAM,SAAS,EAAEL,KAAK,SAAS,CAC5ErD,QACC;YAEP;YACA,IAAI2B,oBAAoB,sBAAsB,EAC5Ce,QAAQ,IAAI,CACV,8FACAf,oBAAoB,sBAAsB;YAK9C,MAAMwC,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAClDxC,oBAAoB,sBAAsB,IAAI,CAAC,cAAc,EAAEwC,mBAAmB;YAElF,IAAI;gBACF,MAAM9D,QAAQ,YAAY,CAAC6D,YAAY,KAAK;YAC9C,EAAE,OAAOR,OAAY;gBAEnBzB;gBACA,MAAMmC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,iCAAiC,EAAEV,OAAO,WAAWW,OAAOX,QAAQ;gBACrItE,MACE,yFACAsE,iBAAiBY,QAAQZ,MAAM,OAAO,GAAGW,OAAOX,QAChD,6CACAzB;YAEJ;YAEA,IAAIA,8BAA8B3C,uCAChC,OAAOe,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACwB,YAAY,wBACf;YAIF,EAAEpB;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMwC,WAAW,CAAC,UAAU,EAAExC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO1B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAC5C,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMyC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACNtC;gBACA,QAAQI;YACV;YACAtB;QACF;IACF;IAEQ,oBACN4D,IAAsE,EACtEC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOjE,OAAOsE;gBACtB,MAAM,EAAEvE,IAAI,EAAE,GAAGuE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ1E,KAAK,GAAG,GAAG;wBACT0E;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1E,KAAK,KAAK,GAAG0E,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,mBACjB1E,KAAK,iBAAiB,GAAG0E,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMvE,YAAYoE,YAAY,SAAS;gBACvCnE,OAAOD,WAAW;gBAElB,MAAMwE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACd,MAAMC,gBACJb,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIU,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAE,GAAGZ,KAAK,EAAE,EAAEC,QAAQ;oBACrC;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DvF,MAAM;oBACN,MAAMoG,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,aACAC,KACAY,sBACAX,kBACAlE;gBAEJ,EAAE,OAAOgD,OAAO;oBACd,IAAIA,iBAAiBgC,cACnBV,UAAUtB,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEiC,IAAI,EAAEpC,OAAO,EAAE0B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBACV;oBACLjF,OACEgF,MAAM,CAACP,YAAY,KAAKpD,QACxB;oBAEF4D,eAAgBD,IAAY,CAACP,YAAY;gBAC3C;gBAGF,IAAIZ,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCrF,KAAK,OAAO,GAAGgD;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQqC;oBACR,KAAKb;oBACLxB;gBACF;YACF;QACF;QAEA,OAAOsB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,aACAC,KACAC;QAGF,MAAMhE,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACwE;QAE1C,IAAI,CAAC/D,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEwC,OAAO,EAAE,GAAGzC;QAE5B,OAAO;YACLC;YACAwC;YACA3C;QACF;IACF;IAEA,MAAM,QACJiF,SAAsB,EACtBlB,GAA+B,EAC/BD,WAAyB,EACO;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMzF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW0F;QAE1B,MAAMpF,SAASP,QAAQ,SAAS;QAChC,MAAM,EACJ4F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA1F,OAAOkF,WAAW;QAClBlF,OAAOsF,WAAW;QAClBtF,OAAOuF,iBAAiB;QAExBvF,OACEuF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,aACA4B,mBACA1B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACwE;YAO3C,IAAI/D,QAAQ,QACV,OAAO;gBACL,QAAQkB;gBACRpB;YACF;YAGF8F,eACE5F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEgF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMtD,UAAU,CAAC,kBAAkB,EAAE2C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGvD;oBAAQ;iBAAE,EAC9DmB,aACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAM1G,QAAQ,YAAY,CAAC0G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO1G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEqG,cAAc;IACnE;IA9rBA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,kBAAkB,GAAGA,KAAK,kBAAkB;QACjD,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAkqBF;AAEO,eAAexF,gBACpBsF,iBAAoC,EACpCzF,iBAAuC,EACvCiB,MAAwB;IAExB,IAAI,CAACjB,mBAAmB,QACtB,OAAOiB;IAGT,IAAI,CAACwE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC9F;IACvB;IAEA,MAAM,EAAE+F,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMtG,SAAS,MAAM0B;QAErB,MAAMkB,QAAQ6D;QACd,IAAI7D,OACF,MAAMA;QAER,OAAO5C;IACT,SAAU;QACRwG;IACF;AACF"}
+{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import {\n  AIResponseParseError,\n  ConversationHistory,\n  autoGLMPlanning,\n  plan,\n  uiTarsPlanning,\n} from '@/ai-model';\nimport { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeepThinkOption,\n  DeviceAction,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { type IModelConfig, getCurrentTime } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTimestamp?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTimestamp?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTimestamp = opts.useDeviceTimestamp;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Get a readable time string using device time when configured.\n   * This method respects the useDeviceTimestamp configuration.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    const timestamp = await getCurrentTime(\n      this.interface,\n      this.useDeviceTimestamp,\n    );\n    return getReadableTimeString(format, timestamp);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      options,\n    );\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: DeepThinkOption,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        modelConfigForPlanning,\n        modelConfigForDefaultIntent,\n        includeBboxInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n      );\n    });\n  }\n\n  private async runAction(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: DeepThinkOption,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    const conversationHistory = new ConversationHistory();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const { modelFamily } = modelConfigForPlanning;\n            const timing = executorContext.task.timing;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl = isUITars(modelFamily)\n              ? uiTarsPlanning\n              : isAutoGLM(modelFamily)\n                ? autoGLMPlanning\n                : plan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: uiContext,\n                actionContext: param.aiActContext,\n                interfaceType: this.interface.interfaceType as InterfaceType,\n                actionSpace,\n                modelConfig: modelConfigForPlanning,\n                conversationHistory,\n                includeBbox: includeBboxInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = planError.usage;\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                };\n              }\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n            };\n            executorContext.task.usage = usage;\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = uiContext;\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              assert(\n                false,\n                `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`,\n              );\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          modelConfigForPlanning,\n          modelConfigForDefaultIntent,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Set initial time context for the first planning call\n      const initialTimeString = await this.getTimeString();\n      conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;\n\n      try {\n        await session.appendAndRun(executables.tasks);\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message || String(error)}`;\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        domIncluded: opt?.domIncluded,\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n            if (data?.[keyOfResult] !== undefined) {\n              outputResult = (data as any)[keyOfResult];\n            } else if (data?.result !== undefined) {\n              outputResult = (data as any).result;\n            } else {\n              assert(false, 'No result in query data');\n            }\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelConfig,\n          modelConfig,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["debug","getDebug","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","format","timestamp","getCurrentTime","getReadableTimeString","plans","modelConfigForPlanning","modelConfigForDefaultIntent","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeBboxInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","conversationHistory","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","outputString","subGoalStatus","memoriesStatus","modelFamily","timing","actionSpace","action","Array","console","planImpl","isUITars","uiTarsPlanning","isAutoGLM","autoGLMPlanning","plan","planResult","setTimingFieldOnce","planError","AIResponseParseError","JSON","actions","thought","log","memory","error","usage","rawResponse","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","timeString","String","Error","errorMsg","type","demand","modelConfig","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA2DA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,wCAAwC;AAIvC,MAAMC;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IAQA,MAAc,cAAcG,MAAe,EAAmB;QAC5D,MAAMC,YAAY,MAAMC,eACtB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,kBAAkB;QAEzB,OAAOC,sBAAsBH,QAAQC;IACvC;IAEA,MAAa,wBACXG,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCT,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAC3BO,OACAC,wBACAC,6BACAT;IAEJ;IAEA,MAAM,uBAAuBU,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOH;QAGtB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJpB,KAAa,EACbQ,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACf;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACb;QAC5C,MAAM,EAAEqB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,wBACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAA2B,EAC3BC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EASzB;QACA,OAAOC,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAf,wBACAC,6BACAe,uBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC;IAGN;IAEA,MAAc,UACZT,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAA2B,EAC3BE,UAAoB,EACpBC,WAAyB,EASzB;QACA,MAAME,sBAAsB,IAAIC;QAEhC,MAAMvB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOU;QAEtB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIwB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJX,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DT,OACEoB,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAClC,IAAIC;QAGJ,MAAO,KAAM;YAEX,IAAIT,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMU,gBAAgBR,oBAAoB,cAAc,MAAMK;YAG9D,MAAMI,iBAAiBT,oBAAoB,cAAc,MAAMK;YAE/D,MAAMlB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjBE;oBACAG;oBACAC;oBACA,GAAIa,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO5B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM,EAAE2B,WAAW,EAAE,GAAGpC;oBACxB,MAAMqC,SAAS7B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAM8B,cAAc,IAAI,CAAC,cAAc;oBACvCnD,MACE,sCACAmD,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD7B,OAAO8B,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WAAWC,SAASP,eACtBQ,iBACAC,UAAUT,eACRU,kBACAC;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBZ,QAAQ;wBAC3BW,aAAa,MAAMN,SAASnC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;4BAC3C+B;4BACA,aAAatC;4BACb0B;4BACA,aAAaV;4BACbI;4BACAC;4BACAG;wBACF;oBACF,EAAE,OAAO0B,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7C3C,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,UAAU,KAAK;4BAC5C1C,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAa0C,UAAU,WAAW;4BACpC;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBZ,QAAQ;oBAC7B;oBACAlD,MAAM,cAAciE,KAAK,SAAS,CAACJ,YAAY,MAAM;oBAErD,MAAM,EACJK,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACNC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGhB;oBACJf,eAAe6B;oBAEftD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCmD;oBACF;oBACAnD,gBAAgB,IAAI,CAAC,KAAK,GAAGkD;oBAC7BlD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGoD;oBACzCpD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS6C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUR,WAAW,QAAQ;wBAC7B,QAAQc;wBACR,wBAAwBd,WAAW,sBAAsB;wBACzDe;wBACAC;oBACF;oBACAxD,gBAAgB,SAAS,GAAGC;oBAE5BC,OAAO,CAAC+C,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEF,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBACFnD,OACE,OACA,CAAC,aAAa,EAAEoD,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMP,aAAanC,QAAQ;YAG3B,MAAMd,QAAQiD,YAAY,WAAW,EAAE;YACvCnB,SAAS,IAAI,IAAKmB,YAAY,YAAY,EAAE;YAE5C,IAAIiB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9ClE,OACAC,wBACAC,6BACA;oBACEiB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAOiC,OAAO;gBACd,OAAOrD,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEqD,MAAM,SAAS,EAAEL,KAAK,SAAS,CAC5ErD,QACC;YAEP;YACA,IAAI2B,oBAAoB,sBAAsB,EAC5Ce,QAAQ,IAAI,CACV,8FACAf,oBAAoB,sBAAsB;YAK9C,MAAMwC,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAClDxC,oBAAoB,sBAAsB,IAAI,CAAC,cAAc,EAAEwC,mBAAmB;YAElF,IAAI;gBACF,MAAM9D,QAAQ,YAAY,CAAC6D,YAAY,KAAK;YAC9C,EAAE,OAAOR,OAAY;gBAEnBzB;gBACA,MAAMmC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,iCAAiC,EAAEV,OAAO,WAAWW,OAAOX,QAAQ;gBACrItE,MACE,yFACAsE,iBAAiBY,QAAQZ,MAAM,OAAO,GAAGW,OAAOX,QAChD,6CACAzB;YAEJ;YAEA,IAAIA,8BAA8B3C,uCAChC,OAAOe,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACwB,YAAY,wBACf;YAIF,EAAEpB;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMwC,WAAW,CAAC,UAAU,EAAExC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO1B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAC5C,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMyC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACNtC;gBACA,QAAQI;YACV;YACAtB;QACF;IACF;IAEQ,oBACN4D,IAAsE,EACtEC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOjE,OAAOsE;gBACtB,MAAM,EAAEvE,IAAI,EAAE,GAAGuE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ1E,KAAK,GAAG,GAAG;wBACT0E;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1E,KAAK,KAAK,GAAG0E,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,mBACjB1E,KAAK,iBAAiB,GAAG0E,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMvE,YAAYoE,YAAY,SAAS;gBACvCnE,OAAOD,WAAW;gBAElB,MAAMwE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACd,MAAMC,gBACJb,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIU,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAE,GAAGZ,KAAK,EAAE,EAAEC,QAAQ;oBACrC;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DvF,MAAM;oBACN,MAAMoG,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,aACAC,KACAY,sBACAX,kBACAlE;gBAEJ,EAAE,OAAOgD,OAAO;oBACd,IAAIA,iBAAiBgC,cACnBV,UAAUtB,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEiC,IAAI,EAAEpC,OAAO,EAAE0B,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAKpD,QAC1B4D,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAW3D,QAC1B4D,eAAgBD,KAAa,MAAM;qBAEnChF,OAAO,OAAO;gBAKpB,IAAI6D,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCrF,KAAK,OAAO,GAAGgD;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQqC;oBACR,KAAKb;oBACLxB;gBACF;YACF;QACF;QAEA,OAAOsB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,aACAC,KACAC;QAGF,MAAMhE,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACwE;QAE1C,IAAI,CAAC/D,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEwC,OAAO,EAAE,GAAGzC;QAE5B,OAAO;YACLC;YACAwC;YACA3C;QACF;IACF;IAEA,MAAM,QACJiF,SAAsB,EACtBlB,GAA+B,EAC/BD,WAAyB,EACO;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMzF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW0F;QAE1B,MAAMpF,SAASP,QAAQ,SAAS;QAChC,MAAM,EACJ4F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA1F,OAAOkF,WAAW;QAClBlF,OAAOsF,WAAW;QAClBtF,OAAOuF,iBAAiB;QAExBvF,OACEuF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,aACA4B,mBACA1B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACwE;YAO3C,IAAI/D,QAAQ,QACV,OAAO;gBACL,QAAQkB;gBACRpB;YACF;YAGF8F,eACE5F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEgF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMtD,UAAU,CAAC,kBAAkB,EAAE2C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGvD;oBAAQ;iBAAE,EAC9DmB,aACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAM1G,QAAQ,YAAY,CAAC0G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO1G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEqG,cAAc;IACnE;IAjsBA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,kBAAkB,GAAGA,KAAK,kBAAkB;QACjD,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAqqBF;AAEO,eAAexF,gBACpBsF,iBAAoC,EACpCzF,iBAAuC,EACvCiB,MAAwB;IAExB,IAAI,CAACjB,mBAAmB,QACtB,OAAOiB;IAGT,IAAI,CAACwE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC9F;IACvB;IAEA,MAAM,EAAE+F,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMtG,SAAS,MAAM0B;QAErB,MAAMkB,QAAQ6D;QACd,IAAI7D,OACF,MAAMA;QAER,OAAO5C;IACT,SAAU;QACRwG;IACF;AACF"}

package/dist/es/agent/utils.mjs

@@ -123,7 +123,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.6.2";
+const getMidsceneVersion = ()=>"1.6.3-beta-20260403070857.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/auto-glm/actions.mjs

@@ -206,7 +206,9 @@ function transformAutoGLMAction(action, size, actionSpace) {
                                 return [
                                     {
                                         type: 'Launch',
-                                        param: launchAction.app,
+                                        param: {
+                                            uri: launchAction.app
+                                        },
                                         thought: launchAction.think || ''
                                     }
                                 ];

package/dist/es/ai-model/auto-glm/actions.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/auto-glm/actions.mjs","sources":["../../../../src/ai-model/auto-glm/actions.ts"],"sourcesContent":["import { adaptBbox, pointToBbox } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\n\nconst debug = getDebug('auto-glm-actions');\n\n/**\n * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]\n */\nconst AUTO_GLM_COORDINATE_MAX = 1000;\n\n/**\n * Convert auto-glm coordinate [0,1000] to bbox in pixel coordinates\n */\nfunction autoGLMCoordinateToBbox(\n  x: number,\n  y: number,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const bbox = pointToBbox(x, y, 10);\n  return adaptBbox(bbox, width, height, 'auto-glm');\n}\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nconst BACK_BUTTON_NAMES = ['AndroidBackButton', 'HarmonyBackButton'];\nconst HOME_BUTTON_NAMES = ['AndroidHomeButton', 'HarmonyHomeButton'];\n\n/**\n * Find the action name in actionSpace that matches one of the known names.\n * Falls back to defaultName if no match found or actionSpace is not provided.\n */\nfunction findActionName(\n  actionSpace: DeviceAction[] | undefined,\n  knownNames: string[],\n  defaultName: string,\n): string {\n  if (!actionSpace) return defaultName;\n  const match = actionSpace.find((a) => knownNames.includes(a.name));\n  return match ? match.name : defaultName;\n}\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n  actionSpace?: DeviceAction[],\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              tapAction.element[0],\n              tapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              doubleTapAction.element[0],\n              doubleTapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate locate using start coordinate\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              swipeAction.start[0],\n              swipeAction.start[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              longPressAction.element[0],\n              longPressAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'LongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  BACK_BUTTON_NAMES,\n                  'AndroidBackButton',\n                ),\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  HOME_BUTTON_NAMES,\n                  'AndroidHomeButton',\n                ),\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: launchAction.app,\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["debug","getDebug","AUTO_GLM_COORDINATE_MAX","autoGLMCoordinateToBbox","x","y","width","height","bbox","pointToBbox","adaptBbox","BACK_BUTTON_NAMES","HOME_BUTTON_NAMES","findActionName","actionSpace","knownNames","defaultName","match","a","transformAutoGLMAction","action","size","finishAction","doAction","tapAction","x1","y1","x2","y2","locate","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;AAKA,MAAMA,QAAQC,SAAS;AAKvB,MAAMC,0BAA0B;AAKhC,SAASC,wBACPC,CAAS,EACTC,CAAS,EACTC,KAAa,EACbC,MAAc;IAEd,MAAMC,OAAOC,YAAYL,GAAGC,GAAG;IAC/B,OAAOK,UAAUF,MAAMF,OAAOC,QAAQ;AACxC;AAwGA,MAAMI,oBAAoB;IAAC;IAAqB;CAAoB;AACpE,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AAMpE,SAASC,eACPC,WAAuC,EACvCC,UAAoB,EACpBC,WAAmB;IAEnB,IAAI,CAACF,aAAa,OAAOE;IACzB,MAAMC,QAAQH,YAAY,IAAI,CAAC,CAACI,IAAMH,WAAW,QAAQ,CAACG,EAAE,IAAI;IAChE,OAAOD,QAAQA,MAAM,IAAI,GAAGD;AAC9B;AAEO,SAASG,uBACdC,MAAoB,EACpBC,IAAuC,EACvCP,WAA4B;IAE5B,IAAI;QACF,OAAQM,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBpB,MAAM,4BAA4BsB;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBvB,MAAM,yBAAyBwB;gCAC/B,MAAM,CAACC,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqB,UAAU,OAAO,CAAC,EAAE,EACpBA,UAAU,OAAO,CAAC,EAAE,EACpBH,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBP;gCACxBvB,MAAM,gCAAgC8B;gCACtC,MAAM,CAACL,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB2B,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BT,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaR;gCACnBvB,MAAM,0BAA0B+B;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcT;gCACpBvB,MAAM,2BAA2BgC;gCAGjC,MAAM,CAACP,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB6B,YAAY,KAAK,CAAC,EAAE,EACpBA,YAAY,KAAK,CAAC,EAAE,EACpBX,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAGA,MAAMK,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAYlB,KAAK,MAAM,GAAInB;oCAE9BiC,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAYhB,KAAK,KAAK,GAAInB;oCAE7BiC,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEAjC,MACE,CAAC,2BAA2B,EAAEmC,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLP;4CAEAO;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBjB;gCACxBvB,MAAM,gCAAgCwC;gCACtC,MAAM,CAACf,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqC,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BnB,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;wCACA,SAASW,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAalB;gCACnBvB,MAAM,0BAA0ByC;gCAChC,OAAO;oCACL;wCACE,MAAM5B,eACJC,aACAH,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAanB;gCACnBvB,MAAM,0BAA0B0C;gCAChC,OAAO;oCACL;wCACE,MAAM7B,eACJC,aACAF,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAapB;gCACnBvB,MAAM,0BAA0B2C;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAerB;gCACrBvB,MAAM,4BAA4B4C;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAOA,aAAa,GAAG;wCACvB,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGtB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIsB,MACR,CAAC,yBAAyB,EAAGzB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAO0B,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrE9C,MAAM,oBAAoB+C;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}
+{"version":3,"file":"ai-model/auto-glm/actions.mjs","sources":["../../../../src/ai-model/auto-glm/actions.ts"],"sourcesContent":["import { adaptBbox, pointToBbox } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\n\nconst debug = getDebug('auto-glm-actions');\n\n/**\n * Auto-GLM coordinate system range: [0, AUTO_GLM_COORDINATE_MAX]\n */\nconst AUTO_GLM_COORDINATE_MAX = 1000;\n\n/**\n * Convert auto-glm coordinate [0,1000] to bbox in pixel coordinates\n */\nfunction autoGLMCoordinateToBbox(\n  x: number,\n  y: number,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const bbox = pointToBbox(x, y, 10);\n  return adaptBbox(bbox, width, height, 'auto-glm');\n}\n\nexport interface BaseAction {\n  _metadata: string;\n  think?: string;\n}\n\nexport interface TapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Tap';\n  element: [number, number];\n}\n\nexport interface DoubleTapAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Double Tap';\n  element: [number, number];\n}\n\nexport interface TypeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Type';\n  text: string;\n}\n\nexport interface SwipeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Swipe';\n  start: [number, number];\n  end: [number, number];\n}\n\nexport interface LongPressAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Long Press';\n  element: [number, number];\n}\n\nexport interface LaunchAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Launch';\n  app: string;\n}\n\nexport interface BackAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Back';\n}\n\nexport interface HomeAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Home';\n}\n\nexport interface WaitAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Wait';\n  durationMs: number;\n}\n\nexport interface InteractAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Interact';\n}\n\nexport interface CallAPIAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Call_API';\n  instruction: string;\n}\n\nexport interface TakeoverAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Take_over';\n  message: string;\n}\n\nexport interface NoteAction extends BaseAction {\n  _metadata: 'do';\n  action: 'Note';\n  message: string;\n}\n\nexport interface FinishAction extends BaseAction {\n  _metadata: 'finish';\n  message: string;\n}\n\nexport type ParsedAction =\n  | TapAction\n  | DoubleTapAction\n  | TypeAction\n  | SwipeAction\n  | LongPressAction\n  | LaunchAction\n  | BackAction\n  | HomeAction\n  | WaitAction\n  | InteractAction\n  | CallAPIAction\n  | TakeoverAction\n  | NoteAction\n  | FinishAction;\n\nconst BACK_BUTTON_NAMES = ['AndroidBackButton', 'HarmonyBackButton'];\nconst HOME_BUTTON_NAMES = ['AndroidHomeButton', 'HarmonyHomeButton'];\n\n/**\n * Find the action name in actionSpace that matches one of the known names.\n * Falls back to defaultName if no match found or actionSpace is not provided.\n */\nfunction findActionName(\n  actionSpace: DeviceAction[] | undefined,\n  knownNames: string[],\n  defaultName: string,\n): string {\n  if (!actionSpace) return defaultName;\n  const match = actionSpace.find((a) => knownNames.includes(a.name));\n  return match ? match.name : defaultName;\n}\n\nexport function transformAutoGLMAction(\n  action: ParsedAction,\n  size: { width: number; height: number },\n  actionSpace?: DeviceAction[],\n): PlanningAction[] {\n  try {\n    switch (action._metadata) {\n      case 'finish': {\n        const finishAction = action as FinishAction;\n        debug('Transform finish action:', finishAction);\n        return [\n          {\n            type: 'Finished',\n            param: {},\n            thought: finishAction.message,\n          },\n        ];\n      }\n      case 'do': {\n        const doAction = action as\n          | TapAction\n          | DoubleTapAction\n          | TypeAction\n          | SwipeAction\n          | LongPressAction\n          | LaunchAction\n          | BackAction\n          | HomeAction\n          | WaitAction\n          | InteractAction\n          | CallAPIAction\n          | TakeoverAction\n          | NoteAction;\n\n        switch ((doAction as any).action) {\n          case 'Tap': {\n            const tapAction = doAction as TapAction;\n            debug('Transform Tap action:', tapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              tapAction.element[0],\n              tapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'Tap',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Double Tap': {\n            const doubleTapAction = doAction as DoubleTapAction;\n            debug('Transform Double Tap action:', doubleTapAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              doubleTapAction.element[0],\n              doubleTapAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'DoubleClick',\n                param: {\n                  locate,\n                },\n              },\n            ];\n          }\n          case 'Type': {\n            const typeAction = doAction as TypeAction;\n            debug('Transform Type action:', typeAction);\n\n            return [\n              {\n                type: 'Input',\n                param: {\n                  value: typeAction.text,\n                },\n              },\n            ];\n          }\n          case 'Swipe': {\n            const swipeAction = doAction as SwipeAction;\n            debug('Transform Swipe action:', swipeAction);\n\n            // Calculate locate using start coordinate\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              swipeAction.start[0],\n              swipeAction.start[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            // Calculate horizontal and vertical delta in [0,AUTO_GLM_COORDINATE_MAX] coordinate system\n            const deltaX = swipeAction.end[0] - swipeAction.start[0];\n            const deltaY = swipeAction.end[1] - swipeAction.start[1];\n\n            // Determine direction and distance\n            let direction: 'up' | 'down' | 'left' | 'right';\n            let distance: number;\n\n            const absDeltaX = Math.abs(deltaX);\n            const absDeltaY = Math.abs(deltaY);\n\n            if (absDeltaY > absDeltaX) {\n              // Vertical scroll\n              distance = Math.round(\n                (absDeltaY * size.height) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaY > 0 ? 'up' : 'down';\n            } else {\n              // Horizontal scroll\n              distance = Math.round(\n                (absDeltaX * size.width) / AUTO_GLM_COORDINATE_MAX,\n              );\n              direction = deltaX > 0 ? 'left' : 'right';\n            }\n\n            debug(\n              `Calculate swipe direction: ${direction}, distance: ${distance}`,\n            );\n\n            return [\n              {\n                type: 'Scroll',\n                param: {\n                  locate,\n                  // The scrolling direction here all refers to which direction of the page's content will appear on the screen.\n                  distance,\n                  direction,\n                },\n                thought: swipeAction.think || '',\n              },\n            ];\n          }\n          case 'Long Press': {\n            const longPressAction = doAction as LongPressAction;\n            debug('Transform Long Press action:', longPressAction);\n            const [x1, y1, x2, y2] = autoGLMCoordinateToBbox(\n              longPressAction.element[0],\n              longPressAction.element[1],\n              size.width,\n              size.height,\n            );\n\n            const locate: {\n              prompt: string;\n              bbox: [number, number, number, number];\n            } = {\n              prompt: '',\n              bbox: [x1, y1, x2, y2],\n            };\n\n            return [\n              {\n                type: 'LongPress',\n                param: {\n                  locate,\n                },\n                thought: longPressAction.think || '',\n              },\n            ];\n          }\n          case 'Back': {\n            const backAction = doAction as BackAction;\n            debug('Transform Back action:', backAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  BACK_BUTTON_NAMES,\n                  'AndroidBackButton',\n                ),\n                param: {},\n                thought: backAction.think || '',\n              },\n            ];\n          }\n          case 'Home': {\n            const homeAction = doAction as HomeAction;\n            debug('Transform Home action:', homeAction);\n            return [\n              {\n                type: findActionName(\n                  actionSpace,\n                  HOME_BUTTON_NAMES,\n                  'AndroidHomeButton',\n                ),\n                param: {},\n                thought: homeAction.think || '',\n              },\n            ];\n          }\n          case 'Wait': {\n            const waitAction = doAction as WaitAction;\n            debug('Transform Wait action:', waitAction);\n            return [\n              {\n                type: 'Sleep',\n                param: {\n                  timeMs: waitAction.durationMs,\n                },\n                thought: waitAction.think || '',\n              },\n            ];\n          }\n          case 'Launch': {\n            const launchAction = doAction as LaunchAction;\n            debug('Transform Launch action:', launchAction);\n            return [\n              {\n                type: 'Launch',\n                param: { uri: launchAction.app },\n                thought: launchAction.think || '',\n              },\n            ];\n          }\n          case 'Interact': {\n            throw new Error(\n              `Action \"Interact\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Call_API': {\n            throw new Error(\n              `Action \"Call_API\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Take_over': {\n            throw new Error(\n              `Action \"Take_over\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          case 'Note': {\n            throw new Error(\n              `Action \"Note\" from auto-glm is not supported in the current implementation.`,\n            );\n          }\n          default:\n            throw new Error(\n              `Unknown do() action type: ${(doAction as any).action}`,\n            );\n        }\n      }\n      default:\n        throw new Error(\n          `Unknown action metadata: ${(action as any)._metadata}`,\n        );\n    }\n  } catch (error) {\n    const errorMessage = error instanceof Error ? error.message : String(error);\n    debug('Transform error:', errorMessage);\n    throw new Error(`Failed to transform action: ${errorMessage}`);\n  }\n}\n"],"names":["debug","getDebug","AUTO_GLM_COORDINATE_MAX","autoGLMCoordinateToBbox","x","y","width","height","bbox","pointToBbox","adaptBbox","BACK_BUTTON_NAMES","HOME_BUTTON_NAMES","findActionName","actionSpace","knownNames","defaultName","match","a","transformAutoGLMAction","action","size","finishAction","doAction","tapAction","x1","y1","x2","y2","locate","doubleTapAction","typeAction","swipeAction","deltaX","deltaY","direction","distance","absDeltaX","Math","absDeltaY","longPressAction","backAction","homeAction","waitAction","launchAction","Error","error","errorMessage","String"],"mappings":";;AAKA,MAAMA,QAAQC,SAAS;AAKvB,MAAMC,0BAA0B;AAKhC,SAASC,wBACPC,CAAS,EACTC,CAAS,EACTC,KAAa,EACbC,MAAc;IAEd,MAAMC,OAAOC,YAAYL,GAAGC,GAAG;IAC/B,OAAOK,UAAUF,MAAMF,OAAOC,QAAQ;AACxC;AAwGA,MAAMI,oBAAoB;IAAC;IAAqB;CAAoB;AACpE,MAAMC,oBAAoB;IAAC;IAAqB;CAAoB;AAMpE,SAASC,eACPC,WAAuC,EACvCC,UAAoB,EACpBC,WAAmB;IAEnB,IAAI,CAACF,aAAa,OAAOE;IACzB,MAAMC,QAAQH,YAAY,IAAI,CAAC,CAACI,IAAMH,WAAW,QAAQ,CAACG,EAAE,IAAI;IAChE,OAAOD,QAAQA,MAAM,IAAI,GAAGD;AAC9B;AAEO,SAASG,uBACdC,MAAoB,EACpBC,IAAuC,EACvCP,WAA4B;IAE5B,IAAI;QACF,OAAQM,OAAO,SAAS;YACtB,KAAK;gBAAU;oBACb,MAAME,eAAeF;oBACrBpB,MAAM,4BAA4BsB;oBAClC,OAAO;wBACL;4BACE,MAAM;4BACN,OAAO,CAAC;4BACR,SAASA,aAAa,OAAO;wBAC/B;qBACD;gBACH;YACA,KAAK;gBAAM;oBACT,MAAMC,WAAWH;oBAejB,OAASG,SAAiB,MAAM;wBAC9B,KAAK;4BAAO;gCACV,MAAMC,YAAYD;gCAClBvB,MAAM,yBAAyBwB;gCAC/B,MAAM,CAACC,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqB,UAAU,OAAO,CAAC,EAAE,EACpBA,UAAU,OAAO,CAAC,EAAE,EACpBH,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMC,kBAAkBP;gCACxBvB,MAAM,gCAAgC8B;gCACtC,MAAM,CAACL,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB2B,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BT,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAME,aAAaR;gCACnBvB,MAAM,0BAA0B+B;gCAEhC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,OAAOA,WAAW,IAAI;wCACxB;oCACF;iCACD;4BACH;wBACA,KAAK;4BAAS;gCACZ,MAAMC,cAAcT;gCACpBvB,MAAM,2BAA2BgC;gCAGjC,MAAM,CAACP,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvB6B,YAAY,KAAK,CAAC,EAAE,EACpBA,YAAY,KAAK,CAAC,EAAE,EACpBX,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAGA,MAAMK,SAASD,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCACxD,MAAME,SAASF,YAAY,GAAG,CAAC,EAAE,GAAGA,YAAY,KAAK,CAAC,EAAE;gCAGxD,IAAIG;gCACJ,IAAIC;gCAEJ,MAAMC,YAAYC,KAAK,GAAG,CAACL;gCAC3B,MAAMM,YAAYD,KAAK,GAAG,CAACJ;gCAE3B,IAAIK,YAAYF,WAAW;oCAEzBD,WAAWE,KAAK,KAAK,CAClBC,YAAYlB,KAAK,MAAM,GAAInB;oCAE9BiC,YAAYD,SAAS,IAAI,OAAO;gCAClC,OAAO;oCAELE,WAAWE,KAAK,KAAK,CAClBD,YAAYhB,KAAK,KAAK,GAAInB;oCAE7BiC,YAAYF,SAAS,IAAI,SAAS;gCACpC;gCAEAjC,MACE,CAAC,2BAA2B,EAAEmC,UAAU,YAAY,EAAEC,UAAU;gCAGlE,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLP;4CAEAO;4CACAD;wCACF;wCACA,SAASH,YAAY,KAAK,IAAI;oCAChC;iCACD;4BACH;wBACA,KAAK;4BAAc;gCACjB,MAAMQ,kBAAkBjB;gCACxBvB,MAAM,gCAAgCwC;gCACtC,MAAM,CAACf,IAAIC,IAAIC,IAAIC,GAAG,GAAGzB,wBACvBqC,gBAAgB,OAAO,CAAC,EAAE,EAC1BA,gBAAgB,OAAO,CAAC,EAAE,EAC1BnB,KAAK,KAAK,EACVA,KAAK,MAAM;gCAGb,MAAMQ,SAGF;oCACF,QAAQ;oCACR,MAAM;wCAACJ;wCAAIC;wCAAIC;wCAAIC;qCAAG;gCACxB;gCAEA,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACLC;wCACF;wCACA,SAASW,gBAAgB,KAAK,IAAI;oCACpC;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAalB;gCACnBvB,MAAM,0BAA0ByC;gCAChC,OAAO;oCACL;wCACE,MAAM5B,eACJC,aACAH,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAanB;gCACnBvB,MAAM,0BAA0B0C;gCAChC,OAAO;oCACL;wCACE,MAAM7B,eACJC,aACAF,mBACA;wCAEF,OAAO,CAAC;wCACR,SAAS8B,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAQ;gCACX,MAAMC,aAAapB;gCACnBvB,MAAM,0BAA0B2C;gCAChC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CACL,QAAQA,WAAW,UAAU;wCAC/B;wCACA,SAASA,WAAW,KAAK,IAAI;oCAC/B;iCACD;4BACH;wBACA,KAAK;4BAAU;gCACb,MAAMC,eAAerB;gCACrBvB,MAAM,4BAA4B4C;gCAClC,OAAO;oCACL;wCACE,MAAM;wCACN,OAAO;4CAAE,KAAKA,aAAa,GAAG;wCAAC;wCAC/B,SAASA,aAAa,KAAK,IAAI;oCACjC;iCACD;4BACH;wBACA,KAAK;4BACH,MAAM,IAAIC,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ,KAAK;4BACH,MAAM,IAAIA,MACR;wBAGJ;4BACE,MAAM,IAAIA,MACR,CAAC,0BAA0B,EAAGtB,SAAiB,MAAM,EAAE;oBAE7D;gBACF;YACA;gBACE,MAAM,IAAIsB,MACR,CAAC,yBAAyB,EAAGzB,OAAe,SAAS,EAAE;QAE7D;IACF,EAAE,OAAO0B,OAAO;QACd,MAAMC,eAAeD,iBAAiBD,QAAQC,MAAM,OAAO,GAAGE,OAAOF;QACrE9C,MAAM,oBAAoB+C;QAC1B,MAAM,IAAIF,MAAM,CAAC,4BAA4B,EAAEE,cAAc;IAC/D;AACF"}

package/dist/es/ai-model/prompt/extraction.mjs

@@ -36,6 +36,8 @@ The user will give you a screenshot, the contents of it (optional), and some dat
 
 If a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.
 
+When DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.
+
 If the user provides multiple reference images, please carefully review the reference images with the screenshot and provide the correct answer for <DATA_DEMAND>.
 
 
@@ -99,7 +101,7 @@ If the DATA_DEMAND is:
 
 <DATA_DEMAND>
 {
-  "result": "Boolean, is it currently the SMS page?"
+  "StatementIsTruthy": "Boolean, is it currently the SMS page?"
 }
 </DATA_DEMAND>
 
@@ -107,7 +109,7 @@ By viewing the screenshot and page contents, you can extract the following data:
 
 <thought>According to the screenshot, i can see ...</thought>
 <data-json>
-{ "result": true }
+{ "StatementIsTruthy": true }
 </data-json>
 `;
 }

package/dist/es/ai-model/prompt/extraction.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/extraction.mjs","sources":["../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["import type { AIDataExtractionResponse } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\nimport { safeParseJson } from '../service-caller/index';\nimport { extractXMLTag } from './util';\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = safeParseJson(dataJsonStr, undefined) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = safeParseJson(errorsStr, undefined);\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract() {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\nThe user will give you a screenshot, the contents of it (optional), and some data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nIf the user provides multiple reference images, please carefully review the reference images with the screenshot and provide the correct answer for <DATA_DEMAND>.\n\n\nReturn in the following XML format:\n<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"result\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{ \"result\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","safeParseJson","undefined","e","errors","parsedErrors","Array","systemPromptToExtract","preferredLanguage","getPreferredLanguage","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;AASO,SAASA,2BACdC,SAAiB;IAEjB,MAAMC,UAAUC,cAAcF,WAAW;IACzC,MAAMG,cAAcD,cAAcF,WAAW;IAC7C,MAAMI,YAAYF,cAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,cAAcJ,aAAaK;IACpC,EAAE,OAAOC,GAAG;QACV,MAAM,IAAIJ,MAAM,CAAC,2BAA2B,EAAEI,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIN,WACF,IAAI;QACF,MAAMO,eAAeJ,cAAcH,WAAWI;QAC9C,IAAII,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIR,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAII,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG;IACd,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;;;;;;;;;0EAWgE,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqE9F,CAAC;AACD;AAEO,MAAME,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}
+{"version":3,"file":"ai-model/prompt/extraction.mjs","sources":["../../../../src/ai-model/prompt/extraction.ts"],"sourcesContent":["import type { AIDataExtractionResponse } from '@/types';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport type { ResponseFormatJSONSchema } from 'openai/resources/index';\nimport { safeParseJson } from '../service-caller/index';\nimport { extractXMLTag } from './util';\n\n/**\n * Parse XML response from LLM and convert to AIDataExtractionResponse\n */\nexport function parseXMLExtractionResponse<T>(\n  xmlString: string,\n): AIDataExtractionResponse<T> {\n  const thought = extractXMLTag(xmlString, 'thought');\n  const dataJsonStr = extractXMLTag(xmlString, 'data-json');\n  const errorsStr = extractXMLTag(xmlString, 'errors');\n\n  // Parse data-json (required)\n  if (!dataJsonStr) {\n    throw new Error('Missing required field: data-json');\n  }\n\n  let data: T;\n  try {\n    data = safeParseJson(dataJsonStr, undefined) as T;\n  } catch (e) {\n    throw new Error(`Failed to parse data-json: ${e}`);\n  }\n\n  // Parse errors (optional)\n  let errors: string[] | undefined;\n  if (errorsStr) {\n    try {\n      const parsedErrors = safeParseJson(errorsStr, undefined);\n      if (Array.isArray(parsedErrors)) {\n        errors = parsedErrors;\n      }\n    } catch (e) {\n      // If errors parsing fails, just ignore it\n    }\n  }\n\n  return {\n    ...(thought ? { thought } : {}),\n    data,\n    ...(errors && errors.length > 0 ? { errors } : {}),\n  };\n}\n\nexport function systemPromptToExtract() {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are a versatile professional in software UI design and testing. Your outstanding contributions will impact the user experience of billions of users.\n\nThe user will give you a screenshot, the contents of it (optional), and some data requirements in <DATA_DEMAND>. You need to understand the user's requirements and extract the data satisfying the <DATA_DEMAND>.\n\nIf a key specifies a JSON data type (such as Number, String, Boolean, Object, Array), ensure the returned value strictly matches that data type.\n\nWhen DATA_DEMAND is a JSON object, the keys in your response must exactly match the keys in DATA_DEMAND. Do not rename, translate, or substitute any key.\n\nIf the user provides multiple reference images, please carefully review the reference images with the screenshot and provide the correct answer for <DATA_DEMAND>.\n\n\nReturn in the following XML format:\n<thought>the thinking process of the extraction, less than 300 words. Use ${preferredLanguage} in this field.</thought>\n<data-json>the extracted data as JSON. Make sure both the value and scheme meet the DATA_DEMAND. If you want to write some description in this field, use the same language as the DATA_DEMAND.</data-json>\n<errors>optional error messages as JSON array, e.g., [\"error1\", \"error2\"]</errors>\n\n# Example 1\nFor example, if the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"name\": \"name shows on the left panel, string\",\n  \"age\": \"age shows on the right panel, number\",\n  \"isAdmin\": \"if the user is admin, boolean\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{\n  \"name\": \"John\",\n  \"age\": 30,\n  \"isAdmin\": true\n}\n</data-json>\n\n# Example 2\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe todo items list, string[]\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n[\"todo 1\", \"todo 2\", \"todo 3\"]\n</data-json>\n\n# Example 3\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\nthe page title, string\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n\"todo list\"\n</data-json>\n\n# Example 4\nIf the DATA_DEMAND is:\n\n<DATA_DEMAND>\n{\n  \"StatementIsTruthy\": \"Boolean, is it currently the SMS page?\"\n}\n</DATA_DEMAND>\n\nBy viewing the screenshot and page contents, you can extract the following data:\n\n<thought>According to the screenshot, i can see ...</thought>\n<data-json>\n{ \"StatementIsTruthy\": true }\n</data-json>\n`;\n}\n\nexport const extractDataQueryPrompt = (\n  pageDescription: string,\n  dataQuery: string | Record<string, string>,\n) => {\n  let dataQueryText = '';\n  if (typeof dataQuery === 'string') {\n    dataQueryText = dataQuery;\n  } else {\n    dataQueryText = JSON.stringify(dataQuery, null, 2);\n  }\n\n  return `\n<PageDescription>\n${pageDescription}\n</PageDescription>\n\n<DATA_DEMAND>\n${dataQueryText}\n</DATA_DEMAND>\n  `;\n};\n"],"names":["parseXMLExtractionResponse","xmlString","thought","extractXMLTag","dataJsonStr","errorsStr","Error","data","safeParseJson","undefined","e","errors","parsedErrors","Array","systemPromptToExtract","preferredLanguage","getPreferredLanguage","extractDataQueryPrompt","pageDescription","dataQuery","dataQueryText","JSON"],"mappings":";;;AASO,SAASA,2BACdC,SAAiB;IAEjB,MAAMC,UAAUC,cAAcF,WAAW;IACzC,MAAMG,cAAcD,cAAcF,WAAW;IAC7C,MAAMI,YAAYF,cAAcF,WAAW;IAG3C,IAAI,CAACG,aACH,MAAM,IAAIE,MAAM;IAGlB,IAAIC;IACJ,IAAI;QACFA,OAAOC,cAAcJ,aAAaK;IACpC,EAAE,OAAOC,GAAG;QACV,MAAM,IAAIJ,MAAM,CAAC,2BAA2B,EAAEI,GAAG;IACnD;IAGA,IAAIC;IACJ,IAAIN,WACF,IAAI;QACF,MAAMO,eAAeJ,cAAcH,WAAWI;QAC9C,IAAII,MAAM,OAAO,CAACD,eAChBD,SAASC;IAEb,EAAE,OAAOF,GAAG,CAEZ;IAGF,OAAO;QACL,GAAIR,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9BK;QACA,GAAII,UAAUA,OAAO,MAAM,GAAG,IAAI;YAAEA;QAAO,IAAI,CAAC,CAAC;IACnD;AACF;AAEO,SAASG;IACd,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;;;;;;;;;;;0EAagE,EAAED,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqE9F,CAAC;AACD;AAEO,MAAME,yBAAyB,CACpCC,iBACAC;IAEA,IAAIC,gBAAgB;IAElBA,gBADE,AAAqB,YAArB,OAAOD,YACOA,YAEAE,KAAK,SAAS,CAACF,WAAW,MAAM;IAGlD,OAAO,CAAC;;AAEV,EAAED,gBAAgB;;;;AAIlB,EAAEE,cAAc;;EAEd,CAAC;AACH"}

package/dist/es/report.mjs

@@ -17,6 +17,15 @@ function _define_property(obj, key, value) {
     return obj;
 }
 class ReportMergingTool {
+    createEmptyDumpString(groupName, groupDescription) {
+        return new ReportActionDump({
+            sdkVersion: '',
+            groupName,
+            groupDescription,
+            modelBriefs: [],
+            executions: []
+        }).serialize();
+    }
     append(reportInfo) {
         this.reportInfos.push(reportInfo);
     }
@@ -49,13 +58,13 @@ class ReportMergingTool {
         return base.serialize();
     }
     mergeReports(reportFileName = 'AUTO', opts) {
-        if (this.reportInfos.length <= 1) {
-            logMsg('Not enough reports to merge');
+        const { rmOriginalReports = false, overwrite = false } = opts ?? {};
+        if (0 === this.reportInfos.length) {
+            logMsg('No reports to merge');
             return null;
         }
-        const { rmOriginalReports = false, overwrite = false } = opts ?? {};
         const targetDir = getMidsceneRunSubDir('report');
-        const hasDirectoryModeReport = this.reportInfos.some((info)=>this.isDirectoryModeReport(info.reportFilePath));
+        const hasDirectoryModeReport = this.reportInfos.some((info)=>info.reportFilePath && this.isDirectoryModeReport(info.reportFilePath));
         const resolvedName = 'AUTO' === reportFileName ? getReportFileName('merged-report') : reportFileName;
         const outputFilePath = hasDirectoryModeReport ? resolve(targetDir, resolvedName, 'index.html') : resolve(targetDir, `${resolvedName}.html`);
         if ('AUTO' !== reportFileName && existsSync(outputFilePath)) {
@@ -71,29 +80,38 @@ class ReportMergingTool {
         });
         logMsg(`Start merging ${this.reportInfos.length} reports...\nCreating template file...`);
         try {
-            appendFileSync(outputFilePath, getReportTpl());
+            const htmlEndTag = '</html>';
+            const tpl = getReportTpl();
+            const htmlEndIdx = tpl.lastIndexOf(htmlEndTag);
+            const tplWithoutClose = -1 !== htmlEndIdx ? tpl.slice(0, htmlEndIdx) : tpl;
+            appendFileSync(outputFilePath, tplWithoutClose);
             if (hasDirectoryModeReport) appendFileSync(outputFilePath, getBaseUrlFixScript());
             for(let i = 0; i < this.reportInfos.length; i++){
                 const reportInfo = this.reportInfos[i];
                 logMsg(`Processing report ${i + 1}/${this.reportInfos.length}`);
-                if (this.isDirectoryModeReport(reportInfo.reportFilePath)) {
-                    const reportDir = dirname(reportInfo.reportFilePath);
-                    const screenshotsDir = join(reportDir, 'screenshots');
-                    const mergedScreenshotsDir = join(dirname(outputFilePath), 'screenshots');
-                    mkdirSync(mergedScreenshotsDir, {
-                        recursive: true
-                    });
-                    for (const file of readdirSync(screenshotsDir)){
-                        const src = join(screenshotsDir, file);
-                        const dest = join(mergedScreenshotsDir, file);
-                        copyFileSync(src, dest);
-                    }
-                } else streamImageScriptsToFile(reportInfo.reportFilePath, outputFilePath);
-                const allDumps = extractAllDumpScriptsSync(reportInfo.reportFilePath).filter((d)=>d.openTag.includes('data-group-id'));
-                const groupIdMatch = allDumps[0]?.openTag.match(/data-group-id="([^"]+)"/);
-                const mergedGroupId = groupIdMatch ? decodeURIComponent(groupIdMatch[1]) : `merged-group-${i}`;
-                const dumpString = allDumps.length > 0 ? this.mergeDumpScripts(allDumps.map((d)=>d.content)) : extractLastDumpScriptSync(reportInfo.reportFilePath);
                 const { reportAttributes } = reportInfo;
+                let dumpString = this.createEmptyDumpString(reportAttributes.testTitle, reportAttributes.testDescription);
+                let mergedGroupId = `merged-group-${i}`;
+                if (reportInfo.reportFilePath) {
+                    if (this.isDirectoryModeReport(reportInfo.reportFilePath)) {
+                        const reportDir = dirname(reportInfo.reportFilePath);
+                        const screenshotsDir = join(reportDir, 'screenshots');
+                        const mergedScreenshotsDir = join(dirname(outputFilePath), 'screenshots');
+                        mkdirSync(mergedScreenshotsDir, {
+                            recursive: true
+                        });
+                        for (const file of readdirSync(screenshotsDir)){
+                            const src = join(screenshotsDir, file);
+                            const dest = join(mergedScreenshotsDir, file);
+                            copyFileSync(src, dest);
+                        }
+                    } else streamImageScriptsToFile(reportInfo.reportFilePath, outputFilePath);
+                    const allDumps = extractAllDumpScriptsSync(reportInfo.reportFilePath).filter((d)=>d.openTag.includes('data-group-id'));
+                    const groupIdMatch = allDumps[0]?.openTag.match(/data-group-id="([^"]+)"/);
+                    if (groupIdMatch) mergedGroupId = decodeURIComponent(groupIdMatch[1]);
+                    const extractedDumpString = allDumps.length > 0 ? this.mergeDumpScripts(allDumps.map((d)=>d.content)) : extractLastDumpScriptSync(reportInfo.reportFilePath);
+                    if (extractedDumpString) dumpString = extractedDumpString;
+                }
                 const reportHtmlStr = `${reportHTMLContent({
                     dumpString,
                     attributes: {
@@ -102,14 +120,16 @@ class ReportMergingTool {
                         playwright_test_status: reportAttributes.testStatus,
                         playwright_test_title: reportAttributes.testTitle,
                         playwright_test_id: reportAttributes.testId,
-                        playwright_test_description: reportAttributes.testDescription
+                        playwright_test_description: reportAttributes.testDescription,
+                        is_merged: true
                     }
                 }, void 0, void 0, false)}\n`;
                 appendFileSync(outputFilePath, reportHtmlStr);
             }
+            appendFileSync(outputFilePath, `${htmlEndTag}\n`);
             logMsg(`Successfully merged new report: ${outputFilePath}`);
             if (rmOriginalReports) {
-                for (const info of this.reportInfos)try {
+                for (const info of this.reportInfos)if (info.reportFilePath) try {
                     if (this.isDirectoryModeReport(info.reportFilePath)) {
                         const reportDir = dirname(info.reportFilePath);
                         rmSync(reportDir, {

package/dist/es/report.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"report.mjs","sources":["../../src/report.ts"],"sourcesContent":["import {\n  appendFileSync,\n  copyFileSync,\n  existsSync,\n  mkdirSync,\n  readdirSync,\n  rmSync,\n  unlinkSync,\n} from 'node:fs';\nimport * as path from 'node:path';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { antiEscapeScriptTag, logMsg } from '@midscene/shared/utils';\nimport { getReportFileName } from './agent';\nimport {\n  extractAllDumpScriptsSync,\n  extractLastDumpScriptSync,\n  getBaseUrlFixScript,\n  streamImageScriptsToFile,\n} from './dump/html-utils';\nimport { ReportActionDump } from './types';\nimport type { ReportFileWithAttributes } from './types';\nimport { getReportTpl, reportHTMLContent } from './utils';\n\nexport class ReportMergingTool {\n  private reportInfos: ReportFileWithAttributes[] = [];\n  public append(reportInfo: ReportFileWithAttributes) {\n    this.reportInfos.push(reportInfo);\n  }\n  public clear() {\n    this.reportInfos = [];\n  }\n\n  /**\n   * Check if a report is in directory mode (html-and-external-assets).\n   * Directory mode reports: {name}/index.html + {name}/screenshots/\n   */\n  private isDirectoryModeReport(reportFilePath: string): boolean {\n    const reportDir = path.dirname(reportFilePath);\n    return (\n      path.basename(reportFilePath) === 'index.html' &&\n      existsSync(path.join(reportDir, 'screenshots'))\n    );\n  }\n\n  /**\n   * Merge multiple dump script contents (from the same source report)\n   * into a single serialized ReportActionDump string.\n   * If there's only one dump, return it as-is. If multiple, merge\n   * all executions into the first dump's group structure.\n   */\n  private mergeDumpScripts(contents: string[]): string {\n    const unescaped = contents\n      .map((c) => antiEscapeScriptTag(c))\n      .filter((c) => c.length > 0);\n    if (unescaped.length === 0) return '';\n    if (unescaped.length === 1) return unescaped[0];\n\n    // Parse all dumps and collect executions, deduplicating by id (keep last).\n    // Only executions with a stable id are deduped; old-format entries without\n    // id are always kept (they may be distinct despite sharing the same name).\n    const base = ReportActionDump.fromSerializedString(unescaped[0]);\n    const allExecutions = [...base.executions];\n    for (let i = 1; i < unescaped.length; i++) {\n      const other = ReportActionDump.fromSerializedString(unescaped[i]);\n      allExecutions.push(...other.executions);\n    }\n    let noIdCounter = 0;\n    const deduped = new Map<string, (typeof allExecutions)[0]>();\n    for (const exec of allExecutions) {\n      const key = exec.id || `__no_id_${noIdCounter++}`;\n      deduped.set(key, exec);\n    }\n    base.executions = Array.from(deduped.values());\n    return base.serialize();\n  }\n\n  public mergeReports(\n    reportFileName: 'AUTO' | string = 'AUTO',\n    opts?: {\n      rmOriginalReports?: boolean;\n      overwrite?: boolean;\n    },\n  ): string | null {\n    if (this.reportInfos.length <= 1) {\n      logMsg('Not enough reports to merge');\n      return null;\n    }\n\n    const { rmOriginalReports = false, overwrite = false } = opts ?? {};\n    const targetDir = getMidsceneRunSubDir('report');\n\n    // Check if any source report is directory mode\n    const hasDirectoryModeReport = this.reportInfos.some((info) =>\n      this.isDirectoryModeReport(info.reportFilePath),\n    );\n\n    const resolvedName =\n      reportFileName === 'AUTO'\n        ? getReportFileName('merged-report')\n        : reportFileName;\n\n    // Directory mode: output as {name}/index.html to keep relative paths working\n    // Inline mode: output as {name}.html (single file)\n    const outputFilePath = hasDirectoryModeReport\n      ? path.resolve(targetDir, resolvedName, 'index.html')\n      : path.resolve(targetDir, `${resolvedName}.html`);\n\n    if (reportFileName !== 'AUTO' && existsSync(outputFilePath)) {\n      if (!overwrite) {\n        throw new Error(\n          `Report file already exists: ${outputFilePath}\\nSet overwrite to true to overwrite this file.`,\n        );\n      }\n      if (hasDirectoryModeReport) {\n        rmSync(path.dirname(outputFilePath), { recursive: true, force: true });\n      } else {\n        unlinkSync(outputFilePath);\n      }\n    }\n\n    if (hasDirectoryModeReport) {\n      mkdirSync(path.dirname(outputFilePath), { recursive: true });\n    }\n\n    logMsg(\n      `Start merging ${this.reportInfos.length} reports...\\nCreating template file...`,\n    );\n\n    try {\n      // Write template\n      appendFileSync(outputFilePath, getReportTpl());\n\n      // For directory-mode output, inject base URL fix script\n      if (hasDirectoryModeReport) {\n        appendFileSync(outputFilePath, getBaseUrlFixScript());\n      }\n\n      // Process all reports one by one\n      for (let i = 0; i < this.reportInfos.length; i++) {\n        const reportInfo = this.reportInfos[i];\n        logMsg(`Processing report ${i + 1}/${this.reportInfos.length}`);\n\n        if (this.isDirectoryModeReport(reportInfo.reportFilePath)) {\n          // Directory mode: copy external screenshot files\n          const reportDir = path.dirname(reportInfo.reportFilePath);\n          const screenshotsDir = path.join(reportDir, 'screenshots');\n          const mergedScreenshotsDir = path.join(\n            path.dirname(outputFilePath),\n            'screenshots',\n          );\n          mkdirSync(mergedScreenshotsDir, { recursive: true });\n          for (const file of readdirSync(screenshotsDir)) {\n            const src = path.join(screenshotsDir, file);\n            const dest = path.join(mergedScreenshotsDir, file);\n            copyFileSync(src, dest);\n          }\n        } else {\n          // Inline mode: stream image scripts to output file\n          streamImageScriptsToFile(reportInfo.reportFilePath, outputFilePath);\n        }\n\n        // Extract all dump scripts from the source report.\n        // After the per-execution append refactor, a single source report\n        // may contain multiple <script type=\"midscene_web_dump\"> tags\n        // (one per execution). We merge them into a single ReportActionDump.\n        // Filter by data-group-id to exclude false matches from the template's\n        // bundled JS code, which also references the midscene_web_dump type string.\n        const allDumps = extractAllDumpScriptsSync(\n          reportInfo.reportFilePath,\n        ).filter((d) => d.openTag.includes('data-group-id'));\n        const groupIdMatch = allDumps[0]?.openTag.match(\n          /data-group-id=\"([^\"]+)\"/,\n        );\n        const mergedGroupId = groupIdMatch\n          ? decodeURIComponent(groupIdMatch[1])\n          : `merged-group-${i}`;\n        const dumpString =\n          allDumps.length > 0\n            ? this.mergeDumpScripts(allDumps.map((d) => d.content))\n            : extractLastDumpScriptSync(reportInfo.reportFilePath);\n        const { reportAttributes } = reportInfo;\n\n        const reportHtmlStr = `${reportHTMLContent(\n          {\n            dumpString,\n            attributes: {\n              'data-group-id': mergedGroupId,\n              playwright_test_duration: reportAttributes.testDuration,\n              playwright_test_status: reportAttributes.testStatus,\n              playwright_test_title: reportAttributes.testTitle,\n              playwright_test_id: reportAttributes.testId,\n              playwright_test_description: reportAttributes.testDescription,\n            },\n          },\n          undefined,\n          undefined,\n          false,\n        )}\\n`;\n\n        appendFileSync(outputFilePath, reportHtmlStr);\n      }\n\n      logMsg(`Successfully merged new report: ${outputFilePath}`);\n\n      // Remove original reports if needed\n      if (rmOriginalReports) {\n        for (const info of this.reportInfos) {\n          try {\n            if (this.isDirectoryModeReport(info.reportFilePath)) {\n              // Directory mode: remove the entire report directory\n              const reportDir = path.dirname(info.reportFilePath);\n              rmSync(reportDir, { recursive: true, force: true });\n            } else {\n              unlinkSync(info.reportFilePath);\n            }\n          } catch (error) {\n            logMsg(`Error deleting report ${info.reportFilePath}: ${error}`);\n          }\n        }\n        logMsg(`Removed ${this.reportInfos.length} original reports`);\n      }\n      return outputFilePath;\n    } catch (error) {\n      logMsg(`Error in mergeReports: ${error}`);\n      throw error;\n    }\n  }\n}\n"],"names":["ReportMergingTool","reportInfo","reportFilePath","reportDir","path","existsSync","contents","unescaped","c","antiEscapeScriptTag","base","ReportActionDump","allExecutions","i","other","noIdCounter","deduped","Map","exec","key","Array","reportFileName","opts","logMsg","rmOriginalReports","overwrite","targetDir","getMidsceneRunSubDir","hasDirectoryModeReport","info","resolvedName","getReportFileName","outputFilePath","Error","rmSync","unlinkSync","mkdirSync","appendFileSync","getReportTpl","getBaseUrlFixScript","screenshotsDir","mergedScreenshotsDir","file","readdirSync","src","dest","copyFileSync","streamImageScriptsToFile","allDumps","extractAllDumpScriptsSync","d","groupIdMatch","mergedGroupId","decodeURIComponent","dumpString","extractLastDumpScriptSync","reportAttributes","reportHtmlStr","reportHTMLContent","undefined","error"],"mappings":";;;;;;;;;;;;;;;;;;AAuBO,MAAMA;IAEJ,OAAOC,UAAoC,EAAE;QAClD,IAAI,CAAC,WAAW,CAAC,IAAI,CAACA;IACxB;IACO,QAAQ;QACb,IAAI,CAAC,WAAW,GAAG,EAAE;IACvB;IAMQ,sBAAsBC,cAAsB,EAAW;QAC7D,MAAMC,YAAYC,QAAaF;QAC/B,OACEE,AAAkC,iBAAlCA,SAAcF,mBACdG,WAAWD,KAAUD,WAAW;IAEpC;IAQQ,iBAAiBG,QAAkB,EAAU;QACnD,MAAMC,YAAYD,SACf,GAAG,CAAC,CAACE,IAAMC,oBAAoBD,IAC/B,MAAM,CAAC,CAACA,IAAMA,EAAE,MAAM,GAAG;QAC5B,IAAID,AAAqB,MAArBA,UAAU,MAAM,EAAQ,OAAO;QACnC,IAAIA,AAAqB,MAArBA,UAAU,MAAM,EAAQ,OAAOA,SAAS,CAAC,EAAE;QAK/C,MAAMG,OAAOC,iBAAiB,oBAAoB,CAACJ,SAAS,CAAC,EAAE;QAC/D,MAAMK,gBAAgB;eAAIF,KAAK,UAAU;SAAC;QAC1C,IAAK,IAAIG,IAAI,GAAGA,IAAIN,UAAU,MAAM,EAAEM,IAAK;YACzC,MAAMC,QAAQH,iBAAiB,oBAAoB,CAACJ,SAAS,CAACM,EAAE;YAChED,cAAc,IAAI,IAAIE,MAAM,UAAU;QACxC;QACA,IAAIC,cAAc;QAClB,MAAMC,UAAU,IAAIC;QACpB,KAAK,MAAMC,QAAQN,cAAe;YAChC,MAAMO,MAAMD,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAEH,eAAe;YACjDC,QAAQ,GAAG,CAACG,KAAKD;QACnB;QACAR,KAAK,UAAU,GAAGU,MAAM,IAAI,CAACJ,QAAQ,MAAM;QAC3C,OAAON,KAAK,SAAS;IACvB;IAEO,aACLW,iBAAkC,MAAM,EACxCC,IAGC,EACc;QACf,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,GAAG;YAChCC,OAAO;YACP,OAAO;QACT;QAEA,MAAM,EAAEC,oBAAoB,KAAK,EAAEC,YAAY,KAAK,EAAE,GAAGH,QAAQ,CAAC;QAClE,MAAMI,YAAYC,qBAAqB;QAGvC,MAAMC,yBAAyB,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAACC,OACpD,IAAI,CAAC,qBAAqB,CAACA,KAAK,cAAc;QAGhD,MAAMC,eACJT,AAAmB,WAAnBA,iBACIU,kBAAkB,mBAClBV;QAIN,MAAMW,iBAAiBJ,yBACnBxB,QAAasB,WAAWI,cAAc,gBACtC1B,QAAasB,WAAW,GAAGI,aAAa,KAAK,CAAC;QAElD,IAAIT,AAAmB,WAAnBA,kBAA6BhB,WAAW2B,iBAAiB;YAC3D,IAAI,CAACP,WACH,MAAM,IAAIQ,MACR,CAAC,4BAA4B,EAAED,eAAe,+CAA+C,CAAC;YAGlG,IAAIJ,wBACFM,OAAO9B,QAAa4B,iBAAiB;gBAAE,WAAW;gBAAM,OAAO;YAAK;iBAEpEG,WAAWH;QAEf;QAEA,IAAIJ,wBACFQ,UAAUhC,QAAa4B,iBAAiB;YAAE,WAAW;QAAK;QAG5DT,OACE,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,sCAAsC,CAAC;QAGlF,IAAI;YAEFc,eAAeL,gBAAgBM;YAG/B,IAAIV,wBACFS,eAAeL,gBAAgBO;YAIjC,IAAK,IAAI1B,IAAI,GAAGA,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAEA,IAAK;gBAChD,MAAMZ,aAAa,IAAI,CAAC,WAAW,CAACY,EAAE;gBACtCU,OAAO,CAAC,kBAAkB,EAAEV,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;gBAE9D,IAAI,IAAI,CAAC,qBAAqB,CAACZ,WAAW,cAAc,GAAG;oBAEzD,MAAME,YAAYC,QAAaH,WAAW,cAAc;oBACxD,MAAMuC,iBAAiBpC,KAAUD,WAAW;oBAC5C,MAAMsC,uBAAuBrC,KAC3BA,QAAa4B,iBACb;oBAEFI,UAAUK,sBAAsB;wBAAE,WAAW;oBAAK;oBAClD,KAAK,MAAMC,QAAQC,YAAYH,gBAAiB;wBAC9C,MAAMI,MAAMxC,KAAUoC,gBAAgBE;wBACtC,MAAMG,OAAOzC,KAAUqC,sBAAsBC;wBAC7CI,aAAaF,KAAKC;oBACpB;gBACF,OAEEE,yBAAyB9C,WAAW,cAAc,EAAE+B;gBAStD,MAAMgB,WAAWC,0BACfhD,WAAW,cAAc,EACzB,MAAM,CAAC,CAACiD,IAAMA,EAAE,OAAO,CAAC,QAAQ,CAAC;gBACnC,MAAMC,eAAeH,QAAQ,CAAC,EAAE,EAAE,QAAQ,MACxC;gBAEF,MAAMI,gBAAgBD,eAClBE,mBAAmBF,YAAY,CAAC,EAAE,IAClC,CAAC,aAAa,EAAEtC,GAAG;gBACvB,MAAMyC,aACJN,SAAS,MAAM,GAAG,IACd,IAAI,CAAC,gBAAgB,CAACA,SAAS,GAAG,CAAC,CAACE,IAAMA,EAAE,OAAO,KACnDK,0BAA0BtD,WAAW,cAAc;gBACzD,MAAM,EAAEuD,gBAAgB,EAAE,GAAGvD;gBAE7B,MAAMwD,gBAAgB,GAAGC,kBACvB;oBACEJ;oBACA,YAAY;wBACV,iBAAiBF;wBACjB,0BAA0BI,iBAAiB,YAAY;wBACvD,wBAAwBA,iBAAiB,UAAU;wBACnD,uBAAuBA,iBAAiB,SAAS;wBACjD,oBAAoBA,iBAAiB,MAAM;wBAC3C,6BAA6BA,iBAAiB,eAAe;oBAC/D;gBACF,GACAG,QACAA,QACA,OACA,EAAE,CAAC;gBAELtB,eAAeL,gBAAgByB;YACjC;YAEAlC,OAAO,CAAC,gCAAgC,EAAES,gBAAgB;YAG1D,IAAIR,mBAAmB;gBACrB,KAAK,MAAMK,QAAQ,IAAI,CAAC,WAAW,CACjC,IAAI;oBACF,IAAI,IAAI,CAAC,qBAAqB,CAACA,KAAK,cAAc,GAAG;wBAEnD,MAAM1B,YAAYC,QAAayB,KAAK,cAAc;wBAClDK,OAAO/B,WAAW;4BAAE,WAAW;4BAAM,OAAO;wBAAK;oBACnD,OACEgC,WAAWN,KAAK,cAAc;gBAElC,EAAE,OAAO+B,OAAO;oBACdrC,OAAO,CAAC,sBAAsB,EAAEM,KAAK,cAAc,CAAC,EAAE,EAAE+B,OAAO;gBACjE;gBAEFrC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;YAC9D;YACA,OAAOS;QACT,EAAE,OAAO4B,OAAO;YACdrC,OAAO,CAAC,uBAAuB,EAAEqC,OAAO;YACxC,MAAMA;QACR;IACF;;QA1MA,uBAAQ,eAA0C,EAAE;;AA2MtD"}
+{"version":3,"file":"report.mjs","sources":["../../src/report.ts"],"sourcesContent":["import {\n  appendFileSync,\n  copyFileSync,\n  existsSync,\n  mkdirSync,\n  readdirSync,\n  rmSync,\n  unlinkSync,\n} from 'node:fs';\nimport * as path from 'node:path';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport { antiEscapeScriptTag, logMsg } from '@midscene/shared/utils';\nimport { getReportFileName } from './agent';\nimport {\n  extractAllDumpScriptsSync,\n  extractLastDumpScriptSync,\n  getBaseUrlFixScript,\n  streamImageScriptsToFile,\n} from './dump/html-utils';\nimport { ReportActionDump } from './types';\nimport type { ReportFileWithAttributes } from './types';\nimport { getReportTpl, reportHTMLContent } from './utils';\n\nexport class ReportMergingTool {\n  private reportInfos: ReportFileWithAttributes[] = [];\n\n  private createEmptyDumpString(groupName: string, groupDescription?: string) {\n    return new ReportActionDump({\n      sdkVersion: '',\n      groupName,\n      groupDescription,\n      modelBriefs: [],\n      executions: [],\n    }).serialize();\n  }\n\n  public append(reportInfo: ReportFileWithAttributes) {\n    this.reportInfos.push(reportInfo);\n  }\n  public clear() {\n    this.reportInfos = [];\n  }\n\n  /**\n   * Check if a report is in directory mode (html-and-external-assets).\n   * Directory mode reports: {name}/index.html + {name}/screenshots/\n   */\n  private isDirectoryModeReport(reportFilePath: string): boolean {\n    const reportDir = path.dirname(reportFilePath);\n    return (\n      path.basename(reportFilePath) === 'index.html' &&\n      existsSync(path.join(reportDir, 'screenshots'))\n    );\n  }\n\n  /**\n   * Merge multiple dump script contents (from the same source report)\n   * into a single serialized ReportActionDump string.\n   * If there's only one dump, return it as-is. If multiple, merge\n   * all executions into the first dump's group structure.\n   */\n  private mergeDumpScripts(contents: string[]): string {\n    const unescaped = contents\n      .map((c) => antiEscapeScriptTag(c))\n      .filter((c) => c.length > 0);\n    if (unescaped.length === 0) return '';\n    if (unescaped.length === 1) return unescaped[0];\n\n    // Parse all dumps and collect executions, deduplicating by id (keep last).\n    // Only executions with a stable id are deduped; old-format entries without\n    // id are always kept (they may be distinct despite sharing the same name).\n    const base = ReportActionDump.fromSerializedString(unescaped[0]);\n    const allExecutions = [...base.executions];\n    for (let i = 1; i < unescaped.length; i++) {\n      const other = ReportActionDump.fromSerializedString(unescaped[i]);\n      allExecutions.push(...other.executions);\n    }\n    let noIdCounter = 0;\n    const deduped = new Map<string, (typeof allExecutions)[0]>();\n    for (const exec of allExecutions) {\n      const key = exec.id || `__no_id_${noIdCounter++}`;\n      deduped.set(key, exec);\n    }\n    base.executions = Array.from(deduped.values());\n    return base.serialize();\n  }\n\n  public mergeReports(\n    reportFileName: 'AUTO' | string = 'AUTO',\n    opts?: {\n      rmOriginalReports?: boolean;\n      overwrite?: boolean;\n    },\n  ): string | null {\n    const { rmOriginalReports = false, overwrite = false } = opts ?? {};\n\n    if (this.reportInfos.length === 0) {\n      logMsg('No reports to merge');\n      return null;\n    }\n\n    const targetDir = getMidsceneRunSubDir('report');\n\n    // Check if any source report is directory mode\n    const hasDirectoryModeReport = this.reportInfos.some(\n      (info) =>\n        info.reportFilePath && this.isDirectoryModeReport(info.reportFilePath),\n    );\n\n    const resolvedName =\n      reportFileName === 'AUTO'\n        ? getReportFileName('merged-report')\n        : reportFileName;\n\n    // Directory mode: output as {name}/index.html to keep relative paths working\n    // Inline mode: output as {name}.html (single file)\n    const outputFilePath = hasDirectoryModeReport\n      ? path.resolve(targetDir, resolvedName, 'index.html')\n      : path.resolve(targetDir, `${resolvedName}.html`);\n\n    if (reportFileName !== 'AUTO' && existsSync(outputFilePath)) {\n      if (!overwrite) {\n        throw new Error(\n          `Report file already exists: ${outputFilePath}\\nSet overwrite to true to overwrite this file.`,\n        );\n      }\n      if (hasDirectoryModeReport) {\n        rmSync(path.dirname(outputFilePath), { recursive: true, force: true });\n      } else {\n        unlinkSync(outputFilePath);\n      }\n    }\n\n    if (hasDirectoryModeReport) {\n      mkdirSync(path.dirname(outputFilePath), { recursive: true });\n    }\n\n    logMsg(\n      `Start merging ${this.reportInfos.length} reports...\\nCreating template file...`,\n    );\n\n    try {\n      // Write template without closing </html> tag so we can append\n      // dump scripts before it. The closing tag is added at the end.\n      const htmlEndTag = '</html>';\n      const tpl = getReportTpl();\n      const htmlEndIdx = tpl.lastIndexOf(htmlEndTag);\n      const tplWithoutClose =\n        htmlEndIdx !== -1 ? tpl.slice(0, htmlEndIdx) : tpl;\n      appendFileSync(outputFilePath, tplWithoutClose);\n\n      // For directory-mode output, inject base URL fix script\n      if (hasDirectoryModeReport) {\n        appendFileSync(outputFilePath, getBaseUrlFixScript());\n      }\n\n      // Process all reports one by one\n      for (let i = 0; i < this.reportInfos.length; i++) {\n        const reportInfo = this.reportInfos[i];\n        logMsg(`Processing report ${i + 1}/${this.reportInfos.length}`);\n\n        const { reportAttributes } = reportInfo;\n        let dumpString = this.createEmptyDumpString(\n          reportAttributes.testTitle,\n          reportAttributes.testDescription,\n        );\n        let mergedGroupId = `merged-group-${i}`;\n\n        if (reportInfo.reportFilePath) {\n          if (this.isDirectoryModeReport(reportInfo.reportFilePath)) {\n            // Directory mode: copy external screenshot files\n            const reportDir = path.dirname(reportInfo.reportFilePath);\n            const screenshotsDir = path.join(reportDir, 'screenshots');\n            const mergedScreenshotsDir = path.join(\n              path.dirname(outputFilePath),\n              'screenshots',\n            );\n            mkdirSync(mergedScreenshotsDir, { recursive: true });\n            for (const file of readdirSync(screenshotsDir)) {\n              const src = path.join(screenshotsDir, file);\n              const dest = path.join(mergedScreenshotsDir, file);\n              copyFileSync(src, dest);\n            }\n          } else {\n            // Inline mode: stream image scripts to output file\n            streamImageScriptsToFile(reportInfo.reportFilePath, outputFilePath);\n          }\n\n          // Extract all dump scripts from the source report.\n          // After the per-execution append refactor, a single source report\n          // may contain multiple <script type=\"midscene_web_dump\"> tags\n          // (one per execution). We merge them into a single ReportActionDump.\n          // Filter by data-group-id to exclude false matches from the template's\n          // bundled JS code, which also references the midscene_web_dump type string.\n          const allDumps = extractAllDumpScriptsSync(\n            reportInfo.reportFilePath,\n          ).filter((d) => d.openTag.includes('data-group-id'));\n          const groupIdMatch = allDumps[0]?.openTag.match(\n            /data-group-id=\"([^\"]+)\"/,\n          );\n          if (groupIdMatch) {\n            mergedGroupId = decodeURIComponent(groupIdMatch[1]);\n          }\n          const extractedDumpString =\n            allDumps.length > 0\n              ? this.mergeDumpScripts(allDumps.map((d) => d.content))\n              : extractLastDumpScriptSync(reportInfo.reportFilePath);\n          if (extractedDumpString) {\n            dumpString = extractedDumpString;\n          }\n        }\n\n        const reportHtmlStr = `${reportHTMLContent(\n          {\n            dumpString,\n            attributes: {\n              'data-group-id': mergedGroupId,\n              playwright_test_duration: reportAttributes.testDuration,\n              playwright_test_status: reportAttributes.testStatus,\n              playwright_test_title: reportAttributes.testTitle,\n              playwright_test_id: reportAttributes.testId,\n              playwright_test_description: reportAttributes.testDescription,\n              is_merged: true,\n            },\n          },\n          undefined,\n          undefined,\n          false,\n        )}\\n`;\n\n        appendFileSync(outputFilePath, reportHtmlStr);\n      }\n\n      // Close the HTML document\n      appendFileSync(outputFilePath, `${htmlEndTag}\\n`);\n\n      logMsg(`Successfully merged new report: ${outputFilePath}`);\n\n      // Remove original reports if needed\n      if (rmOriginalReports) {\n        for (const info of this.reportInfos) {\n          if (!info.reportFilePath) continue;\n          try {\n            if (this.isDirectoryModeReport(info.reportFilePath)) {\n              // Directory mode: remove the entire report directory\n              const reportDir = path.dirname(info.reportFilePath);\n              rmSync(reportDir, { recursive: true, force: true });\n            } else {\n              unlinkSync(info.reportFilePath);\n            }\n          } catch (error) {\n            logMsg(`Error deleting report ${info.reportFilePath}: ${error}`);\n          }\n        }\n        logMsg(`Removed ${this.reportInfos.length} original reports`);\n      }\n      return outputFilePath;\n    } catch (error) {\n      logMsg(`Error in mergeReports: ${error}`);\n      throw error;\n    }\n  }\n}\n"],"names":["ReportMergingTool","groupName","groupDescription","ReportActionDump","reportInfo","reportFilePath","reportDir","path","existsSync","contents","unescaped","c","antiEscapeScriptTag","base","allExecutions","i","other","noIdCounter","deduped","Map","exec","key","Array","reportFileName","opts","rmOriginalReports","overwrite","logMsg","targetDir","getMidsceneRunSubDir","hasDirectoryModeReport","info","resolvedName","getReportFileName","outputFilePath","Error","rmSync","unlinkSync","mkdirSync","htmlEndTag","tpl","getReportTpl","htmlEndIdx","tplWithoutClose","appendFileSync","getBaseUrlFixScript","reportAttributes","dumpString","mergedGroupId","screenshotsDir","mergedScreenshotsDir","file","readdirSync","src","dest","copyFileSync","streamImageScriptsToFile","allDumps","extractAllDumpScriptsSync","d","groupIdMatch","decodeURIComponent","extractedDumpString","extractLastDumpScriptSync","reportHtmlStr","reportHTMLContent","undefined","error"],"mappings":";;;;;;;;;;;;;;;;;;AAuBO,MAAMA;IAGH,sBAAsBC,SAAiB,EAAEC,gBAAyB,EAAE;QAC1E,OAAO,IAAIC,iBAAiB;YAC1B,YAAY;YACZF;YACAC;YACA,aAAa,EAAE;YACf,YAAY,EAAE;QAChB,GAAG,SAAS;IACd;IAEO,OAAOE,UAAoC,EAAE;QAClD,IAAI,CAAC,WAAW,CAAC,IAAI,CAACA;IACxB;IACO,QAAQ;QACb,IAAI,CAAC,WAAW,GAAG,EAAE;IACvB;IAMQ,sBAAsBC,cAAsB,EAAW;QAC7D,MAAMC,YAAYC,QAAaF;QAC/B,OACEE,AAAkC,iBAAlCA,SAAcF,mBACdG,WAAWD,KAAUD,WAAW;IAEpC;IAQQ,iBAAiBG,QAAkB,EAAU;QACnD,MAAMC,YAAYD,SACf,GAAG,CAAC,CAACE,IAAMC,oBAAoBD,IAC/B,MAAM,CAAC,CAACA,IAAMA,EAAE,MAAM,GAAG;QAC5B,IAAID,AAAqB,MAArBA,UAAU,MAAM,EAAQ,OAAO;QACnC,IAAIA,AAAqB,MAArBA,UAAU,MAAM,EAAQ,OAAOA,SAAS,CAAC,EAAE;QAK/C,MAAMG,OAAOV,iBAAiB,oBAAoB,CAACO,SAAS,CAAC,EAAE;QAC/D,MAAMI,gBAAgB;eAAID,KAAK,UAAU;SAAC;QAC1C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,UAAU,MAAM,EAAEK,IAAK;YACzC,MAAMC,QAAQb,iBAAiB,oBAAoB,CAACO,SAAS,CAACK,EAAE;YAChED,cAAc,IAAI,IAAIE,MAAM,UAAU;QACxC;QACA,IAAIC,cAAc;QAClB,MAAMC,UAAU,IAAIC;QACpB,KAAK,MAAMC,QAAQN,cAAe;YAChC,MAAMO,MAAMD,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAEH,eAAe;YACjDC,QAAQ,GAAG,CAACG,KAAKD;QACnB;QACAP,KAAK,UAAU,GAAGS,MAAM,IAAI,CAACJ,QAAQ,MAAM;QAC3C,OAAOL,KAAK,SAAS;IACvB;IAEO,aACLU,iBAAkC,MAAM,EACxCC,IAGC,EACc;QACf,MAAM,EAAEC,oBAAoB,KAAK,EAAEC,YAAY,KAAK,EAAE,GAAGF,QAAQ,CAAC;QAElE,IAAI,AAA4B,MAA5B,IAAI,CAAC,WAAW,CAAC,MAAM,EAAQ;YACjCG,OAAO;YACP,OAAO;QACT;QAEA,MAAMC,YAAYC,qBAAqB;QAGvC,MAAMC,yBAAyB,IAAI,CAAC,WAAW,CAAC,IAAI,CAClD,CAACC,OACCA,KAAK,cAAc,IAAI,IAAI,CAAC,qBAAqB,CAACA,KAAK,cAAc;QAGzE,MAAMC,eACJT,AAAmB,WAAnBA,iBACIU,kBAAkB,mBAClBV;QAIN,MAAMW,iBAAiBJ,yBACnBvB,QAAaqB,WAAWI,cAAc,gBACtCzB,QAAaqB,WAAW,GAAGI,aAAa,KAAK,CAAC;QAElD,IAAIT,AAAmB,WAAnBA,kBAA6Bf,WAAW0B,iBAAiB;YAC3D,IAAI,CAACR,WACH,MAAM,IAAIS,MACR,CAAC,4BAA4B,EAAED,eAAe,+CAA+C,CAAC;YAGlG,IAAIJ,wBACFM,OAAO7B,QAAa2B,iBAAiB;gBAAE,WAAW;gBAAM,OAAO;YAAK;iBAEpEG,WAAWH;QAEf;QAEA,IAAIJ,wBACFQ,UAAU/B,QAAa2B,iBAAiB;YAAE,WAAW;QAAK;QAG5DP,OACE,CAAC,cAAc,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,sCAAsC,CAAC;QAGlF,IAAI;YAGF,MAAMY,aAAa;YACnB,MAAMC,MAAMC;YACZ,MAAMC,aAAaF,IAAI,WAAW,CAACD;YACnC,MAAMI,kBACJD,AAAe,OAAfA,aAAoBF,IAAI,KAAK,CAAC,GAAGE,cAAcF;YACjDI,eAAeV,gBAAgBS;YAG/B,IAAIb,wBACFc,eAAeV,gBAAgBW;YAIjC,IAAK,IAAI9B,IAAI,GAAGA,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,EAAEA,IAAK;gBAChD,MAAMX,aAAa,IAAI,CAAC,WAAW,CAACW,EAAE;gBACtCY,OAAO,CAAC,kBAAkB,EAAEZ,IAAI,EAAE,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE;gBAE9D,MAAM,EAAE+B,gBAAgB,EAAE,GAAG1C;gBAC7B,IAAI2C,aAAa,IAAI,CAAC,qBAAqB,CACzCD,iBAAiB,SAAS,EAC1BA,iBAAiB,eAAe;gBAElC,IAAIE,gBAAgB,CAAC,aAAa,EAAEjC,GAAG;gBAEvC,IAAIX,WAAW,cAAc,EAAE;oBAC7B,IAAI,IAAI,CAAC,qBAAqB,CAACA,WAAW,cAAc,GAAG;wBAEzD,MAAME,YAAYC,QAAaH,WAAW,cAAc;wBACxD,MAAM6C,iBAAiB1C,KAAUD,WAAW;wBAC5C,MAAM4C,uBAAuB3C,KAC3BA,QAAa2B,iBACb;wBAEFI,UAAUY,sBAAsB;4BAAE,WAAW;wBAAK;wBAClD,KAAK,MAAMC,QAAQC,YAAYH,gBAAiB;4BAC9C,MAAMI,MAAM9C,KAAU0C,gBAAgBE;4BACtC,MAAMG,OAAO/C,KAAU2C,sBAAsBC;4BAC7CI,aAAaF,KAAKC;wBACpB;oBACF,OAEEE,yBAAyBpD,WAAW,cAAc,EAAE8B;oBAStD,MAAMuB,WAAWC,0BACftD,WAAW,cAAc,EACzB,MAAM,CAAC,CAACuD,IAAMA,EAAE,OAAO,CAAC,QAAQ,CAAC;oBACnC,MAAMC,eAAeH,QAAQ,CAAC,EAAE,EAAE,QAAQ,MACxC;oBAEF,IAAIG,cACFZ,gBAAgBa,mBAAmBD,YAAY,CAAC,EAAE;oBAEpD,MAAME,sBACJL,SAAS,MAAM,GAAG,IACd,IAAI,CAAC,gBAAgB,CAACA,SAAS,GAAG,CAAC,CAACE,IAAMA,EAAE,OAAO,KACnDI,0BAA0B3D,WAAW,cAAc;oBACzD,IAAI0D,qBACFf,aAAae;gBAEjB;gBAEA,MAAME,gBAAgB,GAAGC,kBACvB;oBACElB;oBACA,YAAY;wBACV,iBAAiBC;wBACjB,0BAA0BF,iBAAiB,YAAY;wBACvD,wBAAwBA,iBAAiB,UAAU;wBACnD,uBAAuBA,iBAAiB,SAAS;wBACjD,oBAAoBA,iBAAiB,MAAM;wBAC3C,6BAA6BA,iBAAiB,eAAe;wBAC7D,WAAW;oBACb;gBACF,GACAoB,QACAA,QACA,OACA,EAAE,CAAC;gBAELtB,eAAeV,gBAAgB8B;YACjC;YAGApB,eAAeV,gBAAgB,GAAGK,WAAW,EAAE,CAAC;YAEhDZ,OAAO,CAAC,gCAAgC,EAAEO,gBAAgB;YAG1D,IAAIT,mBAAmB;gBACrB,KAAK,MAAMM,QAAQ,IAAI,CAAC,WAAW,CACjC,IAAKA,KAAK,cAAc,EACxB,IAAI;oBACF,IAAI,IAAI,CAAC,qBAAqB,CAACA,KAAK,cAAc,GAAG;wBAEnD,MAAMzB,YAAYC,QAAawB,KAAK,cAAc;wBAClDK,OAAO9B,WAAW;4BAAE,WAAW;4BAAM,OAAO;wBAAK;oBACnD,OACE+B,WAAWN,KAAK,cAAc;gBAElC,EAAE,OAAOoC,OAAO;oBACdxC,OAAO,CAAC,sBAAsB,EAAEI,KAAK,cAAc,CAAC,EAAE,EAAEoC,OAAO;gBACjE;gBAEFxC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,iBAAiB,CAAC;YAC9D;YACA,OAAOO;QACT,EAAE,OAAOiC,OAAO;YACdxC,OAAO,CAAC,uBAAuB,EAAEwC,OAAO;YACxC,MAAMA;QACR;IACF;;QA7OA,uBAAQ,eAA0C,EAAE;;AA8OtD"}