@midscene/core 1.0.1-beta-20251202112442.0 → 1.0.1-beta-20251203073716.0

package/dist/lib/agent/tasks.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { ConversationHistory, plan, uiTarsPlanning } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ThinkingStrategy,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly taskBuilder: TaskBuilder;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      hooks?: TaskExecutorHooks;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.hooks = opts.hooks;\n    this.conversationHistory = new ConversationHistory();\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      subTask?: boolean;\n    },\n  ) {\n    return this.taskBuilder.build(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      options,\n    );\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    thinkingStrategy: ThinkingStrategy,\n    backgroundKnowledge?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            aiActionContext: backgroundKnowledge,\n          },\n          executor: async (param, executorContext) => {\n            const startTime = Date.now();\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const { vlMode } = modelConfigForPlanning;\n            const uiTarsModelVersion =\n              vlMode === 'vlm-ui-tars'\n                ? modelConfigForPlanning.uiTarsModelVersion\n                : undefined;\n\n            assert(\n              this.interface.actionSpace,\n              'actionSpace for device is not implemented',\n            );\n            const actionSpace = await this.interface.actionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planResult = await (uiTarsModelVersion\n              ? uiTarsPlanning\n              : plan)(param.userInstruction, {\n              context: uiContext,\n              actionContext: param.aiActionContext,\n              interfaceType: this.interface.interfaceType as InterfaceType,\n              actionSpace,\n              modelConfig: modelConfigForPlanning,\n              conversationHistory: this.conversationHistory,\n              includeBbox: includeBboxInPlanning,\n              thinkingStrategy,\n            });\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              log,\n              more_actions_needed_by_instruction,\n              error,\n              usage,\n              rawResponse,\n              sleep,\n            } = planResult;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n            };\n            executorContext.task.usage = usage;\n\n            const finalActions = actions || [];\n\n            if (sleep) {\n              const timeNow = Date.now();\n              const timeRemaining = sleep - (timeNow - startTime);\n              if (timeRemaining > 0) {\n                finalActions.push(this.sleepPlan(timeRemaining));\n              }\n            }\n\n            if (finalActions.length === 0) {\n              assert(\n                !more_actions_needed_by_instruction || sleep,\n                error ? `Failed to plan: ${error}` : 'No plan found',\n              );\n            }\n\n            return {\n              output: {\n                actions: finalActions,\n                more_actions_needed_by_instruction,\n                log,\n                yamlFlow: planResult.yamlFlow,\n              },\n              cache: {\n                hit: false,\n              },\n              uiContext,\n            };\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          modelConfigForPlanning,\n          modelConfigForDefaultIntent,\n          {\n            cacheable,\n            subTask: true,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (this.conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          this.conversationHistory.pendingFeedbackMessage,\n        );\n      }\n      try {\n        await session.appendAndRun(executables.tasks);\n      } catch (error: any) {\n        errorCountInOnePlanningLoop++;\n        this.conversationHistory.pendingFeedbackMessage = `Error executing running tasks: ${error?.message || String(error)}`;\n        debug(\n          'error when executing running tasks, but continue to run:',\n          error instanceof Error ? error.message : String(error),\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check if task is complete\n      if (!planResult?.more_actions_needed_by_instruction) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!this.conversationHistory.pendingFeedbackMessage) {\n        this.conversationHistory.pendingFeedbackMessage =\n          'I have finished the action previously planned.';\n      }\n    }\n\n    const finalResult = {\n      output: {\n        yamlFlow,\n      },\n      runner,\n    };\n    return finalResult;\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n          };\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, usage, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.usage = usage;\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  private sleepPlan(timeMs: number): PlanningAction<PlanningActionParamSleep> {\n    return {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n    };\n  }\n\n  async taskForSleep(timeMs: number, _modelConfig: IModelConfig) {\n    return this.taskBuilder.createSleepTask({\n      timeMs,\n    });\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        undefined,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - currentCheckStart);\n        const sleepTask = this.taskBuilder.createSleepTask({\n          timeMs: timeRemaining,\n        });\n        await session.append(sleepTask);\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","_this_hooks","ExecutionSession","Promise","plans","modelConfigForPlanning","modelConfigForDefaultIntent","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeBboxInPlanning","thinkingStrategy","backgroundKnowledge","cacheable","replanningCycleLimitOverride","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","startTime","Date","vlMode","uiTarsModelVersion","actionSpace","action","Array","console","planResult","uiTarsPlanning","plan","JSON","actions","log","more_actions_needed_by_instruction","error","usage","rawResponse","sleep","finalActions","timeNow","timeRemaining","executables","String","Error","errorMsg","finalResult","type","demand","modelConfig","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","thought","outputResult","timeMs","_modelConfig","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","lastCheckStart","errorThought","currentCheckStart","now","sleepTask","interfaceInstance","service","opts","ConversationHistory","TaskBuilder"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACyCA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,wCAAwC;AAIvC,MAAMC;IAkBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IA0BQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;YAOkBC;QANlB,OAAO,IAAIC,8CAAAA,gBAAgBA,CACzBH,OACA,IAAMI,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOH,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK;YACrB,cAAc,QAAAC,CAAAA,cAAAA,IAAI,CAAC,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,YAAY;QACxC;IAEJ;IAEA,MAAa,wBACXG,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCN,OAGC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAC3BI,OACAC,wBACAC,6BACAN;IAEJ;IAEA,MAAM,uBAAuBO,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,UAAUH;QAGzB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJjB,KAAa,EACbK,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACf;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACV;QAC5C,MAAM,EAAEkB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,wBACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,gBAAkC,EAClCC,mBAA4B,EAC5BC,SAAmB,EACnBC,4BAAqC,EAQrC;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMhB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,UAAUU;QAEzB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIiB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJH,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DV,IAAAA,sBAAAA,MAAAA,AAAAA,EACEa,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAGlC,MAAO,KAAM;YACX,MAAMZ,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjB,iBAAiBG;gBACnB;gBACA,UAAU,OAAOX,OAAOC;oBACtB,MAAMkB,YAAYC,KAAK,GAAG;oBAC1B,MAAM,EAAElB,SAAS,EAAE,GAAGD;oBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;oBAClB,MAAM,EAAEmB,MAAM,EAAE,GAAG5B;oBACnB,MAAM6B,qBACJD,AAAW,kBAAXA,SACI5B,uBAAuB,kBAAkB,GACzCwB;oBAENd,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;oBAEF,MAAMoB,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;oBACpDxC,MACE,sCACAwC,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhDrB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsB,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,aAAa,MAAOL,AAAAA,CAAAA,qBACtBM,yBAAAA,cAAcA,GACdC,yBAAAA,IAAG,AAAHA,EAAM7B,MAAM,eAAe,EAAE;wBAC/B,SAASE;wBACT,eAAeF,MAAM,eAAe;wBACpC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;wBAC3CuB;wBACA,aAAa9B;wBACb,qBAAqB,IAAI,CAAC,mBAAmB;wBAC7C,aAAagB;wBACbC;oBACF;oBACA3B,MAAM,cAAc+C,KAAK,SAAS,CAACH,YAAY,MAAM;oBAErD,MAAM,EACJI,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClCC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,KAAK,EACN,GAAGV;oBAEJ1B,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCmC;oBACF;oBACAnC,gBAAgB,IAAI,CAAC,KAAK,GAAGkC;oBAE7B,MAAMG,eAAeP,WAAW,EAAE;oBAElC,IAAIM,OAAO;wBACT,MAAME,UAAUnB,KAAK,GAAG;wBACxB,MAAMoB,gBAAgBH,QAASE,CAAAA,UAAUpB,SAAQ;wBACjD,IAAIqB,gBAAgB,GAClBF,aAAa,IAAI,CAAC,IAAI,CAAC,SAAS,CAACE;oBAErC;oBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrBnC,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAAC8B,sCAAsCI,OACvCH,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;oBAIzC,OAAO;wBACL,QAAQ;4BACN,SAASI;4BACTL;4BACAD;4BACA,UAAUL,WAAW,QAAQ;wBAC/B;wBACA,OAAO;4BACL,KAAK;wBACP;wBACAzB;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMyB,aAAarB,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YAGjC,MAAMd,QAAQmC,AAAAA,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,OAAO,AAAD,KAAK,EAAE;YACvCZ,SAAS,IAAI,IAAKY,AAAAA,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,QAAQ,AAAD,KAAK,EAAE;YAE5C,IAAIc;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CjD,OACAC,wBACAC,6BACA;oBACEkB;oBACA,SAAS;gBACX;YAEJ,EAAE,OAAOsB,OAAO;gBACd,OAAOrC,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEqC,MAAM,SAAS,EAAEJ,KAAK,SAAS,CAC5EtC,QACC;YAEP;YACA,IAAI,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,EACjDkC,QAAQ,IAAI,CACV,8FACA,IAAI,CAAC,mBAAmB,CAAC,sBAAsB;YAGnD,IAAI;gBACF,MAAM7B,QAAQ,YAAY,CAAC4C,YAAY,KAAK;YAC9C,EAAE,OAAOP,OAAY;gBACnBhB;gBACA,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,GAAG,CAAC,+BAA+B,EAAEgB,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,OAAO,AAAD,KAAKQ,OAAOR,QAAQ;gBACrHnD,MACE,4DACAmD,iBAAiBS,QAAQT,MAAM,OAAO,GAAGQ,OAAOR;YAEpD;YAEA,IAAIhB,8BAA8BjC,uCAChC,OAAOY,QAAQ,eAAe,CAAC;YAIjC,IAAI,CAAC8B,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,kCAAkC,AAAD,GAChD;YAIF,EAAEb;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAM4B,WAAW,CAAC,UAAU,EAAE5B,qBAAqB,4JAA4J,CAAC;gBAChN,OAAOnB,QAAQ,eAAe,CAAC+C;YACjC;YAEA,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,EAClD,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,GAC7C;QAEN;QAEA,MAAMC,cAAc;YAClB,QAAQ;gBACN9B;YACF;YACAX;QACF;QACA,OAAOyC;IACT;IAEQ,oBACNC,IAAsE,EACtEC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,YAAYI,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAO/C,OAAOoD;gBACtB,MAAM,EAAErD,IAAI,EAAE,GAAGqD;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZxD,KAAK,GAAG,GAAG;wBACTwD;oBACF;gBACF;gBAGA,MAAMrD,YAAYkD,YAAY,SAAS;gBACvCjD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElB,MAAMsD,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACd,MAAMC,gBACJb,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIU,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGZ,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DlE,MAAM;oBACN,MAAM+E,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,AAAAA,IAAAA,0BAAAA,iBAAAA,AAAAA,EAC3BD,MACA,KACA,OACAb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,aACAC,KACAY,sBACAX;gBAEJ,EAAE,OAAOhB,OAAO;oBACd,IAAIA,iBAAiB8B,kCAAAA,YAAYA,EAC/BV,UAAUpB,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE+B,IAAI,EAAE9B,KAAK,EAAE+B,OAAO,EAAEX,IAAI,EAAE,GAAGK;gBACvCN,UAAUC;gBAEV,IAAIY,eAAeF;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTE,eAAeF;qBACV,IAAInB,AAAS,cAATA,MAEPqB,eADEF,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTE,eAAe;qBACV;oBACLhE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8D,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,IAAM,CAACP,YAAY,AAAD,MAAMzC,QACxB;oBAEFkD,eAAgBF,IAAY,CAACP,YAAY;gBAC3C;gBAGF,IAAIZ,AAAS,aAATA,QAAqB,CAACqB,cAAc;oBACtCpE,KAAK,KAAK,GAAGoC;oBACbpC,KAAK,OAAO,GAAGmE;oBACf,MAAM,IAAIvB,MAAM,CAAC,kBAAkB,EAAEuB,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQC;oBACR,KAAKd;oBACLlB;oBACA+B;gBACF;YACF;QACF;QAEA,OAAOf;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMrD,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEgD,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASjB,KAAK,SAAS,CAACiB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,aACAC,KACAC;QAGF,MAAM9C,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACsD;QAE1C,IAAI,CAAC7C,QACH,MAAM,IAAIqC,MACR;QAIJ,MAAM,EAAEpC,MAAM,EAAE2D,OAAO,EAAE,GAAG5D;QAE5B,OAAO;YACLC;YACA2D;YACA9D;QACF;IACF;IAEQ,UAAUgE,MAAc,EAA4C;QAC1E,OAAO;YACL,MAAM;YACN,OAAO;gBACLA;YACF;QACF;IACF;IAEA,MAAM,aAAaA,MAAc,EAAEC,YAA0B,EAAE;QAC7D,OAAO,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;YACtCD;QACF;IACF;IAEA,MAAM,QACJE,SAAsB,EACtBrB,GAA+B,EAC/BD,WAAyB,EACO;QAChC,MAAM,EAAEuB,UAAU,EAAErB,gBAAgB,EAAE,GAAGsB,AAAAA,IAAAA,kCAAAA,WAAAA,AAAAA,EAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM1E,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,WAAW2E;QAE1B,MAAMrE,SAASP,QAAQ,SAAS;QAChC,MAAM,EAAE6E,SAAS,EAAEC,eAAe,EAAE,GAAG1B;QAEvC9C,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOmE,WAAW;QAClBnE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuE,WAAW;QAClBvE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOwE,iBAAiB;QAExBxE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEwE,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBxD,KAAK,GAAG;QACjC,IAAIyD,iBAAiBD;QACrB,IAAIE,eAAe;QAEnB,MAAOD,iBAAiBD,oBAAoBF,UAAW;YACrD,MAAMK,oBAAoB3D,KAAK,GAAG;YAClCyD,iBAAiBE;YACjB,MAAM5B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAoB,YACAvB,aACA/B,QACAiC;YAGF,MAAM5C,SAAU,MAAMT,QAAQ,YAAY,CAACsD;YAO3C,IAAI7C,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQW;gBACRb;YACF;YAGF0E,eACExE,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACb,CAACA,UAAU,CAAC,0BAA0B,EAAEiE,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMS,MAAM5D,KAAK,GAAG;YACpB,IAAI4D,MAAMD,oBAAoBJ,iBAAiB;gBAC7C,MAAMnC,gBAAgBmC,kBAAmBK,CAAAA,MAAMD,iBAAgB;gBAC/D,MAAME,YAAY,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;oBACjD,QAAQzC;gBACV;gBACA,MAAM3C,QAAQ,MAAM,CAACoF;YACvB;QACF;QAEA,OAAOpF,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEiF,cAAc;IACnE;IA3jBA,YACEI,iBAAoC,EACpCC,OAAgB,EAChBC,IAKC,CACD;QA9BF;QAEA;QAEA;QAEA,uBAAiB,eAAjB;QAEA,uBAAQ,uBAAR;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAiBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;QAC5C,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAG,IAAIC,yBAAAA,mBAAmBA;QAClD,IAAI,CAAC,WAAW,GAAG,IAAIC,yCAAAA,WAAWA,CAAC;YACjCJ;YACAC;YACA,WAAWC,KAAK,SAAS;QAC3B;IACF;AAsiBF"}
+{"version":3,"file":"agent/tasks.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/tasks.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { ConversationHistory, plan, uiTarsPlanning } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ThinkingStrategy,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly taskBuilder: TaskBuilder;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      hooks?: TaskExecutorHooks;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.hooks = opts.hooks;\n    this.conversationHistory = new ConversationHistory();\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      subTask?: boolean;\n    },\n  ) {\n    return this.taskBuilder.build(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      options,\n    );\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    thinkingStrategy: ThinkingStrategy,\n    backgroundKnowledge?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            aiActionContext: backgroundKnowledge,\n          },\n          executor: async (param, executorContext) => {\n            const startTime = Date.now();\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const { vlMode } = modelConfigForPlanning;\n            const uiTarsModelVersion =\n              vlMode === 'vlm-ui-tars'\n                ? modelConfigForPlanning.uiTarsModelVersion\n                : undefined;\n\n            assert(\n              this.interface.actionSpace,\n              'actionSpace for device is not implemented',\n            );\n            const actionSpace = await this.interface.actionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planResult = await (uiTarsModelVersion\n              ? uiTarsPlanning\n              : plan)(param.userInstruction, {\n              context: uiContext,\n              actionContext: param.aiActionContext,\n              interfaceType: this.interface.interfaceType as InterfaceType,\n              actionSpace,\n              modelConfig: modelConfigForPlanning,\n              conversationHistory: this.conversationHistory,\n              includeBbox: includeBboxInPlanning,\n              thinkingStrategy,\n            });\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              log,\n              more_actions_needed_by_instruction,\n              error,\n              usage,\n              rawResponse,\n              sleep,\n            } = planResult;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n            };\n            executorContext.task.usage = usage;\n\n            const finalActions = actions || [];\n\n            if (sleep) {\n              const timeNow = Date.now();\n              const timeRemaining = sleep - (timeNow - startTime);\n              if (timeRemaining > 0) {\n                finalActions.push(this.sleepPlan(timeRemaining));\n              }\n            }\n\n            if (finalActions.length === 0) {\n              assert(\n                !more_actions_needed_by_instruction || sleep,\n                error ? `Failed to plan: ${error}` : 'No plan found',\n              );\n            }\n\n            return {\n              output: {\n                actions: finalActions,\n                more_actions_needed_by_instruction,\n                log,\n                yamlFlow: planResult.yamlFlow,\n              },\n              cache: {\n                hit: false,\n              },\n              uiContext,\n            };\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          modelConfigForPlanning,\n          modelConfigForDefaultIntent,\n          {\n            cacheable,\n            subTask: true,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (this.conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          this.conversationHistory.pendingFeedbackMessage,\n        );\n      }\n      try {\n        await session.appendAndRun(executables.tasks);\n      } catch (error: any) {\n        errorCountInOnePlanningLoop++;\n        this.conversationHistory.pendingFeedbackMessage = `Error executing running tasks: ${error?.message || String(error)}`;\n        debug(\n          'error when executing running tasks, but continue to run:',\n          error instanceof Error ? error.message : String(error),\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check if task is complete\n      if (!planResult?.more_actions_needed_by_instruction) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!this.conversationHistory.pendingFeedbackMessage) {\n        this.conversationHistory.pendingFeedbackMessage =\n          'I have finished the action previously planned.';\n      }\n    }\n\n    const finalResult = {\n      output: {\n        yamlFlow,\n      },\n      runner,\n    };\n    return finalResult;\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n          };\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, usage, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.usage = usage;\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  private sleepPlan(timeMs: number): PlanningAction<PlanningActionParamSleep> {\n    return {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n    };\n  }\n\n  async taskForSleep(timeMs: number, _modelConfig: IModelConfig) {\n    return this.taskBuilder.createSleepTask({\n      timeMs,\n    });\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        undefined,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - currentCheckStart);\n        const sleepTask = this.taskBuilder.createSleepTask({\n          timeMs: timeRemaining,\n        });\n        await session.append(sleepTask);\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","plans","modelConfigForPlanning","modelConfigForDefaultIntent","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeBboxInPlanning","thinkingStrategy","backgroundKnowledge","cacheable","replanningCycleLimitOverride","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","startTime","Date","vlMode","uiTarsModelVersion","actionSpace","action","Array","console","planResult","uiTarsPlanning","plan","JSON","actions","log","more_actions_needed_by_instruction","error","usage","rawResponse","sleep","finalActions","timeNow","timeRemaining","executables","String","Error","errorMsg","finalResult","type","demand","modelConfig","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","thought","outputResult","timeMs","_modelConfig","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","lastCheckStart","errorThought","currentCheckStart","now","sleepTask","interfaceInstance","service","opts","ConversationHistory","TaskBuilder"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACyCA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,wCAAwC;AAIvC,MAAMC;IAkBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IA0BQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,8CAAAA,gBAAgBA,CACzBF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEA,MAAa,wBACXG,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCL,OAGC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAC3BG,OACAC,wBACAC,6BACAL;IAEJ;IAEA,MAAM,uBAAuBM,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,UAAUH;QAGzB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJhB,KAAa,EACbI,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACf;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACT;QAC5C,MAAM,EAAEiB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,wBACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,gBAAkC,EAClCC,mBAA4B,EAC5BC,SAAmB,EACnBC,4BAAqC,EAQrC;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMhB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,UAAUU;QAEzB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIiB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJH,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DV,IAAAA,sBAAAA,MAAAA,AAAAA,EACEa,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAGlC,MAAO,KAAM;YACX,MAAMZ,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjB,iBAAiBG;gBACnB;gBACA,UAAU,OAAOX,OAAOC;oBACtB,MAAMkB,YAAYC,KAAK,GAAG;oBAC1B,MAAM,EAAElB,SAAS,EAAE,GAAGD;oBACtBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;oBAClB,MAAM,EAAEmB,MAAM,EAAE,GAAG5B;oBACnB,MAAM6B,qBACJD,AAAW,kBAAXA,SACI5B,uBAAuB,kBAAkB,GACzCwB;oBAENd,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;oBAEF,MAAMoB,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;oBACpDvC,MACE,sCACAuC,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhDrB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsB,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,aAAa,MAAOL,AAAAA,CAAAA,qBACtBM,yBAAAA,cAAcA,GACdC,yBAAAA,IAAG,AAAHA,EAAM7B,MAAM,eAAe,EAAE;wBAC/B,SAASE;wBACT,eAAeF,MAAM,eAAe;wBACpC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;wBAC3CuB;wBACA,aAAa9B;wBACb,qBAAqB,IAAI,CAAC,mBAAmB;wBAC7C,aAAagB;wBACbC;oBACF;oBACA1B,MAAM,cAAc8C,KAAK,SAAS,CAACH,YAAY,MAAM;oBAErD,MAAM,EACJI,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClCC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,KAAK,EACN,GAAGV;oBAEJ1B,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCmC;oBACF;oBACAnC,gBAAgB,IAAI,CAAC,KAAK,GAAGkC;oBAE7B,MAAMG,eAAeP,WAAW,EAAE;oBAElC,IAAIM,OAAO;wBACT,MAAME,UAAUnB,KAAK,GAAG;wBACxB,MAAMoB,gBAAgBH,QAASE,CAAAA,UAAUpB,SAAQ;wBACjD,IAAIqB,gBAAgB,GAClBF,aAAa,IAAI,CAAC,IAAI,CAAC,SAAS,CAACE;oBAErC;oBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrBnC,AAAAA,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAAC8B,sCAAsCI,OACvCH,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;oBAIzC,OAAO;wBACL,QAAQ;4BACN,SAASI;4BACTL;4BACAD;4BACA,UAAUL,WAAW,QAAQ;wBAC/B;wBACA,OAAO;4BACL,KAAK;wBACP;wBACAzB;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMyB,aAAarB,QAAQ;YAG3B,MAAMd,QAAQmC,YAAY,WAAW,EAAE;YACvCZ,SAAS,IAAI,IAAKY,YAAY,YAAY,EAAE;YAE5C,IAAIc;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9CjD,OACAC,wBACAC,6BACA;oBACEkB;oBACA,SAAS;gBACX;YAEJ,EAAE,OAAOsB,OAAO;gBACd,OAAOrC,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEqC,MAAM,SAAS,EAAEJ,KAAK,SAAS,CAC5EtC,QACC;YAEP;YACA,IAAI,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,EACjDkC,QAAQ,IAAI,CACV,8FACA,IAAI,CAAC,mBAAmB,CAAC,sBAAsB;YAGnD,IAAI;gBACF,MAAM7B,QAAQ,YAAY,CAAC4C,YAAY,KAAK;YAC9C,EAAE,OAAOP,OAAY;gBACnBhB;gBACA,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,GAAG,CAAC,+BAA+B,EAAEgB,OAAO,WAAWQ,OAAOR,QAAQ;gBACrHlD,MACE,4DACAkD,iBAAiBS,QAAQT,MAAM,OAAO,GAAGQ,OAAOR;YAEpD;YAEA,IAAIhB,8BAA8BhC,uCAChC,OAAOW,QAAQ,eAAe,CAAC;YAIjC,IAAI,CAAC8B,YAAY,oCACf;YAIF,EAAEb;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAM4B,WAAW,CAAC,UAAU,EAAE5B,qBAAqB,4JAA4J,CAAC;gBAChN,OAAOnB,QAAQ,eAAe,CAAC+C;YACjC;YAEA,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,EAClD,IAAI,CAAC,mBAAmB,CAAC,sBAAsB,GAC7C;QAEN;QAEA,MAAMC,cAAc;YAClB,QAAQ;gBACN9B;YACF;YACAX;QACF;QACA,OAAOyC;IACT;IAEQ,oBACNC,IAAsE,EACtEC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,YAAYI,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAO/C,OAAOoD;gBACtB,MAAM,EAAErD,IAAI,EAAE,GAAGqD;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZxD,KAAK,GAAG,GAAG;wBACTwD;oBACF;gBACF;gBAGA,MAAMrD,YAAYkD,YAAY,SAAS;gBACvCjD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElB,MAAMsD,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACd,MAAMC,gBACJb,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIU,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGZ,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DjE,MAAM;oBACN,MAAM8E,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,AAAAA,IAAAA,0BAAAA,iBAAAA,AAAAA,EAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,aACAC,KACAY,sBACAX;gBAEJ,EAAE,OAAOhB,OAAO;oBACd,IAAIA,iBAAiB8B,kCAAAA,YAAYA,EAC/BV,UAAUpB,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE+B,IAAI,EAAE9B,KAAK,EAAE+B,OAAO,EAAEX,IAAI,EAAE,GAAGK;gBACvCN,UAAUC;gBAEV,IAAIY,eAAeF;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTE,eAAeF;qBACV,IAAInB,AAAS,cAATA,MAEPqB,eADEF,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTE,eAAe;qBACV;oBACLhE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8D,MAAM,CAACP,YAAY,KAAKzC,QACxB;oBAEFkD,eAAgBF,IAAY,CAACP,YAAY;gBAC3C;gBAGF,IAAIZ,AAAS,aAATA,QAAqB,CAACqB,cAAc;oBACtCpE,KAAK,KAAK,GAAGoC;oBACbpC,KAAK,OAAO,GAAGmE;oBACf,MAAM,IAAIvB,MAAM,CAAC,kBAAkB,EAAEuB,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQC;oBACR,KAAKd;oBACLlB;oBACA+B;gBACF;YACF;QACF;QAEA,OAAOf;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMrD,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EACEgD,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASjB,KAAK,SAAS,CAACiB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,aACAC,KACAC;QAGF,MAAM9C,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACsD;QAE1C,IAAI,CAAC7C,QACH,MAAM,IAAIqC,MACR;QAIJ,MAAM,EAAEpC,MAAM,EAAE2D,OAAO,EAAE,GAAG5D;QAE5B,OAAO;YACLC;YACA2D;YACA9D;QACF;IACF;IAEQ,UAAUgE,MAAc,EAA4C;QAC1E,OAAO;YACL,MAAM;YACN,OAAO;gBACLA;YACF;QACF;IACF;IAEA,MAAM,aAAaA,MAAc,EAAEC,YAA0B,EAAE;QAC7D,OAAO,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;YACtCD;QACF;IACF;IAEA,MAAM,QACJE,SAAsB,EACtBrB,GAA+B,EAC/BD,WAAyB,EACO;QAChC,MAAM,EAAEuB,UAAU,EAAErB,gBAAgB,EAAE,GAAGsB,AAAAA,IAAAA,kCAAAA,WAAAA,AAAAA,EAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM1E,UAAU,IAAI,CAAC,sBAAsB,CACzCC,AAAAA,IAAAA,qCAAAA,YAAAA,AAAAA,EAAa,WAAW2E;QAE1B,MAAMrE,SAASP,QAAQ,SAAS;QAChC,MAAM,EAAE6E,SAAS,EAAEC,eAAe,EAAE,GAAG1B;QAEvC9C,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOmE,WAAW;QAClBnE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuE,WAAW;QAClBvE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOwE,iBAAiB;QAExBxE,IAAAA,sBAAAA,MAAAA,AAAAA,EACEwE,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBxD,KAAK,GAAG;QACjC,IAAIyD,iBAAiBD;QACrB,IAAIE,eAAe;QAEnB,MAAOD,iBAAiBD,oBAAoBF,UAAW;YACrD,MAAMK,oBAAoB3D,KAAK,GAAG;YAClCyD,iBAAiBE;YACjB,MAAM5B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAoB,YACAvB,aACA/B,QACAiC;YAGF,MAAM5C,SAAU,MAAMT,QAAQ,YAAY,CAACsD;YAO3C,IAAI7C,QAAQ,QACV,OAAO;gBACL,QAAQW;gBACRb;YACF;YAGF0E,eACExE,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEiE,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMS,MAAM5D,KAAK,GAAG;YACpB,IAAI4D,MAAMD,oBAAoBJ,iBAAiB;gBAC7C,MAAMnC,gBAAgBmC,kBAAmBK,CAAAA,MAAMD,iBAAgB;gBAC/D,MAAME,YAAY,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;oBACjD,QAAQzC;gBACV;gBACA,MAAM3C,QAAQ,MAAM,CAACoF;YACvB;QACF;QAEA,OAAOpF,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEiF,cAAc;IACnE;IA3jBA,YACEI,iBAAoC,EACpCC,OAAgB,EAChBC,IAKC,CACD;QA9BF;QAEA;QAEA;QAEA,uBAAiB,eAAjB;QAEA,uBAAQ,uBAAR;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAiBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAG,IAAIC,yBAAAA,mBAAmBA;QAClD,IAAI,CAAC,WAAW,GAAG,IAAIC,yCAAAA,WAAWA,CAAC;YACjCJ;YACAC;YACA,WAAWC,KAAK,SAAS;QAC3B;IACF;AAsiBF"}

package/dist/lib/agent/ui-utils.js

@@ -64,10 +64,7 @@ function extractInsightParam(taskParam) {
     if (!taskParam) return {
         content: ''
     };
-    const extractImages = (source)=>{
-        var _source_multimodalPrompt;
-        return (null == source ? void 0 : null == (_source_multimodalPrompt = source.multimodalPrompt) ? void 0 : _source_multimodalPrompt.images) && Array.isArray(source.multimodalPrompt.images) ? source.multimodalPrompt.images : void 0;
-    };
+    const extractImages = (source)=>source?.multimodalPrompt?.images && Array.isArray(source.multimodalPrompt.images) ? source.multimodalPrompt.images : void 0;
     const toContent = (value)=>'string' == typeof value ? value : JSON.stringify(value);
     if (taskParam.demand) return {
         content: toContent(taskParam.demand),
@@ -97,26 +94,17 @@ function taskTitleStr(type, prompt) {
 }
 function paramStr(task) {
     let value;
-    if ('Planning' === task.type) if ('Locate' === task.subType) value = locateParamStr(null == task ? void 0 : task.param);
-    else {
-        var _task_param;
-        value = null == task ? void 0 : null == (_task_param = task.param) ? void 0 : _task_param.userInstruction;
-    }
-    if ('Insight' === task.type) value = extractInsightParam(null == task ? void 0 : task.param).content;
+    if ('Planning' === task.type) value = 'Locate' === task.subType ? locateParamStr(task?.param) : task?.param?.userInstruction;
+    if ('Insight' === task.type) value = extractInsightParam(task?.param).content;
     if ('Action Space' === task.type) {
-        var _task_param1, _task_param2, _task_param3, _task_param4, _task_param5;
-        const locate = null == task ? void 0 : null == (_task_param1 = task.param) ? void 0 : _task_param1.locate;
+        const locate = task?.param?.locate;
         const locateStr = locate ? locateParamStr(locate) : '';
         value = task.thought || '';
-        if ('number' == typeof (null == task ? void 0 : null == (_task_param2 = task.param) ? void 0 : _task_param2.timeMs)) {
-            var _task_param6;
-            value = `${null == task ? void 0 : null == (_task_param6 = task.param) ? void 0 : _task_param6.timeMs}ms`;
-        } else if ('string' == typeof (null == task ? void 0 : null == (_task_param3 = task.param) ? void 0 : _task_param3.scrollType)) value = scrollParamStr(null == task ? void 0 : task.param);
-        else if ('string' == typeof (null == task ? void 0 : null == (_task_param4 = task.param) ? void 0 : _task_param4.direction) && (null == task ? void 0 : task.subType) === 'AndroidPull') value = pullParamStr(null == task ? void 0 : task.param);
-        else if (void 0 !== (null == task ? void 0 : null == (_task_param5 = task.param) ? void 0 : _task_param5.value)) {
-            var _task_param7;
-            value = null == task ? void 0 : null == (_task_param7 = task.param) ? void 0 : _task_param7.value;
-        } else if ((null == task ? void 0 : task.param) && 'object' == typeof (null == task ? void 0 : task.param) && Object.keys((null == task ? void 0 : task.param) || {}).length > 0) value = null == task ? void 0 : task.param;
+        if ('number' == typeof task?.param?.timeMs) value = `${task?.param?.timeMs}ms`;
+        else if ('string' == typeof task?.param?.scrollType) value = scrollParamStr(task?.param);
+        else if ('string' == typeof task?.param?.direction && task?.subType === 'AndroidPull') value = pullParamStr(task?.param);
+        else if (void 0 !== task?.param?.value) value = task?.param?.value;
+        else if (task?.param && 'object' == typeof task?.param && Object.keys(task?.param || {}).length > 0) value = task?.param;
         if (locateStr) value = value && 'object' != typeof value ? `${locateStr} - ${value}` : locateStr;
     }
     if (void 0 === value) return '';

package/dist/lib/agent/ui-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    return JSON.stringify(value, undefined, 2);\n  }\n\n  return String(value);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","_source_multimodalPrompt","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","locateStr","_task_param6","_task_param7","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACMO,SAASI,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAE9B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC;YACdC;QAAP,OAAOA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,OAAQ,gBAAgB,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,AAAD,KACpCC,MAAM,OAAO,CAACF,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BG;IACN;IAGA,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIP,UAAU,MAAM,EAClB,OAAO;QACL,SAASM,UAAUN,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASM,UAAUN,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAES,UAAU,EAAE,GAAGT;QAEvB,IAAI,AAAsB,YAAtB,OAAOS,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQR,cAAcQ;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAEO,SAASC,aACdC,IAcY,EACZlB,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGkB,KAAK,GAAG,EAAElB,QAAQ;IAE9B,OAAOkB;AACT;AAEO,SAASC,SAAStB,IAAmB;IAC1C,IAAIiB;IACJ,IAAIjB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdiB,QAAQhB,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAsC,KAAK;SAC9D;YACGuB;QAARN,QAASjB,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,cAACvB,KAAgC,KAAK,AAAD,IAArCuB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAGF,IAAIvB,AAAc,cAAdA,KAAK,IAAI,EACXiB,QAAQR,oBAAqBT,QAAAA,OAAAA,KAAAA,IAAAA,KAAc,KAAK,EAAE,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;YACjBwB,cAIJC,cAGFC,cAIAC,cAKAC;QAhBT,MAAM1B,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,QAADwB,CAAAA,eAACxB,KAA8B,KAAK,AAAD,IAAnCwB,KAAAA,IAAAA,aAAsC,MAAM;QAC3D,MAAMK,YAAY3B,SAASD,eAAeC,UAAU;QAEpDe,QAAQjB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADyB,CAAAA,eAACzB,KAA8B,KAAK,AAAD,IAAnCyB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXb,QAAQ,GAAIjB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD8B,CAAAA,eAAC9B,KAA8B,KAAK,AAAD,IAAnC8B,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQ9B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDT,QAAQb,eAAgBJ,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAAD2B,CAAAA,eAAC3B,KAA8B,KAAK,AAAD,IAAnC2B,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAAC3B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CiB,QAAQX,aAAcN,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAAD4B,CAAAA,eAAC5B,KAA8B,KAAK,AAAD,IAAnC4B,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARd,QAASjB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD+B,CAAAA,eAAC/B,KAA8B,KAAK,AAAD,IAAnC+B,KAAAA,IAAAA,aAAsC,KAAK;QACrD,OAAO,IACL,AAAC/B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK,AAAD,KACnC,AAAgD,YAAhD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK,AAAD,KAC1CL,OAAO,IAAI,CAAC,AAACK,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK,AAAD,KAAK,CAAC,GAAG,MAAM,GAAG,GAIjEiB,QAASjB,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;QAG9C,IAAI6B,WAEAZ,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGY,UAAU,GAAG,EAAEZ,OAAO,GAEzBY;IAGd;IAEA,IAAI,AAAiB,WAAVZ,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMY,YAAY5B,eAAegB;QACjC,IAAIY,WACF,OAAOA;QAET,OAAOX,KAAK,SAAS,CAACD,OAAOF,QAAW;IAC1C;IAEA,OAAOiB,OAAOf;AAChB"}
+{"version":3,"file":"agent/ui-utils.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/ui-utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    return JSON.stringify(value, undefined, 2);\n  }\n\n  return String(value);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","locateStr","String"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;ACMO,SAASI,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAE9B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAEO,SAASC,aACdC,IAcY,EACZjB,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EAETgB,QADEhB,AAAiB,aAAjBA,KAAK,OAAO,GACNC,eAAgBD,MAAsC,SAErDA,MAAgC,OAAO;IAIpD,IAAIA,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMsB,YAAYpB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCL,OAAO,IAAI,CAAEK,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIsB,WAEAN,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGM,UAAU,GAAG,EAAEN,OAAO,GAEzBM;IAGd;IAEA,IAAI,AAAiB,WAAVN,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMM,YAAYrB,eAAee;QACjC,IAAIM,WACF,OAAOA;QAET,OAAOL,KAAK,SAAS,CAACD,OAAOF,QAAW;IAC1C;IAEA,OAAOS,OAAOP;AAChB"}

package/dist/lib/agent/utils.js

@@ -33,16 +33,16 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    commonContextParser: ()=>commonContextParser,
-    matchElementFromPlan: ()=>matchElementFromPlan,
+    getCurrentExecutionFile: ()=>getCurrentExecutionFile,
+    getMidsceneVersion: ()=>getMidsceneVersion,
     getReportFileName: ()=>getReportFileName,
     matchElementFromCache: ()=>matchElementFromCache,
-    printReportMsg: ()=>printReportMsg,
-    generateCacheId: ()=>generateCacheId,
-    getMidsceneVersion: ()=>getMidsceneVersion,
-    getCurrentExecutionFile: ()=>getCurrentExecutionFile,
+    matchElementFromPlan: ()=>matchElementFromPlan,
+    commonContextParser: ()=>commonContextParser,
     parsePrompt: ()=>parsePrompt,
-    ifPlanLocateParamIsBbox: ()=>ifPlanLocateParamIsBbox
+    generateCacheId: ()=>generateCacheId,
+    ifPlanLocateParamIsBbox: ()=>ifPlanLocateParamIsBbox,
+    printReportMsg: ()=>printReportMsg
 });
 const external_utils_js_namespaceObject = require("../utils.js");
 const env_namespaceObject = require("@midscene/shared/env");
@@ -54,10 +54,9 @@ var external_dayjs_default = /*#__PURE__*/ __webpack_require__.n(external_dayjs_
 const external_task_cache_js_namespaceObject = require("./task-cache.js");
 const debugProfile = (0, logger_namespaceObject.getDebug)('web:tool:profile');
 async function commonContextParser(interfaceInstance, _opt) {
-    var _interfaceInstance_describe;
     (0, utils_namespaceObject.assert)(interfaceInstance, 'interfaceInstance is required');
     debugProfile("Getting interface description");
-    const description = (null == (_interfaceInstance_describe = interfaceInstance.describe) ? void 0 : _interfaceInstance_describe.call(interfaceInstance)) || '';
+    const description = interfaceInstance.describe?.() || '';
     debugProfile("Interface description end");
     debugProfile('Uploading test info to server');
     (0, external_utils_js_namespaceObject.uploadTestInfoToServer)({
@@ -91,7 +90,7 @@ function getCurrentExecutionFile(trace) {
         const stackLines = stackTrace.split('\n');
         for (const line of stackLines)if (line.includes('.spec.') || line.includes('.test.') || line.includes('.ts') || line.includes('.js')) {
             const match = line.match(/(?:at\s+)?(.*?\.(?:spec|test)\.[jt]s)/);
-            if (null == match ? void 0 : match[1]) {
+            if (match?.[1]) {
                 const targetFileName = match[1].replace(pkgDir, '').trim().replace('at ', '');
                 return targetFileName;
             }
@@ -118,20 +117,18 @@ function ifPlanLocateParamIsBbox(planLocateParam) {
 function matchElementFromPlan(planLocateParam) {
     if (!planLocateParam) return;
     if (planLocateParam.bbox) {
-        var _planLocateParam_prompt;
         const centerPosition = {
             x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),
             y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2)
         };
-        const element = (0, extractor_namespaceObject.generateElementByPosition)(centerPosition, 'string' == typeof planLocateParam.prompt ? planLocateParam.prompt : (null == (_planLocateParam_prompt = planLocateParam.prompt) ? void 0 : _planLocateParam_prompt.prompt) || '');
+        const element = (0, extractor_namespaceObject.generateElementByPosition)(centerPosition, 'string' == typeof planLocateParam.prompt ? planLocateParam.prompt : planLocateParam.prompt?.prompt || '');
         return element;
     }
 }
 async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable) {
-    var _context_taskCache;
     if (!cacheEntry) return;
     if (false === cacheable) return void (0, external_task_cache_js_namespaceObject.debug)('cache disabled for prompt: %s', cachePrompt);
-    if (!(null == (_context_taskCache = context.taskCache) ? void 0 : _context_taskCache.isCacheResultUsed)) return;
+    if (!context.taskCache?.isCacheResultUsed) return;
     if (!context.interfaceInstance.rectMatchesCacheFeature) return void (0, external_task_cache_js_namespaceObject.debug)('interface does not implement rectMatchesCacheFeature, skip cache');
     try {
         const rect = await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);
@@ -150,7 +147,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.0.1-beta-20251202112442.0";
+const getMidsceneVersion = ()=>"1.0.1-beta-20251203073716.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/agent/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/agent/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst debugProfile = getDebug('web:tool:profile');\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: { uploadServerUrl?: string },\n): Promise<UIContext> {\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debugProfile('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debugProfile('Interface description end');\n\n  debugProfile('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debugProfile('UploadTestInfoToServer end');\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await interfaceInstance.size();\n  debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);\n\n  return {\n    size,\n    screenshotBase64: screenshotBase64!,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n\n    const element = generateElementByPosition(\n      centerPosition,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","debugProfile","getDebug","commonContextParser","interfaceInstance","_opt","assert","description","uploadTestInfoToServer","screenshotBase64","size","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","logMsg","getCurrentExecutionFile","trace","error","Error","stackTrace","pkgDir","process","stackLines","line","match","targetFileName","testFileIndex","Map","generateCacheId","fileName","taskFile","console","currentIndex","undefined","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","_planLocateParam_prompt","centerPosition","Math","element","generateElementByPosition","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","_context_taskCache","cacheDebug","rect","getMidsceneVersion","__VERSION__","parsePrompt","prompt"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACeA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEvB,eAAeC,oBACpBC,iBAAoC,EACpCC,IAAkC;QAKdD;IAHpBE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,mBAAmB;IAE1BH,aAAa;IACb,MAAMM,cAAcH,AAAAA,SAAAA,CAAAA,8BAAAA,kBAAkB,QAAQ,AAAD,IAAzBA,KAAAA,IAAAA,4BAAAA,IAAAA,CAAAA,kBAAiB,KAAiB;IACtDH,aAAa;IAEbA,aAAa;IACbO,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAJ,aAAa;IAEb,MAAMQ,mBAAmB,MAAML,kBAAkB,gBAAgB;IACjEE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,kBAAmB;IAE1B,MAAMC,OAAO,MAAMN,kBAAkB,IAAI;IACzCH,aAAa,CAAC,MAAM,EAAES,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,MAAM,EAAEA,KAAK,GAAG,EAAE;IAElE,OAAO;QACLA;QACA,kBAAkBD;IACpB;AACF;AAEO,SAASE,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAAA,mBAAAA,CAAAA,iBAAqC,CACzDC,oBAAAA,wBAAwBA;IAE1B,MAAMC,qBAAqBC,2BAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7CC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,gCAAgC,EAAED,UAAU;AACtD;AAMO,SAASE,wBAAwBC,KAAc;IACpD,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,aAAaH,SAASC,MAAM,KAAK;IACvC,MAAMG,SAASC,QAAQ,GAAG,MAAM;IAChC,IAAIF,YAAY;QACd,MAAMG,aAAaH,WAAW,KAAK,CAAC;QACpC,KAAK,MAAMI,QAAQD,WACjB,IACEC,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,UACdA,KAAK,QAAQ,CAAC,QACd;YACA,MAAMC,QAAQD,KAAK,KAAK,CAAC;YACzB,IAAIC,QAAAA,QAAAA,KAAAA,IAAAA,KAAO,CAAC,EAAE,EAAE;gBACd,MAAMC,iBAAiBD,KAAK,CAAC,EAAE,CAC5B,OAAO,CAACJ,QAAQ,IAChB,IAAI,GACJ,OAAO,CAAC,OAAO;gBAClB,OAAOK;YACT;QACF;IAEJ;IACA,OAAO;AACT;AAEA,MAAMC,gBAAgB,IAAIC;AAEnB,SAASC,gBAAgBC,QAAiB;IAC/C,IAAIC,WAAWD,YAAYd;IAC3B,IAAI,CAACe,UAAU;QACbA,WAAWnB,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA;QACXoB,QAAQ,IAAI,CACV;IAEJ;IAEA,IAAIL,cAAc,GAAG,CAACI,WAAW;QAC/B,MAAME,eAAeN,cAAc,GAAG,CAACI;QACvC,IAAIE,AAAiBC,WAAjBD,cACFN,cAAc,GAAG,CAACI,UAAUE,eAAe;IAE/C,OACEN,cAAc,GAAG,CAACI,UAAU;IAE9B,OAAO,GAAGA,SAAS,CAAC,EAAEJ,cAAc,GAAG,CAACI,WAAW;AACrD;AAEO,SAASI,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;YAUlBG;QATN,MAAMC,iBAAiB;YACrB,GAAGC,KAAK,KAAK,CAAEL,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;YACpE,GAAGK,KAAK,KAAK,CAAEL,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;QACtE;QAEA,MAAMM,UAAUC,AAAAA,IAAAA,0BAAAA,yBAAAA,AAAAA,EACdH,gBACA,AAAkC,YAAlC,OAAOJ,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBG,AAAAA,SAAAA,CAAAA,0BAAAA,gBAAgB,MAAM,AAAD,IAArBA,KAAAA,IAAAA,wBAAwB,MAAM,AAAD,KAAK;QAExC,OAAOG;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;QAWzBC;IATL,IAAI,CAACH,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBE,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,iCAAiCH;IAI9C,IAAI,UAACE,CAAAA,qBAAAA,QAAQ,SAAS,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,iBAAiB,AAAD,GACtC;IAGF,IAAI,CAACJ,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDK,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EACE;IAKJ,IAAI;QACF,MAAMC,OACJ,MAAMN,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACND,KAAK,KAAK,CAACU,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCV,KAAK,KAAK,CAACU,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOJ,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAG,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,yBAAyBH;QACpC,OAAOL;IACT,EAAE,OAAOxB,OAAO;QACdgC,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,qCAAqChC;QAChD;IACF;AACF;AAIO,MAAMkC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBrB;IACpB;IAEF,OAAO;QACL,YAAYqB,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACArB;IACN;AACF"}
+{"version":3,"file":"agent/utils.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst debugProfile = getDebug('web:tool:profile');\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: { uploadServerUrl?: string },\n): Promise<UIContext> {\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debugProfile('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debugProfile('Interface description end');\n\n  debugProfile('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debugProfile('UploadTestInfoToServer end');\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await interfaceInstance.size();\n  debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);\n\n  return {\n    size,\n    screenshotBase64: screenshotBase64!,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n\n    const element = generateElementByPosition(\n      centerPosition,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","debugProfile","getDebug","commonContextParser","interfaceInstance","_opt","assert","description","uploadTestInfoToServer","screenshotBase64","size","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","logMsg","getCurrentExecutionFile","trace","error","Error","stackTrace","pkgDir","process","stackLines","line","match","targetFileName","testFileIndex","Map","generateCacheId","fileName","taskFile","console","currentIndex","undefined","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","centerPosition","Math","element","generateElementByPosition","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","rect","getMidsceneVersion","__VERSION__","parsePrompt","prompt"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;ACeA,MAAMI,eAAeC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAEvB,eAAeC,oBACpBC,iBAAoC,EACpCC,IAAkC;IAElCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOF,mBAAmB;IAE1BH,aAAa;IACb,MAAMM,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDH,aAAa;IAEbA,aAAa;IACbO,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAJ,aAAa;IAEb,MAAMQ,mBAAmB,MAAML,kBAAkB,gBAAgB;IACjEE,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOG,kBAAmB;IAE1B,MAAMC,OAAO,MAAMN,kBAAkB,IAAI;IACzCH,aAAa,CAAC,MAAM,EAAES,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,MAAM,EAAEA,KAAK,GAAG,EAAE;IAElE,OAAO;QACLA;QACA,kBAAkBD;IACpB;AACF;AAEO,SAASE,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAAA,mBAAAA,CAAAA,iBAAqC,CACzDC,oBAAAA,wBAAwBA;IAE1B,MAAMC,qBAAqBC,2BAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7CC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,gCAAgC,EAAED,UAAU;AACtD;AAMO,SAASE,wBAAwBC,KAAc;IACpD,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,aAAaH,SAASC,MAAM,KAAK;IACvC,MAAMG,SAASC,QAAQ,GAAG,MAAM;IAChC,IAAIF,YAAY;QACd,MAAMG,aAAaH,WAAW,KAAK,CAAC;QACpC,KAAK,MAAMI,QAAQD,WACjB,IACEC,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,UACdA,KAAK,QAAQ,CAAC,QACd;YACA,MAAMC,QAAQD,KAAK,KAAK,CAAC;YACzB,IAAIC,OAAO,CAAC,EAAE,EAAE;gBACd,MAAMC,iBAAiBD,KAAK,CAAC,EAAE,CAC5B,OAAO,CAACJ,QAAQ,IAChB,IAAI,GACJ,OAAO,CAAC,OAAO;gBAClB,OAAOK;YACT;QACF;IAEJ;IACA,OAAO;AACT;AAEA,MAAMC,gBAAgB,IAAIC;AAEnB,SAASC,gBAAgBC,QAAiB;IAC/C,IAAIC,WAAWD,YAAYd;IAC3B,IAAI,CAACe,UAAU;QACbA,WAAWnB,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA;QACXoB,QAAQ,IAAI,CACV;IAEJ;IAEA,IAAIL,cAAc,GAAG,CAACI,WAAW;QAC/B,MAAME,eAAeN,cAAc,GAAG,CAACI;QACvC,IAAIE,AAAiBC,WAAjBD,cACFN,cAAc,GAAG,CAACI,UAAUE,eAAe;IAE/C,OACEN,cAAc,GAAG,CAACI,UAAU;IAE9B,OAAO,GAAGA,SAAS,CAAC,EAAEJ,cAAc,GAAG,CAACI,WAAW;AACrD;AAEO,SAASI,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;QACxB,MAAMG,iBAAiB;YACrB,GAAGC,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;YACpE,GAAGI,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;QACtE;QAEA,MAAMK,UAAUC,AAAAA,IAAAA,0BAAAA,yBAAAA,AAAAA,EACdH,gBACA,AAAkC,YAAlC,OAAOH,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QAExC,OAAOK;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EACE;IAKJ,IAAI;QACF,MAAMC,OACJ,MAAML,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACND,KAAK,KAAK,CAACS,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCT,KAAK,KAAK,CAACS,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOH,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOvB,OAAO;QACd8B,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,qCAAqC9B;QAChD;IACF;AACF;AAIO,MAAMgC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBnB;IACpB;IAEF,OAAO;QACL,YAAYmB,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAnB;IACN;AACF"}

package/dist/lib/ai-model/conversation-history.js

@@ -70,10 +70,9 @@ class ConversationHistory {
         return this.snapshot();
     }
     constructor(options){
-        var _options_initialMessages;
         _define_property(this, "messages", []);
         _define_property(this, "pendingFeedbackMessage", void 0);
-        if (null == options ? void 0 : null == (_options_initialMessages = options.initialMessages) ? void 0 : _options_initialMessages.length) this.seed(options.initialMessages);
+        if (options?.initialMessages?.length) this.seed(options.initialMessages);
         this.pendingFeedbackMessage = '';
     }
 }

package/dist/lib/ai-model/conversation-history.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/conversation-history.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/conversation-history.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly messages: ChatCompletionMessageParam[] = [];\n\n  public pendingFeedbackMessage: string;\n\n  constructor(options?: ConversationHistoryOptions) {\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n    this.pendingFeedbackMessage = '';\n  }\n\n  resetPendingFeedbackMessageIfExists() {\n    if (this.pendingFeedbackMessage) {\n      this.pendingFeedbackMessage = '';\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n  }\n\n  snapshot(): ChatCompletionMessageParam[] {\n    return [...this.messages];\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ConversationHistory","message","messages","options","_options_initialMessages"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;eCyCGG,OAAO,QAAQ;;AAzCX,MAAMC;IAYX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IAEA,WAAyC;QACvC,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACF,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IA1CA,YAAYI,OAAoC,CAAE;YAC5CC;QALN,uBAAiB,YAAyC,EAAE;QAE5D,uBAAO,0BAAP;QAGE,IAAIA,QAAAA,UAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,QAAS,eAAe,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAClC,IAAI,CAAC,IAAI,CAACD,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAsCF"}
+{"version":3,"file":"ai-model/conversation-history.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/ai-model/conversation-history.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly messages: ChatCompletionMessageParam[] = [];\n\n  public pendingFeedbackMessage: string;\n\n  constructor(options?: ConversationHistoryOptions) {\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n    this.pendingFeedbackMessage = '';\n  }\n\n  resetPendingFeedbackMessageIfExists() {\n    if (this.pendingFeedbackMessage) {\n      this.pendingFeedbackMessage = '';\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n  }\n\n  snapshot(): ChatCompletionMessageParam[] {\n    return [...this.messages];\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ConversationHistory","message","messages","options"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;eCyCGG,OAAO,QAAQ;;AAzCX,MAAMC;IAYX,sCAAsC;QACpC,IAAI,IAAI,CAAC,sBAAsB,EAC7B,IAAI,CAAC,sBAAsB,GAAG;IAElC;IAEA,OAAOC,OAAmC,EAAE;QAC1C,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IAEA,WAAyC;QACvC,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACF,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IA1CA,YAAYI,OAAoC,CAAE;QAJlD,uBAAiB,YAAyC,EAAE;QAE5D,uBAAO,0BAAP;QAGE,IAAIA,SAAS,iBAAiB,QAC5B,IAAI,CAAC,IAAI,CAACA,QAAQ,eAAe;QAEnC,IAAI,CAAC,sBAAsB,GAAG;IAChC;AAsCF"}

package/dist/lib/ai-model/index.js

@@ -24,35 +24,35 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    uiTarsPlanning: ()=>external_ui_tars_planning_js_namespaceObject.uiTarsPlanning,
-    plan: ()=>external_llm_planning_js_namespaceObject.plan,
-    SizeSchema: ()=>external_common_js_namespaceObject.SizeSchema,
-    TMultimodalPromptSchema: ()=>external_common_js_namespaceObject.TMultimodalPromptSchema,
+    callAIWithObjectResponse: ()=>index_js_namespaceObject.callAIWithObjectResponse,
     TUserPromptSchema: ()=>external_common_js_namespaceObject.TUserPromptSchema,
-    resizeImageForUiTars: ()=>external_ui_tars_planning_js_namespaceObject.resizeImageForUiTars,
-    AiLocateElement: ()=>external_inspect_js_namespaceObject.AiLocateElement,
-    generatePlaywrightTestStream: ()=>playwright_generator_js_namespaceObject.generatePlaywrightTestStream,
-    dumpActionParam: ()=>external_common_js_namespaceObject.dumpActionParam,
-    generateYamlTest: ()=>yaml_generator_js_namespaceObject.generateYamlTest,
-    systemPromptToLocateElement: ()=>llm_locator_js_namespaceObject.systemPromptToLocateElement,
     generatePlaywrightTest: ()=>playwright_generator_js_namespaceObject.generatePlaywrightTest,
-    AiLocateSection: ()=>external_inspect_js_namespaceObject.AiLocateSection,
-    AIActionType: ()=>external_common_js_namespaceObject.AIActionType,
-    callAI: ()=>index_js_namespaceObject.callAI,
-    ConversationHistory: ()=>external_conversation_history_js_namespaceObject.ConversationHistory,
-    findAllMidsceneLocatorField: ()=>external_common_js_namespaceObject.findAllMidsceneLocatorField,
+    describeUserPage: ()=>util_js_namespaceObject.describeUserPage,
+    plan: ()=>external_llm_planning_js_namespaceObject.plan,
+    parseActionParam: ()=>external_common_js_namespaceObject.parseActionParam,
     AiExtractElementInfo: ()=>external_inspect_js_namespaceObject.AiExtractElementInfo,
+    callAIWithStringResponse: ()=>index_js_namespaceObject.callAIWithStringResponse,
+    generateYamlTest: ()=>yaml_generator_js_namespaceObject.generateYamlTest,
+    SizeSchema: ()=>external_common_js_namespaceObject.SizeSchema,
+    AiJudgeOrderSensitive: ()=>external_inspect_js_namespaceObject.AiJudgeOrderSensitive,
+    getMidsceneLocationSchema: ()=>external_common_js_namespaceObject.getMidsceneLocationSchema,
+    ConversationHistory: ()=>external_conversation_history_js_namespaceObject.ConversationHistory,
     generateYamlTestStream: ()=>yaml_generator_js_namespaceObject.generateYamlTestStream,
     loadActionParam: ()=>external_common_js_namespaceObject.loadActionParam,
-    RectSchema: ()=>external_common_js_namespaceObject.RectSchema,
+    dumpActionParam: ()=>external_common_js_namespaceObject.dumpActionParam,
     PointSchema: ()=>external_common_js_namespaceObject.PointSchema,
+    AIActionType: ()=>external_common_js_namespaceObject.AIActionType,
+    RectSchema: ()=>external_common_js_namespaceObject.RectSchema,
+    resizeImageForUiTars: ()=>external_ui_tars_planning_js_namespaceObject.resizeImageForUiTars,
+    systemPromptToLocateElement: ()=>llm_locator_js_namespaceObject.systemPromptToLocateElement,
+    callAI: ()=>index_js_namespaceObject.callAI,
     adaptBboxToRect: ()=>external_common_js_namespaceObject.adaptBboxToRect,
-    AiJudgeOrderSensitive: ()=>external_inspect_js_namespaceObject.AiJudgeOrderSensitive,
-    callAIWithObjectResponse: ()=>index_js_namespaceObject.callAIWithObjectResponse,
-    parseActionParam: ()=>external_common_js_namespaceObject.parseActionParam,
-    getMidsceneLocationSchema: ()=>external_common_js_namespaceObject.getMidsceneLocationSchema,
-    callAIWithStringResponse: ()=>index_js_namespaceObject.callAIWithStringResponse,
-    describeUserPage: ()=>util_js_namespaceObject.describeUserPage
+    AiLocateElement: ()=>external_inspect_js_namespaceObject.AiLocateElement,
+    uiTarsPlanning: ()=>external_ui_tars_planning_js_namespaceObject.uiTarsPlanning,
+    findAllMidsceneLocatorField: ()=>external_common_js_namespaceObject.findAllMidsceneLocatorField,
+    generatePlaywrightTestStream: ()=>playwright_generator_js_namespaceObject.generatePlaywrightTestStream,
+    AiLocateSection: ()=>external_inspect_js_namespaceObject.AiLocateSection,
+    TMultimodalPromptSchema: ()=>external_common_js_namespaceObject.TMultimodalPromptSchema
 });
 const index_js_namespaceObject = require("./service-caller/index.js");
 const llm_locator_js_namespaceObject = require("./prompt/llm-locator.js");

package/dist/lib/ai-model/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/index.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}
+{"version":3,"file":"ai-model/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/ai-model/inspect.js

@@ -24,10 +24,10 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    AiExtractElementInfo: ()=>AiExtractElementInfo,
+    AiLocateElement: ()=>AiLocateElement,
     AiJudgeOrderSensitive: ()=>AiJudgeOrderSensitive,
-    AiLocateSection: ()=>AiLocateSection,
-    AiLocateElement: ()=>AiLocateElement
+    AiExtractElementInfo: ()=>AiExtractElementInfo,
+    AiLocateSection: ()=>AiLocateSection
 });
 const dom_util_namespaceObject = require("@midscene/shared/extractor/dom-util");
 const img_namespaceObject = require("@midscene/shared/img");
@@ -46,9 +46,8 @@ const extraTextFromUserPrompt = (prompt)=>{
     return prompt.prompt;
 };
 const promptsToChatParam = async (multimodalPrompt)=>{
-    var _multimodalPrompt_images;
     const msgs = [];
-    if (null == multimodalPrompt ? void 0 : null == (_multimodalPrompt_images = multimodalPrompt.images) ? void 0 : _multimodalPrompt_images.length) {
+    if (multimodalPrompt?.images?.length) {
         msgs.push({
             role: 'user',
             content: [
@@ -99,12 +98,11 @@ async function AiLocateElement(options) {
     let originalImageWidth = imageWidth;
     let originalImageHeight = imageHeight;
     if (options.searchConfig) {
-        var _options_searchConfig_rect, _options_searchConfig_rect1;
         (0, utils_namespaceObject.assert)(options.searchConfig.rect, 'searchArea is provided but its rect cannot be found. Failed to locate element');
         (0, utils_namespaceObject.assert)(options.searchConfig.imageBase64, 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element');
         imagePayload = options.searchConfig.imageBase64;
-        imageWidth = null == (_options_searchConfig_rect = options.searchConfig.rect) ? void 0 : _options_searchConfig_rect.width;
-        imageHeight = null == (_options_searchConfig_rect1 = options.searchConfig.rect) ? void 0 : _options_searchConfig_rect1.height;
+        imageWidth = options.searchConfig.rect?.width;
+        imageHeight = options.searchConfig.rect?.height;
         originalImageWidth = imageWidth;
         originalImageHeight = imageHeight;
     } else if ('qwen2.5-vl' === vlMode) {
@@ -149,8 +147,7 @@ async function AiLocateElement(options) {
     let errors = 'errors' in res.content ? res.content.errors : [];
     try {
         if ('bbox' in res.content && Array.isArray(res.content.bbox) && res.content.bbox.length >= 1) {
-            var _options_searchConfig_rect2, _options_searchConfig, _options_searchConfig_rect3, _options_searchConfig1;
-            resRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(res.content.bbox, imageWidth, imageHeight, null == (_options_searchConfig = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect2 = _options_searchConfig.rect) ? void 0 : _options_searchConfig_rect2.left, null == (_options_searchConfig1 = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect3 = _options_searchConfig1.rect) ? void 0 : _options_searchConfig_rect3.top, originalImageWidth, originalImageHeight, vlMode);
+            resRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(res.content.bbox, imageWidth, imageHeight, options.searchConfig?.rect?.left, options.searchConfig?.rect?.top, originalImageWidth, originalImageHeight, vlMode);
             debugInspect('resRect', resRect);
             const rectCenter = {
                 x: resRect.left + resRect.width / 2,
@@ -164,7 +161,7 @@ async function AiLocateElement(options) {
         }
     } catch (e) {
         const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
-        if (errors && (null == errors ? void 0 : errors.length) !== 0) errors.push(`(${msg})`);
+        if (errors && errors?.length !== 0) errors.push(`(${msg})`);
         else errors = [
             msg
         ];
@@ -253,7 +250,7 @@ async function AiExtractElementInfo(options) {
     const { screenshotBase64 } = context;
     const extractDataPromptText = (0, extraction_js_namespaceObject.extractDataQueryPrompt)(options.pageDescription || '', dataQuery);
     const userContent = [];
-    if ((null == extractOption ? void 0 : extractOption.screenshotIncluded) !== false) userContent.push({
+    if (extractOption?.screenshotIncluded !== false) userContent.push({
         type: 'image_url',
         image_url: {
             url: screenshotBase64,