@midscene/core 1.0.1-beta-20251110130223.0 → 1.0.1-beta-20251111100700.0

package/dist/es/agent/task-builder.mjs

@@ -22,7 +22,6 @@ function locatePlanForLocate(param) {
     } : param;
     const locatePlan = {
         type: 'Locate',
-        locate,
         param: locate,
         thought: ''
     };
@@ -67,7 +66,6 @@ class TaskBuilder {
             subType: 'Finished',
             param: null,
             thought: plan.thought,
-            locate: plan.locate,
             subTask: context.subTask || void 0,
             executor: async ()=>{}
         };
@@ -75,8 +73,7 @@ class TaskBuilder {
     }
     handleSleepPlan(plan, context) {
         const sleepTask = this.createSleepTask(plan.param, {
-            thought: plan.thought,
-            locate: plan.locate
+            thought: plan.thought
         });
         if (context.subTask) sleepTask.subTask = true;
         context.tasks.push(sleepTask);
@@ -87,15 +84,13 @@ class TaskBuilder {
             subType: 'Sleep',
             param,
             thought: null == meta ? void 0 : meta.thought,
-            locate: (null == meta ? void 0 : meta.locate) ?? null,
             executor: async (taskParam)=>{
                 await sleep((null == taskParam ? void 0 : taskParam.timeMs) || 3000);
             }
         };
     }
     async handleLocatePlan(plan, context) {
-        if (!plan.locate || null === plan.locate) return void debug('Locate action with id is null, will be ignored', plan);
-        const taskLocate = this.createLocateTask(plan, plan.locate, context);
+        const taskLocate = this.createLocateTask(plan, plan.param, context);
         context.tasks.push(taskLocate);
     }
     async handleActionPlan(plan, context) {
@@ -209,7 +204,7 @@ class TaskBuilder {
             param: locateParam,
             thought: plan.thought,
             executor: async (param, taskContext)=>{
-                var _this_taskCache, _locateCacheRecord_cacheContent;
+                var _param_prompt, _this_taskCache, _locateCacheRecord_cacheContent;
                 const { task } = taskContext;
                 let { uiContext } = taskContext;
                 assert((null == param ? void 0 : param.prompt) || (null == param ? void 0 : param.bbox), `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
@@ -236,7 +231,7 @@ class TaskBuilder {
                 const elementFromXpath = rectFromXpath ? generateElementByPosition({
                     x: rectFromXpath.left + rectFromXpath.width / 2,
                     y: rectFromXpath.top + rectFromXpath.height / 2
-                }) : void 0;
+                }, 'string' == typeof param.prompt ? param.prompt : (null == (_param_prompt = param.prompt) ? void 0 : _param_prompt.prompt) || '') : void 0;
                 const userExpectedPathHitFlag = !!elementFromXpath;
                 const cachePrompt = param.prompt;
                 const locateCacheRecord = null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchLocateCache(cachePrompt);
@@ -261,9 +256,9 @@ class TaskBuilder {
                 const element = elementFromXpath || elementFromCache || elementFromAiLocate;
                 let currentCacheEntry;
                 if (element && this.taskCache && !cacheHitFlag && (null == param ? void 0 : param.cacheable) !== false) if (this.interface.cacheFeatureForRect) try {
-                    var _param_prompt;
+                    var _param_prompt1;
                     const feature = await this.interface.cacheFeatureForRect(element.rect, {
-                        targetDescription: 'string' == typeof param.prompt ? param.prompt : null == (_param_prompt = param.prompt) ? void 0 : _param_prompt.prompt,
+                        targetDescription: 'string' == typeof param.prompt ? param.prompt : null == (_param_prompt1 = param.prompt) ? void 0 : _param_prompt1.prompt,
                         modelConfig
                     });
                     if (feature && Object.keys(feature).length > 0) {

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["webpack://@midscene/core/./src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type {\n  DetailedLocateParam,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    locate,\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  subTask?: boolean;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfig: IModelConfig;\n  cacheable?: boolean;\n  subTask: boolean;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  constructor({ interfaceInstance, service, taskCache }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfig,\n      cacheable,\n      subTask: !!options?.subTask,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n      [\n        'Sleep',\n        (plan) =>\n          this.handleSleepPlan(\n            plan as PlanningAction<PlanningActionParamSleep>,\n            context,\n          ),\n      ],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      locate: plan.locate,\n      subTask: context.subTask || undefined,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private handleSleepPlan(\n    plan: PlanningAction<PlanningActionParamSleep>,\n    context: PlanBuildContext,\n  ): void {\n    const sleepTask = this.createSleepTask(plan.param, {\n      thought: plan.thought,\n      locate: plan.locate,\n    });\n    if (context.subTask) {\n      sleepTask.subTask = true;\n    }\n    context.tasks.push(sleepTask);\n  }\n\n  public createSleepTask(\n    param: PlanningActionParamSleep,\n    meta?: { thought?: string; locate?: PlanningAction['locate'] | null },\n  ): ExecutionTaskActionApply<PlanningActionParamSleep> {\n    return {\n      type: 'Action Space',\n      subType: 'Sleep',\n      param,\n      thought: meta?.thought,\n      locate: meta?.locate ?? null,\n      executor: async (taskParam) => {\n        await sleep(taskParam?.timeMs || 3000);\n      },\n    };\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    if (!plan.locate || plan.locate === null) {\n      debug('Locate action with id is null, will be ignored', plan);\n      return;\n    }\n\n    const taskLocate = this.createLocateTask(plan, plan.locate, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = await this.interface.actionSpace();\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        if (ifPlanLocateParamIsBbox(param[field])) {\n          debug(\n            'plan locate param is bbox, will match element from plan',\n            param[field],\n          );\n          const elementFromPlan = matchElementFromPlan(param[field]);\n          if (elementFromPlan) {\n            param[field] = elementFromPlan;\n          }\n        } else {\n          const locatePlan = locatePlanForLocate(param[field]);\n          debug(\n            'will prepend locate param for field',\n            `action.type=${planType}`,\n            `param=${JSON.stringify(param[field])}`,\n            `locatePlan=${JSON.stringify(locatePlan)}`,\n          );\n          const locateTask = this.createLocateTask(\n            locatePlan,\n            param[field],\n            context,\n            (result) => {\n              param[field] = result;\n            },\n          );\n          context.tasks.push(locateTask);\n        }\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      subTask: context.subTask || undefined,\n      executor: async (param, taskContext) => {\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug('will call \"beforeInvokeAction\" for interface');\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug('called \"beforeInvokeAction\" for interface');\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema);\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        debug('called action', action.name, 'result:', actionResult);\n\n        const delayAfterRunner = action.delayAfterRunner ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug('will call \"afterInvokeAction\" for interface');\n            await this.interface.afterInvokeAction(action.name, param);\n            debug('called \"afterInvokeAction\" for interface');\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        return {\n          output: {\n            success: true,\n            action: planType,\n            param: param,\n            result: actionResult,\n          },\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfig } = context;\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    const taskFind: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      subTask: context.subTask || undefined,\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n        };\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (param.xpath && this.interface.rectMatchesCacheFeature) {\n          rectFromXpath = await this.interface.rectMatchesCacheFeature({\n            xpaths: [param.xpath],\n          });\n        }\n        const elementFromXpath = rectFromXpath\n          ? generateElementByPosition({\n              x: rectFromXpath.left + rectFromXpath.width / 2,\n              y: rectFromXpath.top + rectFromXpath.height / 2,\n            })\n          : undefined;\n        const userExpectedPathHitFlag = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCache = userExpectedPathHitFlag\n          ? null\n          : await matchElementFromCache(\n              {\n                taskCache: this.taskCache,\n                interfaceInstance: this.interface,\n              },\n              cacheEntry,\n              cachePrompt,\n              param.cacheable,\n            );\n        const cacheHitFlag = !!elementFromCache;\n\n        const planHitFlag = false;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) {\n          try {\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n              },\n              modelConfig,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          }\n        }\n\n        const element =\n          elementFromXpath || elementFromCache || elementFromAiLocate;\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        if (\n          element &&\n          this.taskCache &&\n          !cacheHitFlag &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForRect) {\n            try {\n              const feature = await this.interface.cacheFeatureForRect(\n                element.rect,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig,\n                },\n              );\n              if (feature && Object.keys(feature).length > 0) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForRect failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForRect is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (userExpectedPathHitFlag) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (cacheHitFlag) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element,\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskFind;\n  }\n}\n"],"names":["debug","getDebug","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfig","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","undefined","sleepTask","meta","taskParam","sleep","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamIsBbox","elementFromPlan","matchElementFromPlan","JSON","locateTask","result","assert","task","taskContext","_taskContext_element","uiContext","Promise","originalError","originalMessage","String","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","taskFind","_this_taskCache","_locateCacheRecord_cacheContent","locateDump","locateResult","applyDump","dump","_dump_taskInfo","_dump_taskInfo1","rectFromXpath","elementFromXpath","generateElementByPosition","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","cacheEntry","elementFromCache","matchElementFromCache","cacheHitFlag","planHitFlag","elementFromAiLocate","ServiceError","element","currentCacheEntry","_param_prompt","feature","Object","hitBy","interfaceInstance","service","taskCache"],"mappings":";;;;;;;;;;;;;;;;;AA+BA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACND;QACA,OAAOA;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAoBO,MAAMC;IAaX,MAAa,MACXC,KAAuB,EACvBC,WAAyB,EACzBC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,SAAS;QAEpC,MAAMG,UAA4B;YAChCF;YACAF;YACAG;YACA,SAAS,CAAC,CAACF,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;QAC5B;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;YAC9D;gBACE;gBACA,CAACG,OACC,IAAI,CAAC,eAAe,CAClBA,MACAH;aAEL;SACF;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQR,MAAO;YACxB,MAAMU,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,QAAQA,KAAK,MAAM;YACnB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,WAAa;QACzB;QACAP,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEQ,gBACNH,IAA8C,EAC9CH,OAAyB,EACnB;QACN,MAAMQ,YAAY,IAAI,CAAC,eAAe,CAACL,KAAK,KAAK,EAAE;YACjD,SAASA,KAAK,OAAO;YACrB,QAAQA,KAAK,MAAM;QACrB;QACA,IAAIH,QAAQ,OAAO,EACjBQ,UAAU,OAAO,GAAG;QAEtBR,QAAQ,KAAK,CAAC,IAAI,CAACQ;IACrB;IAEO,gBACLjB,KAA+B,EAC/BkB,IAAqE,EACjB;QACpD,OAAO;YACL,MAAM;YACN,SAAS;YACTlB;YACA,SAASkB,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO;YACtB,QAAQA,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,MAAM,AAAD,KAAK;YACxB,UAAU,OAAOC;gBACf,MAAMC,MAAMD,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;YACnC;QACF;IACF;IAEA,MAAc,iBACZP,IAAyC,EACzCH,OAAyB,EACV;QACf,IAAI,CAACG,KAAK,MAAM,IAAIA,AAAgB,SAAhBA,KAAK,MAAM,EAAW,YACxCf,MAAM,kDAAkDe;QAI1D,MAAMS,aAAa,IAAI,CAAC,gBAAgB,CAACT,MAAMA,KAAK,MAAM,EAAEH;QAC5DA,QAAQ,KAAK,CAAC,IAAI,CAACY;IACrB;IAEA,MAAc,iBACZT,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMa,WAAWV,KAAK,IAAI;QAC1B,MAAMW,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;QACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMtB,QAAQY,KAAK,KAAK;QAExB,IAAI,CAACY,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EACd,IAAIC,wBAAwB/B,KAAK,CAAC8B,MAAM,GAAG;gBACzCjC,MACE,2DACAG,KAAK,CAAC8B,MAAM;gBAEd,MAAME,kBAAkBC,qBAAqBjC,KAAK,CAAC8B,MAAM;gBACzD,IAAIE,iBACFhC,KAAK,CAAC8B,MAAM,GAAGE;YAEnB,OAAO;gBACL,MAAM9B,aAAaH,oBAAoBC,KAAK,CAAC8B,MAAM;gBACnDjC,MACE,uCACA,CAAC,YAAY,EAAEyB,UAAU,EACzB,CAAC,MAAM,EAAEY,KAAK,SAAS,CAAClC,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEI,KAAK,SAAS,CAAChC,aAAa;gBAE5C,MAAMiC,aAAa,IAAI,CAAC,gBAAgB,CACtCjC,YACAF,KAAK,CAAC8B,MAAM,EACZrB,SACA,CAAC2B;oBACCpC,KAAK,CAAC8B,MAAM,GAAGM;gBACjB;gBAEF3B,QAAQ,KAAK,CAAC,IAAI,CAAC0B;YACrB;iBACK;gBACLE,OACE,CAACR,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3EzB,MAAM,CAAC,OAAO,EAAEiC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMgB,OAKF;YACF,MAAM;YACN,SAAShB;YACT,SAASV,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,OAAOhB,OAAOuC;oBAKWC;gBAJjC3C,MACE,oBACAyB,UACAtB,OACA,CAAC,4BAA4B,EAAE,QAAAwC,CAAAA,uBAAAA,YAAY,OAAO,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,MAAM,EAAE;gBAG9D,MAAMC,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBZ,qBAAqB,OAAO,CAAC,CAACC;oBAC5BO,OACErC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEA,IAAI;oBACF,MAAMoB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC7C,MAAM;gCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC2B,OAAO,IAAI,EAAExB;gCACrDH,MAAM;4BACR;wBACF;wBACAuB,MAAM;qBACP;gBACH,EAAE,OAAOuB,eAAoB;oBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,IAAInB,OAAO,WAAW,EACpB,IAAI;oBACFxB,QAAQ8C,iBAAiB9C,OAAOwB,OAAO,WAAW;gBACpD,EAAE,OAAOuB,OAAY;oBACnB,MAAM,IAAIrB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEuB,MAAM,OAAO,CAAC,cAAc,EAAEb,KAAK,SAAS,CAAClC,QAAQ,EACtG;wBAAE,OAAO+C;oBAAM;gBAEnB;gBAGFlD,MAAM,kBAAkB2B,OAAO,IAAI;gBACnC,MAAMwB,WAAWxB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAMyB,eAAe,MAAMD,SAAShD,OAAOuC;gBAC3C1C,MAAM,iBAAiB2B,OAAO,IAAI,EAAE,WAAWyB;gBAE/C,MAAMC,mBAAmB1B,OAAO,gBAAgB,IAAI;gBACpD,IAAI0B,mBAAmB,GACrB,MAAM9B,MAAM8B;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCrD,MAAM;wBACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC2B,OAAO,IAAI,EAAExB;wBACpDH,MAAM;oBACR;gBACF,EAAE,OAAO8C,eAAoB;oBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,OAAO;oBACL,QAAQ;wBACN,SAAS;wBACT,QAAQrB;wBACR,OAAOtB;wBACP,QAAQiD;oBACV;gBACF;YACF;QACF;QAEAxC,QAAQ,KAAK,CAAC,IAAI,CAAC6B;IACrB;IAEQ,iBACN1B,IAAyC,EACzCuC,mBAAiD,EACjD1C,OAAyB,EACzB2C,QAAgD,EACd;QAClC,MAAM,EAAE5C,SAAS,EAAEH,WAAW,EAAE,GAAGI;QACnC,IAAI4C,cAAcF;QAElB,IAAI,AAAuB,YAAvB,OAAOE,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI7C,AAAcQ,WAAdR,WACF6C,cAAc;YACZ,GAAGA,WAAW;YACd7C;QACF;QAGF,MAAM8C,WAA6C;YACjD,MAAM;YACN,SAAS;YACT,SAAS7C,QAAQ,OAAO,IAAIO;YAC5B,OAAOqC;YACP,SAASzC,KAAK,OAAO;YACrB,UAAU,OAAOZ,OAAOuC;oBAkDIgB,iBACPC;gBAlDnB,MAAM,EAAElB,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACErC,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GAC3B,CAAC,qDAAqD,EAAEkC,KAAK,SAAS,CACpElC,QACC;gBAGL,IAAI,CAACyC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,IAAIgB;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;wBAQJC,gBACTC;oBARJ,IAAI,CAACF,MACH;oBAEFH,aAAaG;oBACbtB,KAAK,GAAG,GAAG;wBACTsB;oBACF;oBACAtB,KAAK,KAAK,GAAG,QAAAuB,CAAAA,iBAAAA,KAAK,QAAQ,AAAD,IAAZA,KAAAA,IAAAA,eAAe,KAAK;oBACjC,IAAI,QAAAC,CAAAA,kBAAAA,KAAK,QAAQ,AAAD,IAAZA,KAAAA,IAAAA,gBAAe,eAAe,EAChCxB,KAAK,eAAe,GAAGsB,KAAK,QAAQ,CAAC,eAAe;gBAExD;gBAGA,IAAIG;gBACJ,IAAI/D,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,uBAAuB,EACvD+D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;oBAC3D,QAAQ;wBAAC/D,MAAM,KAAK;qBAAC;gBACvB;gBAEF,MAAMgE,mBAAmBD,gBACrBE,0BAA0B;oBACxB,GAAGF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;oBAC9C,GAAGA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;gBAChD,KACA/C;gBACJ,MAAMkD,0BAA0B,CAAC,CAACF;gBAElC,MAAMG,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBAAoB,QAAAb,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CAACY;gBAC3D,MAAME,aAAab,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,KAAK;gBAEzD,MAAMc,mBAAmBJ,0BACrB,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAErB,MAAMwE,eAAe,CAAC,CAACF;gBAEvB,MAAMG,cAAc;gBAEpB,IAAIC;gBACJ,IAAI,CAACR,2BAA2B,CAACM,gBAAgB,CAACC,aAChD,IAAI;oBACFf,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC1D,OACA;wBACE,SAASyC;oBACX,GACApC;oBAEFsD,UAAUD,aAAa,IAAI;oBAC3BgB,sBAAsBhB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiB4B,cACnBhB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAGF,MAAM6B,UACJZ,oBAAoBM,oBAAoBI;gBAE1C,IAAIG;gBACJ,IACED,WACA,IAAI,CAAC,SAAS,IACd,CAACJ,gBACDxE,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EACpC,IAAI;wBAOQ8E;oBANV,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CACtDH,QAAQ,IAAI,EACZ;wBACE,mBACE,AAAwB,YAAxB,OAAO5E,MAAM,MAAM,GACfA,MAAM,MAAM,WACZ8E,CAAAA,gBAAAA,MAAM,MAAM,AAAD,IAAXA,KAAAA,IAAAA,cAAc,MAAM;wBAC1BzE;oBACF;oBAEF,IAAI0E,WAAWC,OAAO,IAAI,CAACD,SAAS,MAAM,GAAG,GAAG;wBAC9ClF,MACE,uCACAsE,aACAY;wBAEFF,oBAAoBE;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQZ;4BACR,OAAOY;wBACT,GACAX;oBAEJ,OACEvE,MACE,yDACAsE;gBAGN,EAAE,OAAOpB,OAAO;oBACdlD,MAAM,kCAAkCkD;gBAC1C;qBAEAlD,MAAM;gBAIV,IAAI,CAAC+E,SAAS;oBACZ,IAAInB,YACF,MAAM,IAAIkB,aACR,CAAC,oBAAoB,EAAE3E,MAAM,MAAM,EAAE,EACrCyD;oBAGJ,MAAM,IAAI/B,MAAM,CAAC,mBAAmB,EAAE1B,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIiF;gBAEJ,IAAIf,yBACFe,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOjF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIwE,cACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPZ;wBACA,aAAaQ;oBACf;gBACF;gBAGFzB,QAAAA,YAAAA,SAAWwB;gBAEX,OAAO;oBACL,QAAQ;wBACNA;oBACF;oBACAK;gBACF;YACF;QACF;QAEA,OAAO3B;IACT;IAjeA,YAAY,EAAE4B,iBAAiB,EAAEC,OAAO,EAAEC,SAAS,EAAmB,CAAE;QANxE,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAGE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;IACnB;AA8dF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["webpack://@midscene/core/./src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type {\n  DetailedLocateParam,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  subTask?: boolean;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfig: IModelConfig;\n  cacheable?: boolean;\n  subTask: boolean;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  constructor({ interfaceInstance, service, taskCache }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfig,\n      cacheable,\n      subTask: !!options?.subTask,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n      [\n        'Sleep',\n        (plan) =>\n          this.handleSleepPlan(\n            plan as PlanningAction<PlanningActionParamSleep>,\n            context,\n          ),\n      ],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      subTask: context.subTask || undefined,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private handleSleepPlan(\n    plan: PlanningAction<PlanningActionParamSleep>,\n    context: PlanBuildContext,\n  ): void {\n    const sleepTask = this.createSleepTask(plan.param, {\n      thought: plan.thought,\n    });\n    if (context.subTask) {\n      sleepTask.subTask = true;\n    }\n    context.tasks.push(sleepTask);\n  }\n\n  public createSleepTask(\n    param: PlanningActionParamSleep,\n    meta?: { thought?: string },\n  ): ExecutionTaskActionApply<PlanningActionParamSleep> {\n    return {\n      type: 'Action Space',\n      subType: 'Sleep',\n      param,\n      thought: meta?.thought,\n      executor: async (taskParam) => {\n        await sleep(taskParam?.timeMs || 3000);\n      },\n    };\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = await this.interface.actionSpace();\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        if (ifPlanLocateParamIsBbox(param[field])) {\n          debug(\n            'plan locate param is bbox, will match element from plan',\n            param[field],\n          );\n          const elementFromPlan = matchElementFromPlan(param[field]);\n          if (elementFromPlan) {\n            param[field] = elementFromPlan;\n          }\n        } else {\n          const locatePlan = locatePlanForLocate(param[field]);\n          debug(\n            'will prepend locate param for field',\n            `action.type=${planType}`,\n            `param=${JSON.stringify(param[field])}`,\n            `locatePlan=${JSON.stringify(locatePlan)}`,\n          );\n          const locateTask = this.createLocateTask(\n            locatePlan,\n            param[field],\n            context,\n            (result) => {\n              param[field] = result;\n            },\n          );\n          context.tasks.push(locateTask);\n        }\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      subTask: context.subTask || undefined,\n      executor: async (param, taskContext) => {\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug('will call \"beforeInvokeAction\" for interface');\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug('called \"beforeInvokeAction\" for interface');\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema);\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        debug('called action', action.name, 'result:', actionResult);\n\n        const delayAfterRunner = action.delayAfterRunner ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug('will call \"afterInvokeAction\" for interface');\n            await this.interface.afterInvokeAction(action.name, param);\n            debug('called \"afterInvokeAction\" for interface');\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        return {\n          output: {\n            success: true,\n            action: planType,\n            param: param,\n            result: actionResult,\n          },\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfig } = context;\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    const taskFind: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      subTask: context.subTask || undefined,\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n        };\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (param.xpath && this.interface.rectMatchesCacheFeature) {\n          rectFromXpath = await this.interface.rectMatchesCacheFeature({\n            xpaths: [param.xpath],\n          });\n        }\n        const elementFromXpath = rectFromXpath\n          ? generateElementByPosition(\n              {\n                x: rectFromXpath.left + rectFromXpath.width / 2,\n                y: rectFromXpath.top + rectFromXpath.height / 2,\n              },\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n        const userExpectedPathHitFlag = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCache = userExpectedPathHitFlag\n          ? null\n          : await matchElementFromCache(\n              {\n                taskCache: this.taskCache,\n                interfaceInstance: this.interface,\n              },\n              cacheEntry,\n              cachePrompt,\n              param.cacheable,\n            );\n        const cacheHitFlag = !!elementFromCache;\n\n        const planHitFlag = false;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) {\n          try {\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n              },\n              modelConfig,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          }\n        }\n\n        const element =\n          elementFromXpath || elementFromCache || elementFromAiLocate;\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        if (\n          element &&\n          this.taskCache &&\n          !cacheHitFlag &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForRect) {\n            try {\n              const feature = await this.interface.cacheFeatureForRect(\n                element.rect,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig,\n                },\n              );\n              if (feature && Object.keys(feature).length > 0) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForRect failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForRect is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (userExpectedPathHitFlag) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (cacheHitFlag) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element,\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskFind;\n  }\n}\n"],"names":["debug","getDebug","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfig","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","undefined","sleepTask","meta","taskParam","sleep","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamIsBbox","elementFromPlan","matchElementFromPlan","JSON","locateTask","result","assert","task","taskContext","_taskContext_element","uiContext","Promise","originalError","originalMessage","String","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","taskFind","_param_prompt","_this_taskCache","_locateCacheRecord_cacheContent","locateDump","locateResult","applyDump","dump","_dump_taskInfo","_dump_taskInfo1","rectFromXpath","elementFromXpath","generateElementByPosition","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","cacheEntry","elementFromCache","matchElementFromCache","cacheHitFlag","planHitFlag","elementFromAiLocate","ServiceError","element","currentCacheEntry","_param_prompt1","feature","Object","hitBy","interfaceInstance","service","taskCache"],"mappings":";;;;;;;;;;;;;;;;;AA+BA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAoBO,MAAMC;IAaX,MAAa,MACXC,KAAuB,EACvBC,WAAyB,EACzBC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,SAAS;QAEpC,MAAMG,UAA4B;YAChCF;YACAF;YACAG;YACA,SAAS,CAAC,CAACF,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;QAC5B;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;YAC9D;gBACE;gBACA,CAACG,OACC,IAAI,CAAC,eAAe,CAClBA,MACAH;aAEL;SACF;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQR,MAAO;YACxB,MAAMU,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,WAAa;QACzB;QACAP,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEQ,gBACNH,IAA8C,EAC9CH,OAAyB,EACnB;QACN,MAAMQ,YAAY,IAAI,CAAC,eAAe,CAACL,KAAK,KAAK,EAAE;YACjD,SAASA,KAAK,OAAO;QACvB;QACA,IAAIH,QAAQ,OAAO,EACjBQ,UAAU,OAAO,GAAG;QAEtBR,QAAQ,KAAK,CAAC,IAAI,CAACQ;IACrB;IAEO,gBACLjB,KAA+B,EAC/BkB,IAA2B,EACyB;QACpD,OAAO;YACL,MAAM;YACN,SAAS;YACTlB;YACA,SAASkB,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO;YACtB,UAAU,OAAOC;gBACf,MAAMC,MAAMD,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;YACnC;QACF;IACF;IAEA,MAAc,iBACZP,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMY,aAAa,IAAI,CAAC,gBAAgB,CAACT,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACY;IACrB;IAEA,MAAc,iBACZT,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMa,WAAWV,KAAK,IAAI;QAC1B,MAAMW,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;QACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMtB,QAAQY,KAAK,KAAK;QAExB,IAAI,CAACY,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EACd,IAAIC,wBAAwB/B,KAAK,CAAC8B,MAAM,GAAG;gBACzCjC,MACE,2DACAG,KAAK,CAAC8B,MAAM;gBAEd,MAAME,kBAAkBC,qBAAqBjC,KAAK,CAAC8B,MAAM;gBACzD,IAAIE,iBACFhC,KAAK,CAAC8B,MAAM,GAAGE;YAEnB,OAAO;gBACL,MAAM9B,aAAaH,oBAAoBC,KAAK,CAAC8B,MAAM;gBACnDjC,MACE,uCACA,CAAC,YAAY,EAAEyB,UAAU,EACzB,CAAC,MAAM,EAAEY,KAAK,SAAS,CAAClC,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEI,KAAK,SAAS,CAAChC,aAAa;gBAE5C,MAAMiC,aAAa,IAAI,CAAC,gBAAgB,CACtCjC,YACAF,KAAK,CAAC8B,MAAM,EACZrB,SACA,CAAC2B;oBACCpC,KAAK,CAAC8B,MAAM,GAAGM;gBACjB;gBAEF3B,QAAQ,KAAK,CAAC,IAAI,CAAC0B;YACrB;iBACK;gBACLE,OACE,CAACR,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3EzB,MAAM,CAAC,OAAO,EAAEiC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMgB,OAKF;YACF,MAAM;YACN,SAAShB;YACT,SAASV,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,OAAOhB,OAAOuC;oBAKWC;gBAJjC3C,MACE,oBACAyB,UACAtB,OACA,CAAC,4BAA4B,EAAE,QAAAwC,CAAAA,uBAAAA,YAAY,OAAO,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,MAAM,EAAE;gBAG9D,MAAMC,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBZ,qBAAqB,OAAO,CAAC,CAACC;oBAC5BO,OACErC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEA,IAAI;oBACF,MAAMoB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC7C,MAAM;gCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC2B,OAAO,IAAI,EAAExB;gCACrDH,MAAM;4BACR;wBACF;wBACAuB,MAAM;qBACP;gBACH,EAAE,OAAOuB,eAAoB;oBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,IAAInB,OAAO,WAAW,EACpB,IAAI;oBACFxB,QAAQ8C,iBAAiB9C,OAAOwB,OAAO,WAAW;gBACpD,EAAE,OAAOuB,OAAY;oBACnB,MAAM,IAAIrB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEuB,MAAM,OAAO,CAAC,cAAc,EAAEb,KAAK,SAAS,CAAClC,QAAQ,EACtG;wBAAE,OAAO+C;oBAAM;gBAEnB;gBAGFlD,MAAM,kBAAkB2B,OAAO,IAAI;gBACnC,MAAMwB,WAAWxB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAMyB,eAAe,MAAMD,SAAShD,OAAOuC;gBAC3C1C,MAAM,iBAAiB2B,OAAO,IAAI,EAAE,WAAWyB;gBAE/C,MAAMC,mBAAmB1B,OAAO,gBAAgB,IAAI;gBACpD,IAAI0B,mBAAmB,GACrB,MAAM9B,MAAM8B;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCrD,MAAM;wBACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC2B,OAAO,IAAI,EAAExB;wBACpDH,MAAM;oBACR;gBACF,EAAE,OAAO8C,eAAoB;oBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,OAAO;oBACL,QAAQ;wBACN,SAAS;wBACT,QAAQrB;wBACR,OAAOtB;wBACP,QAAQiD;oBACV;gBACF;YACF;QACF;QAEAxC,QAAQ,KAAK,CAAC,IAAI,CAAC6B;IACrB;IAEQ,iBACN1B,IAAyC,EACzCuC,mBAAiD,EACjD1C,OAAyB,EACzB2C,QAAgD,EACd;QAClC,MAAM,EAAE5C,SAAS,EAAEH,WAAW,EAAE,GAAGI;QACnC,IAAI4C,cAAcF;QAElB,IAAI,AAAuB,YAAvB,OAAOE,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI7C,AAAcQ,WAAdR,WACF6C,cAAc;YACZ,GAAGA,WAAW;YACd7C;QACF;QAGF,MAAM8C,WAA6C;YACjD,MAAM;YACN,SAAS;YACT,SAAS7C,QAAQ,OAAO,IAAIO;YAC5B,OAAOqC;YACP,SAASzC,KAAK,OAAO;YACrB,UAAU,OAAOZ,OAAOuC;oBAiDZgB,eAMgBC,iBACPC;gBAvDnB,MAAM,EAAEnB,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACErC,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GAC3B,CAAC,qDAAqD,EAAEkC,KAAK,SAAS,CACpElC,QACC;gBAGL,IAAI,CAACyC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,IAAIiB;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;wBAQJC,gBACTC;oBARJ,IAAI,CAACF,MACH;oBAEFH,aAAaG;oBACbvB,KAAK,GAAG,GAAG;wBACTuB;oBACF;oBACAvB,KAAK,KAAK,GAAG,QAAAwB,CAAAA,iBAAAA,KAAK,QAAQ,AAAD,IAAZA,KAAAA,IAAAA,eAAe,KAAK;oBACjC,IAAI,QAAAC,CAAAA,kBAAAA,KAAK,QAAQ,AAAD,IAAZA,KAAAA,IAAAA,gBAAe,eAAe,EAChCzB,KAAK,eAAe,GAAGuB,KAAK,QAAQ,CAAC,eAAe;gBAExD;gBAGA,IAAIG;gBACJ,IAAIhE,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,uBAAuB,EACvDgE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;oBAC3D,QAAQ;wBAAChE,MAAM,KAAK;qBAAC;gBACvB;gBAEF,MAAMiE,mBAAmBD,gBACrBE,0BACE;oBACE,GAAGF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;oBAC9C,GAAGA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;gBAChD,GACA,AAAwB,YAAxB,OAAOhE,MAAM,MAAM,GACfA,MAAM,MAAM,GACZuD,AAAAA,SAAAA,CAAAA,gBAAAA,MAAM,MAAM,AAAD,IAAXA,KAAAA,IAAAA,cAAc,MAAM,AAAD,KAAK,MAE9BvC;gBACJ,MAAMmD,0BAA0B,CAAC,CAACF;gBAElC,MAAMG,cAAcpE,MAAM,MAAM;gBAChC,MAAMqE,oBAAoB,QAAAb,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CAACY;gBAC3D,MAAME,aAAab,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,KAAK;gBAEzD,MAAMc,mBAAmBJ,0BACrB,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACApE,MAAM,SAAS;gBAErB,MAAMyE,eAAe,CAAC,CAACF;gBAEvB,MAAMG,cAAc;gBAEpB,IAAIC;gBACJ,IAAI,CAACR,2BAA2B,CAACM,gBAAgB,CAACC,aAChD,IAAI;oBACFf,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC3D,OACA;wBACE,SAASyC;oBACX,GACApC;oBAEFuD,UAAUD,aAAa,IAAI;oBAC3BgB,sBAAsBhB,aAAa,OAAO;gBAC5C,EAAE,OAAOZ,OAAO;oBACd,IAAIA,iBAAiB6B,cACnBhB,UAAUb,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAGF,MAAM8B,UACJZ,oBAAoBM,oBAAoBI;gBAE1C,IAAIG;gBACJ,IACED,WACA,IAAI,CAAC,SAAS,IACd,CAACJ,gBACDzE,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EACpC,IAAI;wBAOQ+E;oBANV,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CACtDH,QAAQ,IAAI,EACZ;wBACE,mBACE,AAAwB,YAAxB,OAAO7E,MAAM,MAAM,GACfA,MAAM,MAAM,WACZ+E,CAAAA,iBAAAA,MAAM,MAAM,AAAD,IAAXA,KAAAA,IAAAA,eAAc,MAAM;wBAC1B1E;oBACF;oBAEF,IAAI2E,WAAWC,OAAO,IAAI,CAACD,SAAS,MAAM,GAAG,GAAG;wBAC9CnF,MACE,uCACAuE,aACAY;wBAEFF,oBAAoBE;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQZ;4BACR,OAAOY;wBACT,GACAX;oBAEJ,OACExE,MACE,yDACAuE;gBAGN,EAAE,OAAOrB,OAAO;oBACdlD,MAAM,kCAAkCkD;gBAC1C;qBAEAlD,MAAM;gBAIV,IAAI,CAACgF,SAAS;oBACZ,IAAInB,YACF,MAAM,IAAIkB,aACR,CAAC,oBAAoB,EAAE5E,MAAM,MAAM,EAAE,EACrC0D;oBAGJ,MAAM,IAAIhC,MAAM,CAAC,mBAAmB,EAAE1B,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIkF;gBAEJ,IAAIf,yBACFe,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOlF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIyE,cACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPZ;wBACA,aAAaQ;oBACf;gBACF;gBAGF1B,QAAAA,YAAAA,SAAWyB;gBAEX,OAAO;oBACL,QAAQ;wBACNA;oBACF;oBACAK;gBACF;YACF;QACF;QAEA,OAAO5B;IACT;IA9dA,YAAY,EAAE6B,iBAAiB,EAAEC,OAAO,EAAEC,SAAS,EAAmB,CAAE;QANxE,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAGE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;IACnB;AA2dF"}

package/dist/es/agent/tasks.mjs

@@ -42,7 +42,6 @@ class TaskExecutor {
         const task = {
             type: 'Planning',
             subType: 'LoadYaml',
-            locate: null,
             param: {
                 userInstruction
             },
@@ -78,7 +77,6 @@ class TaskExecutor {
         const task = {
             type: 'Planning',
             subType: 'Plan',
-            locate: null,
             param: {
                 userInstruction,
                 aiActionContext: actionContext
@@ -188,7 +186,6 @@ class TaskExecutor {
         const queryTask = {
             type: 'Insight',
             subType: type,
-            locate: null,
             param: {
                 dataDemand: multimodalPrompt ? {
                     demand,
@@ -274,8 +271,7 @@ class TaskExecutor {
             type: 'Sleep',
             param: {
                 timeMs
-            },
-            locate: null
+            }
         };
     }
     async taskForSleep(timeMs, _modelConfig) {

package/dist/es/agent/tasks.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/tasks.mjs","sources":["webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["import { ConversationHistory, plan, uiTarsPlanning } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly taskBuilder: TaskBuilder;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      hooks?: TaskExecutorHooks;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.hooks = opts.hooks;\n    this.conversationHistory = new ConversationHistory();\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      subTask?: boolean;\n    },\n  ) {\n    return this.taskBuilder.build(plans, modelConfig, options);\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      locate: null,\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  private createPlanningTask(\n    userInstruction: string,\n    actionContext: string | undefined,\n    modelConfig: IModelConfig,\n  ): ExecutionTaskPlanningApply {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      locate: null,\n      param: {\n        userInstruction,\n        aiActionContext: actionContext,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        const { vlMode } = modelConfig;\n        const uiTarsModelVersion =\n          vlMode === 'vlm-ui-tars' ? modelConfig.uiTarsModelVersion : undefined;\n\n        assert(\n          this.interface.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.interface.actionSpace();\n        debug(\n          'actionSpace for this interface is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await (uiTarsModelVersion ? uiTarsPlanning : plan)(\n          param.userInstruction,\n          {\n            context: uiContext,\n            actionContext: param.aiActionContext,\n            interfaceType: this.interface.interfaceType as InterfaceType,\n            actionSpace,\n            modelConfig,\n            conversationHistory: this.conversationHistory,\n          },\n        );\n        debug('planResult', JSON.stringify(planResult, null, 2));\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push(this.sleepPlan(timeRemaining));\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          uiContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(plans, modelConfig);\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  private getReplanningCycleLimit(isVlmUiTars: boolean) {\n    return (\n      this.replanningCycleLimit ||\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) ||\n      (isVlmUiTars\n        ? defaultVlmUiTarsReplanningCycleLimit\n        : defaultReplanningCycleLimit)\n    );\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfig: IModelConfig,\n    actionContext?: string,\n    cacheable?: boolean,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit = this.getReplanningCycleLimit(\n      modelConfig.vlMode === 'vlm-ui-tars',\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanning ${replanningCycleLimit} times, which is more than the limit, please split the task into multiple steps`;\n\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      // Create planning task (automatically includes execution history if available)\n      const planningTask = this.createPlanningTask(\n        userPrompt,\n        actionContext,\n        modelConfig,\n      );\n\n      const result = await session.appendAndRun(planningTask);\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans, modelConfig, {\n          cacheable,\n          subTask: true,\n        });\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      await session.appendAndRun(executables.tasks);\n\n      // Check if task is complete\n      if (!planResult?.more_actions_needed_by_instruction) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      replanCount++;\n    }\n\n    const finalResult = {\n      output: {\n        yamlFlow,\n      },\n      runner,\n    };\n    return finalResult;\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      locate: null,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n          };\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, usage, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.usage = usage;\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  private sleepPlan(timeMs: number): PlanningAction<PlanningActionParamSleep> {\n    return {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n      locate: null,\n    };\n  }\n\n  async taskForSleep(timeMs: number, _modelConfig: IModelConfig) {\n    return this.taskBuilder.createSleepTask({\n      timeMs,\n    });\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        undefined,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - currentCheckStart);\n        const sleepTask = this.taskBuilder.createSleepTask({\n          timeMs: timeRemaining,\n        });\n        await session.append(sleepTask);\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n"],"names":["debug","getDebug","defaultReplanningCycleLimit","defaultVlmUiTarsReplanningCycleLimit","TaskExecutor","title","options","_this_hooks","ExecutionSession","Promise","plans","modelConfig","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","actionContext","startTime","Date","vlMode","uiTarsModelVersion","undefined","actionSpace","action","Array","console","planResult","uiTarsPlanning","plan","JSON","actions","log","more_actions_needed_by_instruction","error","usage","rawResponse","sleep","finalActions","timeNow","timeRemaining","tasks","result","output","isVlmUiTars","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","userPrompt","cacheable","replanCount","yamlFlow","replanningCycleLimit","errorMsg","planningTask","executables","finalResult","type","demand","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","thought","outputResult","Error","timeMs","_modelConfig","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","lastCheckStart","errorThought","currentCheckStart","now","sleepTask","interfaceInstance","service","opts","ConversationHistory","TaskBuilder"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAkDA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,8BAA8B;AACpC,MAAMC,uCAAuC;AAItC,MAAMC;IAkBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IA0BQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;YAOkBC;QANlB,OAAO,IAAIC,iBACTH,OACA,IAAMI,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOH,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK;YACrB,cAAc,QAAAC,CAAAA,cAAAA,IAAI,CAAC,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,YAAY;QACxC;IAEJ;IAEA,MAAa,wBACXG,KAAuB,EACvBC,WAAyB,EACzBL,OAGC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACI,OAAOC,aAAaL;IACpD;IAEA,MAAM,uBAAuBM,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUH;QAGzB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEQ,mBACNT,eAAuB,EACvBU,aAAiC,EACjCX,WAAyB,EACG;QAC5B,MAAMK,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,QAAQ;YACR,OAAO;gBACLJ;gBACA,iBAAiBU;YACnB;YACA,UAAU,OAAOL,OAAOC;gBACtB,MAAMK,YAAYC,KAAK,GAAG;gBAC1B,MAAM,EAAEL,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,MAAM,EAAEM,MAAM,EAAE,GAAGd;gBACnB,MAAMe,qBACJD,AAAW,kBAAXA,SAA2Bd,YAAY,kBAAkB,GAAGgB;gBAE9DP,OACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;gBAEF,MAAMQ,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;gBACpD5B,MACE,sCACA4B,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDT,OAAOU,MAAM,OAAO,CAACF,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;gBAIrG,MAAMC,aAAa,MAAON,AAAAA,CAAAA,qBAAqBO,iBAAiBC,IAAG,EACjEjB,MAAM,eAAe,EACrB;oBACE,SAASE;oBACT,eAAeF,MAAM,eAAe;oBACpC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;oBAC3CW;oBACAjB;oBACA,qBAAqB,IAAI,CAAC,mBAAmB;gBAC/C;gBAEFX,MAAM,cAAcmC,KAAK,SAAS,CAACH,YAAY,MAAM;gBAErD,MAAM,EACJI,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClCC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,KAAK,EACN,GAAGV;gBAEJd,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCuB;gBACF;gBACAvB,gBAAgB,IAAI,CAAC,KAAK,GAAGsB;gBAE7B,MAAMG,eAAeP,WAAW,EAAE;gBAElC,IAAIM,OAAO;oBACT,MAAME,UAAUpB,KAAK,GAAG;oBACxB,MAAMqB,gBAAgBH,QAASE,CAAAA,UAAUrB,SAAQ;oBACjD,IAAIsB,gBAAgB,GAClBF,aAAa,IAAI,CAAC,IAAI,CAAC,SAAS,CAACE;gBAErC;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrBvB,OACE,CAACkB,sCAAsCI,OACvCH,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAASI;wBACTL;wBACAD;wBACA,UAAUL,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACAb;gBACF;YACF;QACF;QAEA,OAAOH;IACT;IAEA,MAAM,SACJX,KAAa,EACbK,KAAuB,EACvBC,WAAyB,EACC;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACT;QAC5C,MAAM,EAAEyC,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACpC,OAAOC;QAC5D,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMiC,SAAS,MAAMjC,QAAQ,YAAY,CAACgC;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACA3B;QACF;IACF;IAEQ,wBAAwB4B,WAAoB,EAAE;QACpD,OACE,IAAI,CAAC,oBAAoB,IACzBC,oBAAoB,oBAAoB,CACtCC,oCAEDF,CAAAA,cACG9C,uCACAD,2BAA0B;IAElC;IAEA,MAAM,OACJkD,UAAkB,EAClBzC,WAAyB,EACzBW,aAAsB,EACtB+B,SAAmB,EAQnB;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMvC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUqC;QAEzB,MAAM/B,SAASP,QAAQ,SAAS;QAEhC,IAAIwC,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBAAuB,IAAI,CAAC,uBAAuB,CACvD7C,AAAuB,kBAAvBA,YAAY,MAAM;QAIpB,MAAO,KAAM;YACX,IAAI2C,cAAcE,sBAAsB;gBACtC,MAAMC,WAAW,CAAC,WAAW,EAAED,qBAAqB,+EAA+E,CAAC;gBAEpI,OAAO1C,QAAQ,eAAe,CAAC2C;YACjC;YAGA,MAAMC,eAAe,IAAI,CAAC,kBAAkB,CAC1CN,YACA9B,eACAX;YAGF,MAAMoC,SAAS,MAAMjC,QAAQ,YAAY,CAAC4C;YAC1C,MAAM1B,aAAae,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YAGjC,MAAMrC,QAAQsB,AAAAA,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,OAAO,AAAD,KAAK,EAAE;YACvCuB,SAAS,IAAI,IAAKvB,AAAAA,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,QAAQ,AAAD,KAAK,EAAE;YAE5C,IAAI2B;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAACjD,OAAOC,aAAa;oBACnE0C;oBACA,SAAS;gBACX;YACF,EAAE,OAAOd,OAAO;gBACd,OAAOzB,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEyB,MAAM,SAAS,EAAEJ,KAAK,SAAS,CAC5EzB,QACC;YAEP;YACA,MAAMI,QAAQ,YAAY,CAAC6C,YAAY,KAAK;YAG5C,IAAI,CAAC3B,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,kCAAkC,AAAD,GAChD;YAIFsB;QACF;QAEA,MAAMM,cAAc;YAClB,QAAQ;gBACNL;YACF;YACAlC;QACF;QACA,OAAOuC;IACT;IAEQ,oBACNC,IAAsE,EACtEC,MAA2B,EAC3BnD,WAAyB,EACzBoD,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,QAAQ;YACR,OAAO;gBACL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO7C,OAAOiD;gBACtB,MAAM,EAAElD,IAAI,EAAE,GAAGkD;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZrD,KAAK,GAAG,GAAG;wBACTqD;oBACF;gBACF;gBAGA,MAAMlD,YAAY+C,YAAY,SAAS;gBACvC9C,OAAOD,WAAW;gBAElB,MAAMmD,mBAAmBT,AAAS,YAATA;gBACzB,IAAIU,cAAcT;gBAClB,IAAIU,cAAc;gBAClB,IAAIF,oBAAqBT,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEW,cAAc;oBACd,MAAMC,gBACJZ,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIS,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGX,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,IAAIY;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1D/D,MAAM;oBACN,MAAM4E,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACA5D,aACAoD,KACAY,sBACAX;gBAEJ,EAAE,OAAOzB,OAAO;oBACd,IAAIA,iBAAiBuC,cACnBV,UAAU7B,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEwC,IAAI,EAAEvC,KAAK,EAAEwC,OAAO,EAAEX,IAAI,EAAE,GAAGK;gBACvCN,UAAUC;gBAEV,IAAIY,eAAeF;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTE,eAAeF;qBACV,IAAIlB,AAAS,cAATA,MAEPoB,eADEF,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTE,eAAe;qBACV;oBACL7D,OACE2D,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,IAAM,CAACP,YAAY,AAAD,MAAM7C,QACxB;oBAEFsD,eAAgBF,IAAY,CAACP,YAAY;gBAC3C;gBAGF,IAAIX,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCjE,KAAK,KAAK,GAAGwB;oBACbxB,KAAK,OAAO,GAAGgE;oBACf,MAAM,IAAIE,MAAM,CAAC,kBAAkB,EAAEF,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQC;oBACR,KAAKd;oBACL3B;oBACAwC;gBACF;YACF;QACF;QAEA,OAAOf;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BnD,WAAyB,EACzBoD,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMlD,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACE8C,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAAS3B,KAAK,SAAS,CAAC2B;QAIzD,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACAnD,aACAoD,KACAC;QAGF,MAAM3C,SAASP,QAAQ,SAAS;QAChC,MAAMiC,SAAS,MAAMjC,QAAQ,YAAY,CAACmD;QAE1C,IAAI,CAAClB,QACH,MAAM,IAAImC,MACR;QAIJ,MAAM,EAAElC,MAAM,EAAEgC,OAAO,EAAE,GAAGjC;QAE5B,OAAO;YACLC;YACAgC;YACA3D;QACF;IACF;IAEQ,UAAU8D,MAAc,EAA4C;QAC1E,OAAO;YACL,MAAM;YACN,OAAO;gBACLA;YACF;YACA,QAAQ;QACV;IACF;IAEA,MAAM,aAAaA,MAAc,EAAEC,YAA0B,EAAE;QAC7D,OAAO,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;YACtCD;QACF;IACF;IAEA,MAAM,QACJE,SAAsB,EACtBtB,GAA+B,EAC/BpD,WAAyB,EACO;QAChC,MAAM,EAAE2E,UAAU,EAAEtB,gBAAgB,EAAE,GAAGuB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMxE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAWyE;QAE1B,MAAMnE,SAASP,QAAQ,SAAS;QAChC,MAAM,EAAE2E,SAAS,EAAEC,eAAe,EAAE,GAAG3B;QAEvC3C,OAAOiE,WAAW;QAClBjE,OAAOqE,WAAW;QAClBrE,OAAOsE,iBAAiB;QAExBtE,OACEsE,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBnE,KAAK,GAAG;QACjC,IAAIoE,iBAAiBD;QACrB,IAAIE,eAAe;QAEnB,MAAOD,iBAAiBD,oBAAoBF,UAAW;YACrD,MAAMK,oBAAoBtE,KAAK,GAAG;YAClCoE,iBAAiBE;YACjB,MAAM7B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAqB,YACA3E,aACAgB,QACAqC;YAGF,MAAMjB,SAAU,MAAMjC,QAAQ,YAAY,CAACmD;YAO3C,IAAIlB,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQpB;gBACRN;YACF;YAGFwE,eACE9C,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACb,CAACA,UAAU,CAAC,0BAA0B,EAAEuC,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMS,MAAMvE,KAAK,GAAG;YACpB,IAAIuE,MAAMD,oBAAoBJ,iBAAiB;gBAC7C,MAAM7C,gBAAgB6C,kBAAmBK,CAAAA,MAAMD,iBAAgB;gBAC/D,MAAME,YAAY,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;oBACjD,QAAQnD;gBACV;gBACA,MAAM/B,QAAQ,MAAM,CAACkF;YACvB;QACF;QAEA,OAAOlF,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAE+E,cAAc;IACnE;IAniBA,YACEI,iBAAoC,EACpCC,OAAgB,EAChBC,IAKC,CACD;QA9BF;QAEA;QAEA;QAEA,uBAAiB,eAAjB;QAEA,uBAAQ,uBAAR;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAiBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;QAC5C,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAG,IAAIC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCJ;YACAC;YACA,WAAWC,KAAK,SAAS;QAC3B;IACF;AA8gBF"}
+{"version":3,"file":"agent/tasks.mjs","sources":["webpack://@midscene/core/./src/agent/tasks.ts"],"sourcesContent":["import { ConversationHistory, plan, uiTarsPlanning } from '@/ai-model';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_REPLANNING_CYCLE_LIMIT,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst defaultReplanningCycleLimit = 10;\nconst defaultVlmUiTarsReplanningCycleLimit = 40;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly taskBuilder: TaskBuilder;\n\n  private conversationHistory: ConversationHistory;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      hooks?: TaskExecutorHooks;\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.hooks = opts.hooks;\n    this.conversationHistory = new ConversationHistory();\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      subTask?: boolean;\n    },\n  ) {\n    return this.taskBuilder.build(plans, modelConfig, options);\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            more_actions_needed_by_instruction: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  private createPlanningTask(\n    userInstruction: string,\n    actionContext: string | undefined,\n    modelConfig: IModelConfig,\n  ): ExecutionTaskPlanningApply {\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'Plan',\n      param: {\n        userInstruction,\n        aiActionContext: actionContext,\n      },\n      executor: async (param, executorContext) => {\n        const startTime = Date.now();\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        const { vlMode } = modelConfig;\n        const uiTarsModelVersion =\n          vlMode === 'vlm-ui-tars' ? modelConfig.uiTarsModelVersion : undefined;\n\n        assert(\n          this.interface.actionSpace,\n          'actionSpace for device is not implemented',\n        );\n        const actionSpace = await this.interface.actionSpace();\n        debug(\n          'actionSpace for this interface is:',\n          actionSpace.map((action) => action.name).join(', '),\n        );\n        assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n        if (actionSpace.length === 0) {\n          console.warn(\n            `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n          );\n        }\n\n        const planResult = await (uiTarsModelVersion ? uiTarsPlanning : plan)(\n          param.userInstruction,\n          {\n            context: uiContext,\n            actionContext: param.aiActionContext,\n            interfaceType: this.interface.interfaceType as InterfaceType,\n            actionSpace,\n            modelConfig,\n            conversationHistory: this.conversationHistory,\n          },\n        );\n        debug('planResult', JSON.stringify(planResult, null, 2));\n\n        const {\n          actions,\n          log,\n          more_actions_needed_by_instruction,\n          error,\n          usage,\n          rawResponse,\n          sleep,\n        } = planResult;\n\n        executorContext.task.log = {\n          ...(executorContext.task.log || {}),\n          rawResponse,\n        };\n        executorContext.task.usage = usage;\n\n        const finalActions = actions || [];\n\n        if (sleep) {\n          const timeNow = Date.now();\n          const timeRemaining = sleep - (timeNow - startTime);\n          if (timeRemaining > 0) {\n            finalActions.push(this.sleepPlan(timeRemaining));\n          }\n        }\n\n        if (finalActions.length === 0) {\n          assert(\n            !more_actions_needed_by_instruction || sleep,\n            error ? `Failed to plan: ${error}` : 'No plan found',\n          );\n        }\n\n        return {\n          output: {\n            actions: finalActions,\n            more_actions_needed_by_instruction,\n            log,\n            yamlFlow: planResult.yamlFlow,\n          },\n          cache: {\n            hit: false,\n          },\n          uiContext,\n        };\n      },\n    };\n\n    return task;\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(plans, modelConfig);\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  private getReplanningCycleLimit(isVlmUiTars: boolean) {\n    return (\n      this.replanningCycleLimit ||\n      globalConfigManager.getEnvConfigInNumber(\n        MIDSCENE_REPLANNING_CYCLE_LIMIT,\n      ) ||\n      (isVlmUiTars\n        ? defaultVlmUiTarsReplanningCycleLimit\n        : defaultReplanningCycleLimit)\n    );\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfig: IModelConfig,\n    actionContext?: string,\n    cacheable?: boolean,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n        }\n      | undefined\n    >\n  > {\n    this.conversationHistory.reset();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Action', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit = this.getReplanningCycleLimit(\n      modelConfig.vlMode === 'vlm-ui-tars',\n    );\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanning ${replanningCycleLimit} times, which is more than the limit, please split the task into multiple steps`;\n\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      // Create planning task (automatically includes execution history if available)\n      const planningTask = this.createPlanningTask(\n        userPrompt,\n        actionContext,\n        modelConfig,\n      );\n\n      const result = await session.appendAndRun(planningTask);\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(plans, modelConfig, {\n          cacheable,\n          subTask: true,\n        });\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      await session.appendAndRun(executables.tasks);\n\n      // Check if task is complete\n      if (!planResult?.more_actions_needed_by_instruction) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      replanCount++;\n    }\n\n    const finalResult = {\n      output: {\n        yamlFlow,\n      },\n      runner,\n    };\n    return finalResult;\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n          };\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          demandInput = {\n            [keyOfResult]: `${type}, ${demand}`,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, usage, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.usage = usage;\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          usage,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  private sleepPlan(timeMs: number): PlanningAction<PlanningActionParamSleep> {\n    return {\n      type: 'Sleep',\n      param: {\n        timeMs,\n      },\n    };\n  }\n\n  async taskForSleep(timeMs: number, _modelConfig: IModelConfig) {\n    return this.taskBuilder.createSleepTask({\n      timeMs,\n    });\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const { timeoutMs, checkIntervalMs } = opt;\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        undefined,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const timeRemaining = checkIntervalMs - (now - currentCheckStart);\n        const sleepTask = this.taskBuilder.createSleepTask({\n          timeMs: timeRemaining,\n        });\n        await session.append(sleepTask);\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n"],"names":["debug","getDebug","defaultReplanningCycleLimit","defaultVlmUiTarsReplanningCycleLimit","TaskExecutor","title","options","_this_hooks","ExecutionSession","Promise","plans","modelConfig","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","actionContext","startTime","Date","vlMode","uiTarsModelVersion","undefined","actionSpace","action","Array","console","planResult","uiTarsPlanning","plan","JSON","actions","log","more_actions_needed_by_instruction","error","usage","rawResponse","sleep","finalActions","timeNow","timeRemaining","tasks","result","output","isVlmUiTars","globalConfigManager","MIDSCENE_REPLANNING_CYCLE_LIMIT","userPrompt","cacheable","replanCount","yamlFlow","replanningCycleLimit","errorMsg","planningTask","executables","finalResult","type","demand","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","thought","outputResult","Error","timeMs","_modelConfig","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","overallStartTime","lastCheckStart","errorThought","currentCheckStart","now","sleepTask","interfaceInstance","service","opts","ConversationHistory","TaskBuilder"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAkDA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,8BAA8B;AACpC,MAAMC,uCAAuC;AAItC,MAAMC;IAkBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IA0BQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;YAOkBC;QANlB,OAAO,IAAIC,iBACTH,OACA,IAAMI,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOH,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK;YACrB,cAAc,QAAAC,CAAAA,cAAAA,IAAI,CAAC,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,YAAY;QACxC;IAEJ;IAEA,MAAa,wBACXG,KAAuB,EACvBC,WAAyB,EACzBL,OAGC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAACI,OAAOC,aAAaL;IACpD;IAEA,MAAM,uBAAuBM,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUH;QAGzB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,oCAAoC;wBACpC,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEQ,mBACNT,eAAuB,EACvBU,aAAiC,EACjCX,WAAyB,EACG;QAC5B,MAAMK,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;gBACA,iBAAiBU;YACnB;YACA,UAAU,OAAOL,OAAOC;gBACtB,MAAMK,YAAYC,KAAK,GAAG;gBAC1B,MAAM,EAAEL,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,MAAM,EAAEM,MAAM,EAAE,GAAGd;gBACnB,MAAMe,qBACJD,AAAW,kBAAXA,SAA2Bd,YAAY,kBAAkB,GAAGgB;gBAE9DP,OACE,IAAI,CAAC,SAAS,CAAC,WAAW,EAC1B;gBAEF,MAAMQ,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;gBACpD5B,MACE,sCACA4B,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;gBAEhDT,OAAOU,MAAM,OAAO,CAACF,cAAc;gBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;gBAIrG,MAAMC,aAAa,MAAON,AAAAA,CAAAA,qBAAqBO,iBAAiBC,IAAG,EACjEjB,MAAM,eAAe,EACrB;oBACE,SAASE;oBACT,eAAeF,MAAM,eAAe;oBACpC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;oBAC3CW;oBACAjB;oBACA,qBAAqB,IAAI,CAAC,mBAAmB;gBAC/C;gBAEFX,MAAM,cAAcmC,KAAK,SAAS,CAACH,YAAY,MAAM;gBAErD,MAAM,EACJI,OAAO,EACPC,GAAG,EACHC,kCAAkC,EAClCC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,KAAK,EACN,GAAGV;gBAEJd,gBAAgB,IAAI,CAAC,GAAG,GAAG;oBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;oBAClCuB;gBACF;gBACAvB,gBAAgB,IAAI,CAAC,KAAK,GAAGsB;gBAE7B,MAAMG,eAAeP,WAAW,EAAE;gBAElC,IAAIM,OAAO;oBACT,MAAME,UAAUpB,KAAK,GAAG;oBACxB,MAAMqB,gBAAgBH,QAASE,CAAAA,UAAUrB,SAAQ;oBACjD,IAAIsB,gBAAgB,GAClBF,aAAa,IAAI,CAAC,IAAI,CAAC,SAAS,CAACE;gBAErC;gBAEA,IAAIF,AAAwB,MAAxBA,aAAa,MAAM,EACrBvB,OACE,CAACkB,sCAAsCI,OACvCH,QAAQ,CAAC,gBAAgB,EAAEA,OAAO,GAAG;gBAIzC,OAAO;oBACL,QAAQ;wBACN,SAASI;wBACTL;wBACAD;wBACA,UAAUL,WAAW,QAAQ;oBAC/B;oBACA,OAAO;wBACL,KAAK;oBACP;oBACAb;gBACF;YACF;QACF;QAEA,OAAOH;IACT;IAEA,MAAM,SACJX,KAAa,EACbK,KAAuB,EACvBC,WAAyB,EACC;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACT;QAC5C,MAAM,EAAEyC,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAACpC,OAAOC;QAC5D,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMiC,SAAS,MAAMjC,QAAQ,YAAY,CAACgC;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACA3B;QACF;IACF;IAEQ,wBAAwB4B,WAAoB,EAAE;QACpD,OACE,IAAI,CAAC,oBAAoB,IACzBC,oBAAoB,oBAAoB,CACtCC,oCAEDF,CAAAA,cACG9C,uCACAD,2BAA0B;IAElC;IAEA,MAAM,OACJkD,UAAkB,EAClBzC,WAAyB,EACzBW,aAAsB,EACtB+B,SAAmB,EAQnB;QACA,IAAI,CAAC,mBAAmB,CAAC,KAAK;QAE9B,MAAMvC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,UAAUqC;QAEzB,MAAM/B,SAASP,QAAQ,SAAS;QAEhC,IAAIwC,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBAAuB,IAAI,CAAC,uBAAuB,CACvD7C,AAAuB,kBAAvBA,YAAY,MAAM;QAIpB,MAAO,KAAM;YACX,IAAI2C,cAAcE,sBAAsB;gBACtC,MAAMC,WAAW,CAAC,WAAW,EAAED,qBAAqB,+EAA+E,CAAC;gBAEpI,OAAO1C,QAAQ,eAAe,CAAC2C;YACjC;YAGA,MAAMC,eAAe,IAAI,CAAC,kBAAkB,CAC1CN,YACA9B,eACAX;YAGF,MAAMoC,SAAS,MAAMjC,QAAQ,YAAY,CAAC4C;YAC1C,MAAM1B,aAAae,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM;YAGjC,MAAMrC,QAAQsB,AAAAA,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,OAAO,AAAD,KAAK,EAAE;YACvCuB,SAAS,IAAI,IAAKvB,AAAAA,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,QAAQ,AAAD,KAAK,EAAE;YAE5C,IAAI2B;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAACjD,OAAOC,aAAa;oBACnE0C;oBACA,SAAS;gBACX;YACF,EAAE,OAAOd,OAAO;gBACd,OAAOzB,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEyB,MAAM,SAAS,EAAEJ,KAAK,SAAS,CAC5EzB,QACC;YAEP;YACA,MAAMI,QAAQ,YAAY,CAAC6C,YAAY,KAAK;YAG5C,IAAI,CAAC3B,CAAAA,QAAAA,aAAAA,KAAAA,IAAAA,WAAY,kCAAkC,AAAD,GAChD;YAIFsB;QACF;QAEA,MAAMM,cAAc;YAClB,QAAQ;gBACNL;YACF;YACAlC;QACF;QACA,OAAOuC;IACT;IAEQ,oBACNC,IAAsE,EACtEC,MAA2B,EAC3BnD,WAAyB,EACzBoD,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASJ;YACT,OAAO;gBACL,YAAYG,mBACP;oBACCF;oBACAE;gBACF,IACAF;YACN;YACA,UAAU,OAAO7C,OAAOiD;gBACtB,MAAM,EAAElD,IAAI,EAAE,GAAGkD;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZrD,KAAK,GAAG,GAAG;wBACTqD;oBACF;gBACF;gBAGA,MAAMlD,YAAY+C,YAAY,SAAS;gBACvC9C,OAAOD,WAAW;gBAElB,MAAMmD,mBAAmBT,AAAS,YAATA;gBACzB,IAAIU,cAAcT;gBAClB,IAAIU,cAAc;gBAClB,IAAIF,oBAAqBT,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEW,cAAc;oBACd,MAAMC,gBACJZ,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIS,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBACTC,cAAc;oBACZ,CAACC,YAAY,EAAE,GAAGX,KAAK,EAAE,EAAEC,QAAQ;gBACrC;gBAGF,IAAIY;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,KAAK,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1D/D,MAAM;oBACN,MAAM4E,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,WAAW,AAAD,MAAM;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACA5D,aACAoD,KACAY,sBACAX;gBAEJ,EAAE,OAAOzB,OAAO;oBACd,IAAIA,iBAAiBuC,cACnBV,UAAU7B,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEwC,IAAI,EAAEvC,KAAK,EAAEwC,OAAO,EAAEX,IAAI,EAAE,GAAGK;gBACvCN,UAAUC;gBAEV,IAAIY,eAAeF;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTE,eAAeF;qBACV,IAAIlB,AAAS,cAATA,MAEPoB,eADEF,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTE,eAAe;qBACV;oBACL7D,OACE2D,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,IAAM,CAACP,YAAY,AAAD,MAAM7C,QACxB;oBAEFsD,eAAgBF,IAAY,CAACP,YAAY;gBAC3C;gBAGF,IAAIX,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCjE,KAAK,KAAK,GAAGwB;oBACbxB,KAAK,OAAO,GAAGgE;oBACf,MAAM,IAAIE,MAAM,CAAC,kBAAkB,EAAEF,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQC;oBACR,KAAKd;oBACL3B;oBACAwC;gBACF;YACF;QACF;QAEA,OAAOf;IACT;IACA,MAAM,yBACJJ,IAA0D,EAC1DC,MAA2B,EAC3BnD,WAAyB,EACzBoD,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMlD,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACE8C,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAAS3B,KAAK,SAAS,CAAC2B;QAIzD,MAAMG,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CJ,MACAC,QACAnD,aACAoD,KACAC;QAGF,MAAM3C,SAASP,QAAQ,SAAS;QAChC,MAAMiC,SAAS,MAAMjC,QAAQ,YAAY,CAACmD;QAE1C,IAAI,CAAClB,QACH,MAAM,IAAImC,MACR;QAIJ,MAAM,EAAElC,MAAM,EAAEgC,OAAO,EAAE,GAAGjC;QAE5B,OAAO;YACLC;YACAgC;YACA3D;QACF;IACF;IAEQ,UAAU8D,MAAc,EAA4C;QAC1E,OAAO;YACL,MAAM;YACN,OAAO;gBACLA;YACF;QACF;IACF;IAEA,MAAM,aAAaA,MAAc,EAAEC,YAA0B,EAAE;QAC7D,OAAO,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;YACtCD;QACF;IACF;IAEA,MAAM,QACJE,SAAsB,EACtBtB,GAA+B,EAC/BpD,WAAyB,EACO;QAChC,MAAM,EAAE2E,UAAU,EAAEtB,gBAAgB,EAAE,GAAGuB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMxE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAWyE;QAE1B,MAAMnE,SAASP,QAAQ,SAAS;QAChC,MAAM,EAAE2E,SAAS,EAAEC,eAAe,EAAE,GAAG3B;QAEvC3C,OAAOiE,WAAW;QAClBjE,OAAOqE,WAAW;QAClBrE,OAAOsE,iBAAiB;QAExBtE,OACEsE,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAME,mBAAmBnE,KAAK,GAAG;QACjC,IAAIoE,iBAAiBD;QACrB,IAAIE,eAAe;QAEnB,MAAOD,iBAAiBD,oBAAoBF,UAAW;YACrD,MAAMK,oBAAoBtE,KAAK,GAAG;YAClCoE,iBAAiBE;YACjB,MAAM7B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAqB,YACA3E,aACAgB,QACAqC;YAGF,MAAMjB,SAAU,MAAMjC,QAAQ,YAAY,CAACmD;YAO3C,IAAIlB,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,EAChB,OAAO;gBACL,QAAQpB;gBACRN;YACF;YAGFwE,eACE9C,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,OAAO,AAAD,KACb,CAACA,UAAU,CAAC,0BAA0B,EAAEuC,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMS,MAAMvE,KAAK,GAAG;YACpB,IAAIuE,MAAMD,oBAAoBJ,iBAAiB;gBAC7C,MAAM7C,gBAAgB6C,kBAAmBK,CAAAA,MAAMD,iBAAgB;gBAC/D,MAAME,YAAY,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC;oBACjD,QAAQnD;gBACV;gBACA,MAAM/B,QAAQ,MAAM,CAACkF;YACvB;QACF;QAEA,OAAOlF,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAE+E,cAAc;IACnE;IA/hBA,YACEI,iBAAoC,EACpCC,OAAgB,EAChBC,IAKC,CACD;QA9BF;QAEA;QAEA;QAEA,uBAAiB,eAAjB;QAEA,uBAAQ,uBAAR;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAiBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,WAAW;QAC5C,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAG,IAAIC;QAC/B,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCJ;YACAC;YACA,WAAWC,KAAK,SAAS;QAC3B;IACF;AA0gBF"}

package/dist/es/agent/ui-utils.mjs

@@ -22,7 +22,7 @@ function locateParamStr(locate) {
 }
 function scrollParamStr(scrollParam) {
     if (!scrollParam) return '';
-    return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;
+    return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;
 }
 function pullParamStr(pullParam) {
     if (!pullParam) return '';
@@ -48,18 +48,18 @@ function paramStr(task) {
         value = (null == task ? void 0 : null == (_task_param1 = task.param) ? void 0 : _task_param1.dataDemand) || (null == task ? void 0 : null == (_task_param2 = task.param) ? void 0 : _task_param2.assertion);
     }
     if ('Action Space' === task.type) {
-        var _task_param3, _task_param4, _task_param5, _task_param6;
-        const locate = null == task ? void 0 : task.locate;
+        var _task_param3, _task_param4, _task_param5, _task_param6, _task_param7;
+        const locate = null == task ? void 0 : null == (_task_param3 = task.param) ? void 0 : _task_param3.locate;
         const locateStr = locate ? locateParamStr(locate) : '';
         value = task.thought || '';
-        if ('number' == typeof (null == task ? void 0 : null == (_task_param3 = task.param) ? void 0 : _task_param3.timeMs)) {
-            var _task_param7;
-            value = `${null == task ? void 0 : null == (_task_param7 = task.param) ? void 0 : _task_param7.timeMs}ms`;
-        } else if ('string' == typeof (null == task ? void 0 : null == (_task_param4 = task.param) ? void 0 : _task_param4.scrollType)) value = scrollParamStr(null == task ? void 0 : task.param);
-        else if ('string' == typeof (null == task ? void 0 : null == (_task_param5 = task.param) ? void 0 : _task_param5.direction) && (null == task ? void 0 : task.subType) === 'AndroidPull') value = pullParamStr(null == task ? void 0 : task.param);
-        else if (void 0 !== (null == task ? void 0 : null == (_task_param6 = task.param) ? void 0 : _task_param6.value)) {
+        if ('number' == typeof (null == task ? void 0 : null == (_task_param4 = task.param) ? void 0 : _task_param4.timeMs)) {
             var _task_param8;
-            value = null == task ? void 0 : null == (_task_param8 = task.param) ? void 0 : _task_param8.value;
+            value = `${null == task ? void 0 : null == (_task_param8 = task.param) ? void 0 : _task_param8.timeMs}ms`;
+        } else if ('string' == typeof (null == task ? void 0 : null == (_task_param5 = task.param) ? void 0 : _task_param5.scrollType)) value = scrollParamStr(null == task ? void 0 : task.param);
+        else if ('string' == typeof (null == task ? void 0 : null == (_task_param6 = task.param) ? void 0 : _task_param6.direction) && (null == task ? void 0 : task.subType) === 'AndroidPull') value = pullParamStr(null == task ? void 0 : task.param);
+        else if (void 0 !== (null == task ? void 0 : null == (_task_param7 = task.param) ? void 0 : _task_param7.value)) {
+            var _task_param9;
+            value = null == task ? void 0 : null == (_task_param9 = task.param) ? void 0 : _task_param9.value;
         } else if ((null == task ? void 0 : task.param) && 'object' == typeof (null == task ? void 0 : task.param) && Object.keys((null == task ? void 0 : task.param) || {}).length > 0) value = null == task ? void 0 : task.param;
         if (locateStr) value = value ? `${locateStr} - ${value}` : locateStr;
     }

package/dist/es/agent/ui-utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/ui-utils.mjs","sources":["webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n      const prompt = locate.prompt.prompt;\n      const images = locate.prompt.images || [];\n\n      if (images.length === 0) return prompt;\n\n      const imagesStr = images\n        .map((image) => {\n          let url = image.url;\n          if (\n            url.startsWith('data:image/') ||\n            (url.startsWith('data:') && url.includes('base64'))\n          ) {\n            url = `${url.substring(0, 15)}...`;\n          }\n          return `[${image.name}](${url})`;\n        })\n        .join(', ');\n\n      return `${prompt}, ${imagesStr}`;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'once'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object' && locateParamStr(value as any)) {\n    return locateParamStr(value as any);\n  }\n\n  return JSON.stringify(value, undefined, 2);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","images","imagesStr","image","url","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","locateStr","_task_param7","_task_param8","Object","JSON","undefined"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAAE;YAC7D,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,MAAME,SAASF,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE;YAEzC,IAAIE,AAAkB,MAAlBA,OAAO,MAAM,EAAQ,OAAOD;YAEhC,MAAME,YAAYD,OACf,GAAG,CAAC,CAACE;gBACJ,IAAIC,MAAMD,MAAM,GAAG;gBACnB,IACEC,IAAI,UAAU,CAAC,kBACdA,IAAI,UAAU,CAAC,YAAYA,IAAI,QAAQ,CAAC,WAEzCA,MAAM,GAAGA,IAAI,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC;gBAEpC,OAAO,CAAC,CAAC,EAAED,MAAM,IAAI,CAAC,EAAE,EAAEC,IAAI,CAAC,CAAC;YAClC,GACC,IAAI,CAAC;YAER,OAAO,GAAGJ,OAAO,EAAE,EAAEE,WAAW;QAClC;IACF;IAEA,OAAO;AACT;AAEO,SAASG,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,OAAO,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACjI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZX,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGW,KAAK,GAAG,EAAEX,QAAQ;IAE9B,OAAOW;AACT;AAEO,SAASC,SAASf,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAsC,KAAK;SAC9D;YACGiB;QAARD,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,cAACjB,KAAgC,KAAK,AAAD,IAArCiB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAGF,IAAIjB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAEzBkB,cACAC;QAFFH,QACEE,AAAClB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAAoC,KAAK,AAAD,IAAzCkB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDlB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAAwC,KAAK,AAAD,IAA7CmB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAInB,AAAc,mBAAdA,KAAK,IAAI,EAAqB;YAKrBoB,cAGFC,cAIAC,cAKAC;QAhBT,MAAMrB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,MAAM;QACpD,MAAMwB,YAAYtB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAA8B,KAAK,AAAD,IAAnCoB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXT,QAAQ,GAAIhB,QAAAA,OAAAA,KAAAA,IAAAA,QAADyB,CAAAA,eAACzB,KAA8B,KAAK,AAAD,IAAnCyB,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQzB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDL,QAAQR,eAAgBR,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAA8B,KAAK,AAAD,IAAnCsB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACtB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CgB,QAAQN,aAAcV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARV,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,KAAK;QACrD,OAAO,IACL,AAAC1B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK,AAAD,KACnC,AAAgD,YAAhD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK,AAAD,KAC1C2B,OAAO,IAAI,CAAC,AAAC3B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK,AAAD,KAAK,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;QAG9C,IAAIwB,WAEAR,QADEA,QACM,GAAGQ,UAAU,GAAG,EAAER,OAAO,GAEzBQ;IAGd;IAEA,IAAI,AAAiB,WAAVR,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,SAAsBf,eAAee,QAC9C,OAAOf,eAAee;IAGxB,OAAOY,KAAK,SAAS,CAACZ,OAAOa,QAAW;AAC1C"}
+{"version":3,"file":"agent/ui-utils.mjs","sources":["webpack://@midscene/core/./src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    if (typeof locate.prompt === 'object' && locate.prompt.prompt) {\n      const prompt = locate.prompt.prompt;\n      const images = locate.prompt.images || [];\n\n      if (images.length === 0) return prompt;\n\n      const imagesStr = images\n        .map((image) => {\n          let url = image.url;\n          if (\n            url.startsWith('data:image/') ||\n            (url.startsWith('data:') && url.includes('base64'))\n          ) {\n            url = `${url.substring(0, 15)}...`;\n          }\n          return `[${image.name}](${url})`;\n        })\n        .join(', ');\n\n      return `${prompt}, ${imagesStr}`;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Action'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      value = (task as ExecutionTaskPlanning)?.param?.userInstruction;\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value =\n      (task as ExecutionTaskInsightQuery)?.param?.dataDemand ||\n      (task as ExecutionTaskInsightAssertion)?.param?.assertion;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'AndroidPull'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value) {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object' && locateParamStr(value as any)) {\n    return locateParamStr(value as any);\n  }\n\n  return JSON.stringify(value, undefined, 2);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","images","imagesStr","image","url","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","taskTitleStr","type","paramStr","value","_task_param","_task_param1","_task_param2","_task_param3","_task_param4","_task_param5","_task_param6","_task_param7","locateStr","_task_param8","_task_param9","Object","JSON","undefined"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAC9B,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAGtB,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IAAiBA,OAAO,MAAM,CAAC,MAAM,EAAE;YAC7D,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,MAAME,SAASF,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE;YAEzC,IAAIE,AAAkB,MAAlBA,OAAO,MAAM,EAAQ,OAAOD;YAEhC,MAAME,YAAYD,OACf,GAAG,CAAC,CAACE;gBACJ,IAAIC,MAAMD,MAAM,GAAG;gBACnB,IACEC,IAAI,UAAU,CAAC,kBACdA,IAAI,UAAU,CAAC,YAAYA,IAAI,QAAQ,CAAC,WAEzCA,MAAM,GAAGA,IAAI,SAAS,CAAC,GAAG,IAAI,GAAG,CAAC;gBAEpC,OAAO,CAAC,CAAC,EAAED,MAAM,IAAI,CAAC,EAAE,EAAEC,IAAI,CAAC,CAAC;YAClC,GACC,IAAI,CAAC;YAER,OAAO,GAAGJ,OAAO,EAAE,EAAEE,WAAW;QAClC;IACF;IAEA,OAAO;AACT;AAEO,SAASG,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,aACdC,IAcY,EACZX,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGW,KAAK,GAAG,EAAEX,QAAQ;IAE9B,OAAOW;AACT;AAEO,SAASC,SAASf,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,QAAAA,OAAAA,KAAAA,IAAAA,KAAsC,KAAK;SAC9D;YACGiB;QAARD,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAADiB,CAAAA,cAACjB,KAAgC,KAAK,AAAD,IAArCiB,KAAAA,IAAAA,YAAwC,eAAe;IACjE;IAGF,IAAIjB,AAAc,cAAdA,KAAK,IAAI,EAAgB;YAEzBkB,cACAC;QAFFH,QACEE,AAAClB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADkB,CAAAA,eAAClB,KAAoC,KAAK,AAAD,IAAzCkB,KAAAA,IAAAA,aAA4C,UAAU,AAAD,KACpDlB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADmB,CAAAA,eAACnB,KAAwC,KAAK,AAAD,IAA7CmB,KAAAA,IAAAA,aAAgD,SAAS,AAAD;IAC5D;IAEA,IAAInB,AAAc,mBAAdA,KAAK,IAAI,EAAqB;YACjBoB,cAIJC,cAGFC,cAIAC,cAKAC;QAhBT,MAAMtB,SAAUF,QAAAA,OAAAA,KAAAA,IAAAA,QAADoB,CAAAA,eAACpB,KAA8B,KAAK,AAAD,IAAnCoB,KAAAA,IAAAA,aAAsC,MAAM;QAC3D,MAAMK,YAAYvB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADqB,CAAAA,eAACrB,KAA8B,KAAK,AAAD,IAAnCqB,KAAAA,IAAAA,aAAsC,MAAM,AAAD,GAAgB;gBACzDK;YAAXV,QAAQ,GAAIhB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD0B,CAAAA,eAAC1B,KAA8B,KAAK,AAAD,IAAnC0B,KAAAA,IAAAA,aAAsC,MAAM,CAAC,EAAE,CAAC;QAC7D,OAAO,IACL,AAA4D,YAA5D,OAAQ1B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADsB,CAAAA,eAACtB,KAA8B,KAAK,AAAD,IAAnCsB,KAAAA,IAAAA,aAAsC,UAAU,AAAD,GAEtDN,QAAQR,eAAgBR,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACtD,IACL,AAA2D,YAA3D,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADuB,CAAAA,eAACvB,KAA8B,KAAK,AAAD,IAAnCuB,KAAAA,IAAAA,aAAsC,SAAS,AAAD,KACrD,AAACvB,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,OAAO,AAAD,MAAM,eAE3CgB,QAAQN,aAAcV,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;aACpD,IACL,AAAuD,WAA/CA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,QAADwB,CAAAA,eAACxB,KAA8B,KAAK,AAAD,IAAnCwB,KAAAA,IAAAA,aAAsC,KAAK,AAAD,GACjD;gBACQG;YAARX,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,QAAD2B,CAAAA,eAAC3B,KAA8B,KAAK,AAAD,IAAnC2B,KAAAA,IAAAA,aAAsC,KAAK;QACrD,OAAO,IACL,AAAC3B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK,AAAD,KACnC,AAAgD,YAAhD,OAAQA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK,AAAD,KAC1C4B,OAAO,IAAI,CAAC,AAAC5B,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK,AAAD,KAAK,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,QAAAA,OAAAA,KAAAA,IAAAA,KAA8B,KAAK;QAG9C,IAAIyB,WAEAT,QADEA,QACM,GAAGS,UAAU,GAAG,EAAET,OAAO,GAEzBS;IAGd;IAEA,IAAI,AAAiB,WAAVT,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,SAAsBf,eAAee,QAC9C,OAAOf,eAAee;IAGxB,OAAOa,KAAK,SAAS,CAACb,OAAOc,QAAW;AAC1C"}

package/dist/es/agent/utils.mjs

@@ -71,11 +71,12 @@ function ifPlanLocateParamIsBbox(planLocateParam) {
 function matchElementFromPlan(planLocateParam) {
     if (!planLocateParam) return;
     if (planLocateParam.bbox) {
+        var _planLocateParam_prompt;
         const centerPosition = {
             x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),
             y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2)
         };
-        const element = generateElementByPosition(centerPosition);
+        const element = generateElementByPosition(centerPosition, 'string' == typeof planLocateParam.prompt ? planLocateParam.prompt : (null == (_planLocateParam_prompt = planLocateParam.prompt) ? void 0 : _planLocateParam_prompt.prompt) || '');
         return element;
     }
 }
@@ -92,7 +93,8 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
                 Math.round(rect.left + rect.width / 2),
                 Math.round(rect.top + rect.height / 2)
             ],
-            rect
+            rect,
+            description: 'string' == typeof cachePrompt ? cachePrompt : cachePrompt.prompt || ''
         };
         debug('cache hit, prompt: %s', cachePrompt);
         return element;
@@ -101,7 +103,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.0.1-beta-20251110130223.0";
+const getMidsceneVersion = ()=>"1.0.1-beta-20251111100700.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/agent/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.mjs","sources":["webpack://@midscene/core/./src/agent/utils.ts"],"sourcesContent":["import type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst debugProfile = getDebug('web:tool:profile');\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: { uploadServerUrl?: string },\n): Promise<UIContext> {\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debugProfile('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debugProfile('Interface description end');\n\n  debugProfile('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debugProfile('UploadTestInfoToServer end');\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await interfaceInstance.size();\n  debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);\n\n  return {\n    size,\n    screenshotBase64: screenshotBase64!,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n\n    const element = generateElementByPosition(centerPosition);\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n"],"names":["debugProfile","getDebug","commonContextParser","interfaceInstance","_opt","assert","description","uploadTestInfoToServer","screenshotBase64","size","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","logMsg","getCurrentExecutionFile","trace","error","Error","stackTrace","pkgDir","process","stackLines","line","match","targetFileName","testFileIndex","Map","generateCacheId","fileName","taskFile","console","currentIndex","undefined","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","centerPosition","Math","element","generateElementByPosition","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","_context_taskCache","cacheDebug","rect","getMidsceneVersion","__VERSION__","parsePrompt","prompt"],"mappings":";;;;;;;AAqBA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,oBACpBC,iBAAoC,EACpCC,IAAkC;QAKdD;IAHpBE,OAAOF,mBAAmB;IAE1BH,aAAa;IACb,MAAMM,cAAcH,AAAAA,SAAAA,CAAAA,8BAAAA,kBAAkB,QAAQ,AAAD,IAAzBA,KAAAA,IAAAA,4BAAAA,IAAAA,CAAAA,kBAAiB,KAAiB;IACtDH,aAAa;IAEbA,aAAa;IACbO,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAJ,aAAa;IAEb,MAAMQ,mBAAmB,MAAML,kBAAkB,gBAAgB;IACjEE,OAAOG,kBAAmB;IAE1B,MAAMC,OAAO,MAAMN,kBAAkB,IAAI;IACzCH,aAAa,CAAC,MAAM,EAAES,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,MAAM,EAAEA,KAAK,GAAG,EAAE;IAElE,OAAO;QACLA;QACA,kBAAkBD;IACpB;AACF;AAEO,SAASE,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7CC,OAAO,CAAC,gCAAgC,EAAED,UAAU;AACtD;AAMO,SAASE,wBAAwBC,KAAc;IACpD,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,aAAaH,SAASC,MAAM,KAAK;IACvC,MAAMG,SAASC,QAAQ,GAAG,MAAM;IAChC,IAAIF,YAAY;QACd,MAAMG,aAAaH,WAAW,KAAK,CAAC;QACpC,KAAK,MAAMI,QAAQD,WACjB,IACEC,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,UACdA,KAAK,QAAQ,CAAC,QACd;YACA,MAAMC,QAAQD,KAAK,KAAK,CAAC;YACzB,IAAIC,QAAAA,QAAAA,KAAAA,IAAAA,KAAO,CAAC,EAAE,EAAE;gBACd,MAAMC,iBAAiBD,KAAK,CAAC,EAAE,CAC5B,OAAO,CAACJ,QAAQ,IAChB,IAAI,GACJ,OAAO,CAAC,OAAO;gBAClB,OAAOK;YACT;QACF;IAEJ;IACA,OAAO;AACT;AAEA,MAAMC,gBAAgB,IAAIC;AAEnB,SAASC,gBAAgBC,QAAiB;IAC/C,IAAIC,WAAWD,YAAYd;IAC3B,IAAI,CAACe,UAAU;QACbA,WAAWnB;QACXoB,QAAQ,IAAI,CACV;IAEJ;IAEA,IAAIL,cAAc,GAAG,CAACI,WAAW;QAC/B,MAAME,eAAeN,cAAc,GAAG,CAACI;QACvC,IAAIE,AAAiBC,WAAjBD,cACFN,cAAc,GAAG,CAACI,UAAUE,eAAe;IAE/C,OACEN,cAAc,GAAG,CAACI,UAAU;IAE9B,OAAO,GAAGA,SAAS,CAAC,EAAEJ,cAAc,GAAG,CAACI,WAAW;AACrD;AAEO,SAASI,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;QACxB,MAAMG,iBAAiB;YACrB,GAAGC,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;YACpE,GAAGI,KAAK,KAAK,CAAEJ,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;QACtE;QAEA,MAAMK,UAAUC,0BAA0BH;QAC1C,OAAOE;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;QAWzBC;IATL,IAAI,CAACH,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBE,MAAW,iCAAiCH;IAI9C,IAAI,UAACE,CAAAA,qBAAAA,QAAQ,SAAS,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,iBAAiB,AAAD,GACtC;IAGF,IAAI,CAACJ,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDK,MACE;IAKJ,IAAI;QACF,MAAMC,OACJ,MAAMN,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACND,KAAK,KAAK,CAACU,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCV,KAAK,KAAK,CAACU,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;QACF;QAEAD,MAAW,yBAAyBH;QACpC,OAAOL;IACT,EAAE,OAAOvB,OAAO;QACd+B,MAAW,qCAAqC/B;QAChD;IACF;AACF;AAIO,MAAMiC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBpB;IACpB;IAEF,OAAO;QACL,YAAYoB,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACApB;IACN;AACF"}
+{"version":3,"file":"agent/utils.mjs","sources":["webpack://@midscene/core/./src/agent/utils.ts"],"sourcesContent":["import type { TMultimodalPrompt, TUserPrompt } from '@/ai-model/common';\nimport type { AbstractInterface } from '@/device';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst debugProfile = getDebug('web:tool:profile');\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: { uploadServerUrl?: string },\n): Promise<UIContext> {\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debugProfile('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debugProfile('Interface description end');\n\n  debugProfile('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debugProfile('UploadTestInfoToServer end');\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  const size = await interfaceInstance.size();\n  debugProfile(`size: ${size.width}x${size.height} dpr: ${size.dpr}`);\n\n  return {\n    size,\n    screenshotBase64: screenshotBase64!,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\n/**\n * Get the current execution file name\n * @returns The name of the current execution file\n */\nexport function getCurrentExecutionFile(trace?: string): string | false {\n  const error = new Error();\n  const stackTrace = trace || error.stack;\n  const pkgDir = process.cwd() || '';\n  if (stackTrace) {\n    const stackLines = stackTrace.split('\\n');\n    for (const line of stackLines) {\n      if (\n        line.includes('.spec.') ||\n        line.includes('.test.') ||\n        line.includes('.ts') ||\n        line.includes('.js')\n      ) {\n        const match = line.match(/(?:at\\s+)?(.*?\\.(?:spec|test)\\.[jt]s)/);\n        if (match?.[1]) {\n          const targetFileName = match[1]\n            .replace(pkgDir, '')\n            .trim()\n            .replace('at ', '');\n          return targetFileName;\n        }\n      }\n    }\n  }\n  return false;\n}\n\nconst testFileIndex = new Map<string, number>();\n\nexport function generateCacheId(fileName?: string): string {\n  let taskFile = fileName || getCurrentExecutionFile();\n  if (!taskFile) {\n    taskFile = uuid();\n    console.warn(\n      'Midscene - using random UUID for cache id. Cache may be invalid.',\n    );\n  }\n\n  if (testFileIndex.has(taskFile)) {\n    const currentIndex = testFileIndex.get(taskFile);\n    if (currentIndex !== undefined) {\n      testFileIndex.set(taskFile, currentIndex + 1);\n    }\n  } else {\n    testFileIndex.set(taskFile, 1);\n  }\n  return `${taskFile}-${testFileIndex.get(taskFile)}`;\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    const centerPosition = {\n      x: Math.floor((planLocateParam.bbox[0] + planLocateParam.bbox[2]) / 2),\n      y: Math.floor((planLocateParam.bbox[1] + planLocateParam.bbox[3]) / 2),\n    };\n\n    const element = generateElementByPosition(\n      centerPosition,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n"],"names":["debugProfile","getDebug","commonContextParser","interfaceInstance","_opt","assert","description","uploadTestInfoToServer","screenshotBase64","size","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","logMsg","getCurrentExecutionFile","trace","error","Error","stackTrace","pkgDir","process","stackLines","line","match","targetFileName","testFileIndex","Map","generateCacheId","fileName","taskFile","console","currentIndex","undefined","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","_planLocateParam_prompt","centerPosition","Math","element","generateElementByPosition","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","_context_taskCache","cacheDebug","rect","getMidsceneVersion","__VERSION__","parsePrompt","prompt"],"mappings":";;;;;;;AAqBA,MAAMA,eAAeC,SAAS;AAEvB,eAAeC,oBACpBC,iBAAoC,EACpCC,IAAkC;QAKdD;IAHpBE,OAAOF,mBAAmB;IAE1BH,aAAa;IACb,MAAMM,cAAcH,AAAAA,SAAAA,CAAAA,8BAAAA,kBAAkB,QAAQ,AAAD,IAAzBA,KAAAA,IAAAA,4BAAAA,IAAAA,CAAAA,kBAAiB,KAAiB;IACtDH,aAAa;IAEbA,aAAa;IACbO,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAJ,aAAa;IAEb,MAAMQ,mBAAmB,MAAML,kBAAkB,gBAAgB;IACjEE,OAAOG,kBAAmB;IAE1B,MAAMC,OAAO,MAAMN,kBAAkB,IAAI;IACzCH,aAAa,CAAC,MAAM,EAAES,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,MAAM,EAAEA,KAAK,GAAG,EAAE;IAElE,OAAO;QACLA;QACA,kBAAkBD;IACpB;AACF;AAEO,SAASE,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7CC,OAAO,CAAC,gCAAgC,EAAED,UAAU;AACtD;AAMO,SAASE,wBAAwBC,KAAc;IACpD,MAAMC,QAAQ,IAAIC;IAClB,MAAMC,aAAaH,SAASC,MAAM,KAAK;IACvC,MAAMG,SAASC,QAAQ,GAAG,MAAM;IAChC,IAAIF,YAAY;QACd,MAAMG,aAAaH,WAAW,KAAK,CAAC;QACpC,KAAK,MAAMI,QAAQD,WACjB,IACEC,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,aACdA,KAAK,QAAQ,CAAC,UACdA,KAAK,QAAQ,CAAC,QACd;YACA,MAAMC,QAAQD,KAAK,KAAK,CAAC;YACzB,IAAIC,QAAAA,QAAAA,KAAAA,IAAAA,KAAO,CAAC,EAAE,EAAE;gBACd,MAAMC,iBAAiBD,KAAK,CAAC,EAAE,CAC5B,OAAO,CAACJ,QAAQ,IAChB,IAAI,GACJ,OAAO,CAAC,OAAO;gBAClB,OAAOK;YACT;QACF;IAEJ;IACA,OAAO;AACT;AAEA,MAAMC,gBAAgB,IAAIC;AAEnB,SAASC,gBAAgBC,QAAiB;IAC/C,IAAIC,WAAWD,YAAYd;IAC3B,IAAI,CAACe,UAAU;QACbA,WAAWnB;QACXoB,QAAQ,IAAI,CACV;IAEJ;IAEA,IAAIL,cAAc,GAAG,CAACI,WAAW;QAC/B,MAAME,eAAeN,cAAc,GAAG,CAACI;QACvC,IAAIE,AAAiBC,WAAjBD,cACFN,cAAc,GAAG,CAACI,UAAUE,eAAe;IAE/C,OACEN,cAAc,GAAG,CAACI,UAAU;IAE9B,OAAO,GAAGA,SAAS,CAAC,EAAEJ,cAAc,GAAG,CAACI,WAAW;AACrD;AAEO,SAASI,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;YAUlBG;QATN,MAAMC,iBAAiB;YACrB,GAAGC,KAAK,KAAK,CAAEL,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;YACpE,GAAGK,KAAK,KAAK,CAAEL,AAAAA,CAAAA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAC,IAAK;QACtE;QAEA,MAAMM,UAAUC,0BACdH,gBACA,AAAkC,YAAlC,OAAOJ,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBG,AAAAA,SAAAA,CAAAA,0BAAAA,gBAAgB,MAAM,AAAD,IAArBA,KAAAA,IAAAA,wBAAwB,MAAM,AAAD,KAAK;QAExC,OAAOG;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;QAWzBC;IATL,IAAI,CAACH,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBE,MAAW,iCAAiCH;IAI9C,IAAI,UAACE,CAAAA,qBAAAA,QAAQ,SAAS,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,iBAAiB,AAAD,GACtC;IAGF,IAAI,CAACJ,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDK,MACE;IAKJ,IAAI;QACF,MAAMC,OACJ,MAAMN,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACND,KAAK,KAAK,CAACU,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCV,KAAK,KAAK,CAACU,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOJ,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAG,MAAW,yBAAyBH;QACpC,OAAOL;IACT,EAAE,OAAOxB,OAAO;QACdgC,MAAW,qCAAqChC;QAChD;IACF;AACF;AAIO,MAAMkC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBrB;IACpB;IAEF,OAAO;QACL,YAAYqB,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACArB;IACN;AACF"}

package/dist/es/ai-model/inspect.mjs

@@ -59,8 +59,9 @@ async function AiLocateElement(options) {
     const { vlMode } = modelConfig;
     const { screenshotBase64 } = context;
     assert(targetElementDescription, "cannot find the target element description");
+    const targetElementDescriptionText = extraTextFromUserPrompt(targetElementDescription);
     const userInstructionPrompt = await findElementPrompt.format({
-        targetElementDescription: extraTextFromUserPrompt(targetElementDescription)
+        targetElementDescription: targetElementDescriptionText
     });
     const systemPrompt = systemPromptToLocateElement(vlMode);
     let imagePayload = screenshotBase64;
@@ -126,7 +127,7 @@ async function AiLocateElement(options) {
                 x: resRect.left + resRect.width / 2,
                 y: resRect.top + resRect.height / 2
             };
-            const element = generateElementByPosition(rectCenter);
+            const element = generateElementByPosition(rectCenter, targetElementDescriptionText);
             errors = [];
             if (element) matchedElements = [
                 element