@midscene/core 0.30.6-beta-20251022093704.0 → 1.0.1-beta-20251022061922.0

package/dist/es/agent/task-builder.mjs

@@ -0,0 +1,315 @@
+import { findAllMidsceneLocatorField, parseActionParam } from "../ai-model/index.mjs";
+import { InsightError } from "../types.mjs";
+import { sleep } from "../utils.mjs";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { matchElementFromCache, matchElementFromPlan } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('agent:task-builder');
+function locatePlanForLocate(param) {
+    const locate = 'string' == typeof param ? {
+        prompt: param
+    } : param;
+    const locatePlan = {
+        type: 'Locate',
+        locate,
+        param: locate,
+        thought: ''
+    };
+    return locatePlan;
+}
+class TaskBuilder {
+    async build(plans, modelConfig, options) {
+        const tasks = [];
+        const cacheable = null == options ? void 0 : options.cacheable;
+        const context = {
+            tasks,
+            modelConfig,
+            cacheable,
+            subTask: !!(null == options ? void 0 : options.subTask)
+        };
+        const planHandlers = new Map([
+            [
+                'Locate',
+                (plan)=>this.handleLocatePlan(plan, context)
+            ],
+            [
+                'Finished',
+                (plan)=>this.handleFinishedPlan(plan, context)
+            ],
+            [
+                'Sleep',
+                (plan)=>this.handleSleepPlan(plan, context)
+            ]
+        ]);
+        const defaultHandler = (plan)=>this.handleActionPlan(plan, context);
+        for (const plan of plans){
+            const handler = planHandlers.get(plan.type) ?? defaultHandler;
+            await handler(plan);
+        }
+        return {
+            tasks
+        };
+    }
+    handleFinishedPlan(plan, context) {
+        const taskActionFinished = {
+            type: 'Action',
+            subType: 'Finished',
+            param: null,
+            thought: plan.thought,
+            locate: plan.locate,
+            subTask: context.subTask || void 0,
+            executor: async ()=>{}
+        };
+        context.tasks.push(taskActionFinished);
+    }
+    handleSleepPlan(plan, context) {
+        const sleepTask = this.createSleepTask(plan.param, {
+            thought: plan.thought,
+            locate: plan.locate
+        });
+        if (context.subTask) sleepTask.subTask = true;
+        context.tasks.push(sleepTask);
+    }
+    createSleepTask(param, meta) {
+        return {
+            type: 'Action',
+            subType: 'Sleep',
+            param,
+            thought: null == meta ? void 0 : meta.thought,
+            locate: (null == meta ? void 0 : meta.locate) ?? null,
+            executor: async (taskParam)=>{
+                await sleep((null == taskParam ? void 0 : taskParam.timeMs) || 3000);
+            }
+        };
+    }
+    async handleLocatePlan(plan, context) {
+        if (!plan.locate || null === plan.locate) return void debug('Locate action with id is null, will be ignored', plan);
+        const taskLocate = this.createLocateTask(plan, plan.locate, context);
+        context.tasks.push(taskLocate);
+    }
+    async handleActionPlan(plan, context) {
+        const planType = plan.type;
+        const actionSpace = await this.interface.actionSpace();
+        const action = actionSpace.find((item)=>item.name === planType);
+        const param = plan.param;
+        if (!action) throw new Error(`Action type '${planType}' not found`);
+        const locateFields = action ? findAllMidsceneLocatorField(action.paramSchema) : [];
+        const requiredLocateFields = action ? findAllMidsceneLocatorField(action.paramSchema, true) : [];
+        locateFields.forEach((field)=>{
+            if (param[field]) {
+                const locatePlan = locatePlanForLocate(param[field]);
+                debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`);
+                const locateTask = this.createLocateTask(locatePlan, param[field], context, (result)=>{
+                    param[field] = result;
+                });
+                context.tasks.push(locateTask);
+            } else {
+                assert(!requiredLocateFields.includes(field), `Required locate field '${field}' is not provided for action ${planType}`);
+                debug(`field '${field}' is not provided for action ${planType}`);
+            }
+        });
+        const task = {
+            type: 'Action',
+            subType: planType,
+            thought: plan.thought,
+            param: plan.param,
+            subTask: context.subTask || void 0,
+            executor: async (param, taskContext)=>{
+                var _taskContext_element;
+                debug('executing action', planType, param, `taskContext.element.center: ${null == (_taskContext_element = taskContext.element) ? void 0 : _taskContext_element.center}`);
+                const uiContext = taskContext.uiContext;
+                assert(uiContext, 'uiContext is required for Action task');
+                requiredLocateFields.forEach((field)=>{
+                    assert(param[field], `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`);
+                });
+                try {
+                    await Promise.all([
+                        (async ()=>{
+                            if (this.interface.beforeInvokeAction) {
+                                debug('will call "beforeInvokeAction" for interface');
+                                await this.interface.beforeInvokeAction(action.name, param);
+                                debug('called "beforeInvokeAction" for interface');
+                            }
+                        })(),
+                        sleep(200)
+                    ]);
+                } catch (originalError) {
+                    const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
+                    throw new Error(`error in running beforeInvokeAction for ${action.name}: ${originalMessage}`, {
+                        cause: originalError
+                    });
+                }
+                if (action.paramSchema) try {
+                    param = parseActionParam(param, action.paramSchema);
+                } catch (error) {
+                    throw new Error(`Invalid parameters for action ${action.name}: ${error.message}\nParameters: ${JSON.stringify(param)}`, {
+                        cause: error
+                    });
+                }
+                debug('calling action', action.name);
+                const actionFn = action.call.bind(this.interface);
+                await actionFn(param, taskContext);
+                debug('called action', action.name);
+                try {
+                    if (this.interface.afterInvokeAction) {
+                        debug('will call "afterInvokeAction" for interface');
+                        await this.interface.afterInvokeAction(action.name, param);
+                        debug('called "afterInvokeAction" for interface');
+                    }
+                } catch (originalError) {
+                    const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
+                    throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalMessage}`, {
+                        cause: originalError
+                    });
+                }
+                return {
+                    output: {
+                        success: true,
+                        action: planType,
+                        param: param
+                    }
+                };
+            }
+        };
+        context.tasks.push(task);
+    }
+    createLocateTask(plan, detailedLocateParam, context, onResult) {
+        const { cacheable, modelConfig } = context;
+        let locateParam = detailedLocateParam;
+        if ('string' == typeof locateParam) locateParam = {
+            prompt: locateParam
+        };
+        if (void 0 !== cacheable) locateParam = {
+            ...locateParam,
+            cacheable
+        };
+        const taskFind = {
+            type: 'Insight',
+            subType: 'Locate',
+            subTask: context.subTask || void 0,
+            param: locateParam,
+            thought: plan.thought,
+            executor: async (param, taskContext)=>{
+                var _this_taskCache, _locateCacheRecord_cacheContent;
+                const { task } = taskContext;
+                let { uiContext } = taskContext;
+                assert((null == param ? void 0 : param.prompt) || (null == param ? void 0 : param.id) || (null == param ? void 0 : param.bbox), `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
+                if (!uiContext) uiContext = await this.insight.contextRetrieverFn();
+                assert(uiContext, 'uiContext is required for Insight task');
+                let locateDump;
+                let locateResult;
+                const applyDump = (dump)=>{
+                    var _dump_taskInfo, _dump_taskInfo1;
+                    if (!dump) return;
+                    locateDump = dump;
+                    task.log = {
+                        dump
+                    };
+                    task.usage = null == (_dump_taskInfo = dump.taskInfo) ? void 0 : _dump_taskInfo.usage;
+                    if (null == (_dump_taskInfo1 = dump.taskInfo) ? void 0 : _dump_taskInfo1.searchAreaUsage) task.searchAreaUsage = dump.taskInfo.searchAreaUsage;
+                };
+                let elementFromXpath;
+                if (param.xpath && this.interface.rectMatchesCacheFeature) elementFromXpath = await this.interface.rectMatchesCacheFeature({
+                    xpaths: [
+                        param.xpath
+                    ]
+                });
+                const userExpectedPathHitFlag = !!elementFromXpath;
+                const cachePrompt = param.prompt;
+                const locateCacheRecord = null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchLocateCache(cachePrompt);
+                const cacheEntry = null == locateCacheRecord ? void 0 : null == (_locateCacheRecord_cacheContent = locateCacheRecord.cacheContent) ? void 0 : _locateCacheRecord_cacheContent.cache;
+                const elementFromCache = userExpectedPathHitFlag ? null : await matchElementFromCache({
+                    taskCache: this.taskCache,
+                    interfaceInstance: this.interface
+                }, cacheEntry, cachePrompt, param.cacheable);
+                const cacheHitFlag = !!elementFromCache;
+                const elementFromPlan = userExpectedPathHitFlag || cacheHitFlag ? void 0 : matchElementFromPlan(param, uiContext.tree);
+                const planHitFlag = !!elementFromPlan;
+                let elementFromAiLocate;
+                if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) try {
+                    locateResult = await this.insight.locate(param, {
+                        context: uiContext
+                    }, modelConfig);
+                    applyDump(locateResult.dump);
+                    elementFromAiLocate = locateResult.element;
+                } catch (error) {
+                    if (error instanceof InsightError) applyDump(error.dump);
+                    throw error;
+                }
+                const element = elementFromXpath || elementFromCache || elementFromPlan || elementFromAiLocate;
+                let currentCacheEntry;
+                if (element && this.taskCache && !cacheHitFlag && (null == param ? void 0 : param.cacheable) !== false) if (this.interface.cacheFeatureForRect) try {
+                    const feature = await this.interface.cacheFeatureForRect(element.rect, void 0 !== element.isOrderSensitive ? {
+                        _orderSensitive: element.isOrderSensitive
+                    } : void 0);
+                    if (feature && Object.keys(feature).length > 0) {
+                        debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
+                        currentCacheEntry = feature;
+                        this.taskCache.updateOrAppendCacheRecord({
+                            type: 'locate',
+                            prompt: cachePrompt,
+                            cache: feature
+                        }, locateCacheRecord);
+                    } else debug('no cache data returned, skip cache update, prompt: %s', cachePrompt);
+                } catch (error) {
+                    debug('cacheFeatureForRect failed: %s', error);
+                }
+                else debug('cacheFeatureForRect is not supported, skip cache update');
+                if (!element) {
+                    if (locateDump) throw new InsightError(`Element not found: ${param.prompt}`, locateDump);
+                    throw new Error(`Element not found: ${param.prompt}`);
+                }
+                let hitBy;
+                if (userExpectedPathHitFlag) hitBy = {
+                    from: 'User expected path',
+                    context: {
+                        xpath: param.xpath
+                    }
+                };
+                else if (cacheHitFlag) hitBy = {
+                    from: 'Cache',
+                    context: {
+                        cacheEntry,
+                        cacheToSave: currentCacheEntry
+                    }
+                };
+                else if (planHitFlag) hitBy = {
+                    from: 'Planning',
+                    context: {
+                        id: null == elementFromPlan ? void 0 : elementFromPlan.id,
+                        bbox: null == elementFromPlan ? void 0 : elementFromPlan.bbox
+                    }
+                };
+                null == onResult || onResult(element);
+                return {
+                    output: {
+                        element
+                    },
+                    hitBy
+                };
+            }
+        };
+        return taskFind;
+    }
+    constructor({ interfaceInstance, insight, taskCache }){
+        _define_property(this, "interface", void 0);
+        _define_property(this, "insight", void 0);
+        _define_property(this, "taskCache", void 0);
+        this.interface = interfaceInstance;
+        this.insight = insight;
+        this.taskCache = taskCache;
+    }
+}
+export { TaskBuilder, locatePlanForLocate };
+
+//# sourceMappingURL=task-builder.mjs.map

package/dist/es/agent/task-builder.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/task-builder.mjs","sources":["webpack://@midscene/core/./src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Insight from '@/insight';\nimport type {\n  DetailedLocateParam,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskInsightLocateApply,\n  InsightDump,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningActionParamSleep,\n  PlanningLocateParam,\n  Rect,\n} from '@/types';\nimport { InsightError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { matchElementFromCache, matchElementFromPlan } from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    locate,\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  insight: Insight;\n  taskCache?: TaskCache;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  subTask?: boolean;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfig: IModelConfig;\n  cacheable?: boolean;\n  subTask: boolean;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly insight: Insight;\n\n  private readonly taskCache?: TaskCache;\n\n  constructor({ interfaceInstance, insight, taskCache }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.insight = insight;\n    this.taskCache = taskCache;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfig,\n      cacheable,\n      subTask: !!options?.subTask,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n      [\n        'Sleep',\n        (plan) =>\n          this.handleSleepPlan(\n            plan as PlanningAction<PlanningActionParamSleep>,\n            context,\n          ),\n      ],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      locate: plan.locate,\n      subTask: context.subTask || undefined,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private handleSleepPlan(\n    plan: PlanningAction<PlanningActionParamSleep>,\n    context: PlanBuildContext,\n  ): void {\n    const sleepTask = this.createSleepTask(plan.param, {\n      thought: plan.thought,\n      locate: plan.locate,\n    });\n    if (context.subTask) {\n      sleepTask.subTask = true;\n    }\n    context.tasks.push(sleepTask);\n  }\n\n  public createSleepTask(\n    param: PlanningActionParamSleep,\n    meta?: { thought?: string; locate?: PlanningAction['locate'] | null },\n  ): ExecutionTaskActionApply<PlanningActionParamSleep> {\n    return {\n      type: 'Action',\n      subType: 'Sleep',\n      param,\n      thought: meta?.thought,\n      locate: meta?.locate ?? null,\n      executor: async (taskParam) => {\n        await sleep(taskParam?.timeMs || 3000);\n      },\n    };\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    if (!plan.locate || plan.locate === null) {\n      debug('Locate action with id is null, will be ignored', plan);\n      return;\n    }\n\n    const taskLocate = this.createLocateTask(plan, plan.locate, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = await this.interface.actionSpace();\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      subTask: context.subTask || undefined,\n      executor: async (param, taskContext) => {\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug('will call \"beforeInvokeAction\" for interface');\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug('called \"beforeInvokeAction\" for interface');\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema);\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        await actionFn(param, taskContext);\n        debug('called action', action.name);\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug('will call \"afterInvokeAction\" for interface');\n            await this.interface.afterInvokeAction(action.name, param);\n            debug('called \"afterInvokeAction\" for interface');\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        return {\n          output: {\n            success: true,\n            action: planType,\n            param: param,\n          },\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskInsightLocateApply {\n    const { cacheable, modelConfig } = context;\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    const taskFind: ExecutionTaskInsightLocateApply = {\n      type: 'Insight',\n      subType: 'Locate',\n      subTask: context.subTask || undefined,\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.id || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.insight.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Insight task');\n\n        let locateDump: InsightDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: InsightDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n        };\n\n        // from xpath\n        let elementFromXpath: Rect | undefined;\n        if (param.xpath && this.interface.rectMatchesCacheFeature) {\n          elementFromXpath = await this.interface.rectMatchesCacheFeature({\n            xpaths: [param.xpath],\n          });\n        }\n        const userExpectedPathHitFlag = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCache = userExpectedPathHitFlag\n          ? null\n          : await matchElementFromCache(\n              {\n                taskCache: this.taskCache,\n                interfaceInstance: this.interface,\n              },\n              cacheEntry,\n              cachePrompt,\n              param.cacheable,\n            );\n        const cacheHitFlag = !!elementFromCache;\n\n        const elementFromPlan =\n          !userExpectedPathHitFlag && !cacheHitFlag\n            ? matchElementFromPlan(param, uiContext.tree)\n            : undefined;\n        const planHitFlag = !!elementFromPlan;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) {\n          try {\n            locateResult = await this.insight.locate(\n              param,\n              {\n                context: uiContext,\n              },\n              modelConfig,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof InsightError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          }\n        }\n\n        const element =\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromPlan ||\n          elementFromAiLocate;\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        if (\n          element &&\n          this.taskCache &&\n          !cacheHitFlag &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForRect) {\n            try {\n              const feature = await this.interface.cacheFeatureForRect(\n                element.rect,\n                element.isOrderSensitive !== undefined\n                  ? { _orderSensitive: element.isOrderSensitive }\n                  : undefined,\n              );\n              if (feature && Object.keys(feature).length > 0) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForRect failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForRect is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new InsightError(\n              `Element not found: ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (userExpectedPathHitFlag) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (cacheHitFlag) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        } else if (planHitFlag) {\n          hitBy = {\n            from: 'Planning',\n            context: {\n              id: elementFromPlan?.id,\n              bbox: elementFromPlan?.bbox,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element,\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskFind;\n  }\n}\n"],"names":["debug","getDebug","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfig","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","undefined","sleepTask","meta","taskParam","sleep","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","locateTask","result","assert","task","taskContext","_taskContext_element","uiContext","Promise","originalError","originalMessage","String","parseActionParam","error","actionFn","detailedLocateParam","onResult","locateParam","taskFind","_this_taskCache","_locateCacheRecord_cacheContent","locateDump","locateResult","applyDump","dump","_dump_taskInfo","_dump_taskInfo1","elementFromXpath","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","cacheEntry","elementFromCache","matchElementFromCache","cacheHitFlag","elementFromPlan","matchElementFromPlan","planHitFlag","elementFromAiLocate","InsightError","element","currentCacheEntry","feature","Object","hitBy","interfaceInstance","insight","taskCache"],"mappings":";;;;;;;;;;;;;;;;AA0BA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACND;QACA,OAAOA;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAoBO,MAAMC;IAaX,MAAa,MACXC,KAAuB,EACvBC,WAAyB,EACzBC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,SAAS;QAEpC,MAAMG,UAA4B;YAChCF;YACAF;YACAG;YACA,SAAS,CAAC,CAACF,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;QAC5B;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;YAC9D;gBACE;gBACA,CAACG,OACC,IAAI,CAAC,eAAe,CAClBA,MACAH;aAEL;SACF;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQR,MAAO;YACxB,MAAMU,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,QAAQA,KAAK,MAAM;YACnB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,WAAa;QACzB;QACAP,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEQ,gBACNH,IAA8C,EAC9CH,OAAyB,EACnB;QACN,MAAMQ,YAAY,IAAI,CAAC,eAAe,CAACL,KAAK,KAAK,EAAE;YACjD,SAASA,KAAK,OAAO;YACrB,QAAQA,KAAK,MAAM;QACrB;QACA,IAAIH,QAAQ,OAAO,EACjBQ,UAAU,OAAO,GAAG;QAEtBR,QAAQ,KAAK,CAAC,IAAI,CAACQ;IACrB;IAEO,gBACLjB,KAA+B,EAC/BkB,IAAqE,EACjB;QACpD,OAAO;YACL,MAAM;YACN,SAAS;YACTlB;YACA,SAASkB,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO;YACtB,QAAQA,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,MAAM,AAAD,KAAK;YACxB,UAAU,OAAOC;gBACf,MAAMC,MAAMD,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;YACnC;QACF;IACF;IAEA,MAAc,iBACZP,IAAyC,EACzCH,OAAyB,EACV;QACf,IAAI,CAACG,KAAK,MAAM,IAAIA,AAAgB,SAAhBA,KAAK,MAAM,EAAW,YACxCf,MAAM,kDAAkDe;QAI1D,MAAMS,aAAa,IAAI,CAAC,gBAAgB,CAACT,MAAMA,KAAK,MAAM,EAAEH;QAC5DA,QAAQ,KAAK,CAAC,IAAI,CAACY;IACrB;IAEA,MAAc,iBACZT,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMa,WAAWV,KAAK,IAAI;QAC1B,MAAMW,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;QACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMtB,QAAQY,KAAK,KAAK;QAExB,IAAI,CAACY,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EAAE;gBAChB,MAAM5B,aAAaH,oBAAoBC,KAAK,CAAC8B,MAAM;gBACnDjC,MACE,uCACA,CAAC,YAAY,EAAEyB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC/B,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAAC7B,aAAa;gBAE5C,MAAM8B,aAAa,IAAI,CAAC,gBAAgB,CACtC9B,YACAF,KAAK,CAAC8B,MAAM,EACZrB,SACA,CAACwB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFxB,QAAQ,KAAK,CAAC,IAAI,CAACuB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3EzB,MAAM,CAAC,OAAO,EAAEiC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMa,OAKF;YACF,MAAM;YACN,SAASb;YACT,SAASV,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,OAAOhB,OAAOoC;oBAKWC;gBAJjCxC,MACE,oBACAyB,UACAtB,OACA,CAAC,4BAA4B,EAAE,QAAAqC,CAAAA,uBAAAA,YAAY,OAAO,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,MAAM,EAAE;gBAG9D,MAAMC,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACElC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEA,IAAI;oBACF,MAAMiB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC1C,MAAM;gCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC2B,OAAO,IAAI,EAAExB;gCACrDH,MAAM;4BACR;wBACF;wBACAuB,MAAM;qBACP;gBACH,EAAE,OAAOoB,eAAoB;oBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;oBACnC,MAAM,IAAId,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEiB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,IAAIhB,OAAO,WAAW,EACpB,IAAI;oBACFxB,QAAQ2C,iBAAiB3C,OAAOwB,OAAO,WAAW;gBACpD,EAAE,OAAOoB,OAAY;oBACnB,MAAM,IAAIlB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,MAAM,OAAO,CAAC,cAAc,EAAEb,KAAK,SAAS,CAAC/B,QAAQ,EACtG;wBAAE,OAAO4C;oBAAM;gBAEnB;gBAGF/C,MAAM,kBAAkB2B,OAAO,IAAI;gBACnC,MAAMqB,WAAWrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAMqB,SAAS7C,OAAOoC;gBACtBvC,MAAM,iBAAiB2B,OAAO,IAAI;gBAElC,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpC3B,MAAM;wBACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC2B,OAAO,IAAI,EAAExB;wBACpDH,MAAM;oBACR;gBACF,EAAE,OAAO2C,eAAoB;oBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;oBACnC,MAAM,IAAId,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEiB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,OAAO;oBACL,QAAQ;wBACN,SAAS;wBACT,QAAQlB;wBACR,OAAOtB;oBACT;gBACF;YACF;QACF;QAEAS,QAAQ,KAAK,CAAC,IAAI,CAAC0B;IACrB;IAEQ,iBACNvB,IAAyC,EACzCkC,mBAAiD,EACjDrC,OAAyB,EACzBsC,QAAgD,EACf;QACjC,MAAM,EAAEvC,SAAS,EAAEH,WAAW,EAAE,GAAGI;QACnC,IAAIuC,cAAcF;QAElB,IAAI,AAAuB,YAAvB,OAAOE,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAIxC,AAAcQ,WAAdR,WACFwC,cAAc;YACZ,GAAGA,WAAW;YACdxC;QACF;QAGF,MAAMyC,WAA4C;YAChD,MAAM;YACN,SAAS;YACT,SAASxC,QAAQ,OAAO,IAAIO;YAC5B,OAAOgC;YACP,SAASpC,KAAK,OAAO;YACrB,UAAU,OAAOZ,OAAOoC;oBA4CIc,iBACPC;gBA5CnB,MAAM,EAAEhB,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACElC,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,EAAE,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GACxC,CAAC,qDAAqD,EAAE+B,KAAK,SAAS,CACpE/B,QACC;gBAGL,IAAI,CAACsC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,IAAIc;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;wBAQJC,gBACTC;oBARJ,IAAI,CAACF,MACH;oBAEFH,aAAaG;oBACbpB,KAAK,GAAG,GAAG;wBACToB;oBACF;oBACApB,KAAK,KAAK,GAAG,QAAAqB,CAAAA,iBAAAA,KAAK,QAAQ,AAAD,IAAZA,KAAAA,IAAAA,eAAe,KAAK;oBACjC,IAAI,QAAAC,CAAAA,kBAAAA,KAAK,QAAQ,AAAD,IAAZA,KAAAA,IAAAA,gBAAe,eAAe,EAChCtB,KAAK,eAAe,GAAGoB,KAAK,QAAQ,CAAC,eAAe;gBAExD;gBAGA,IAAIG;gBACJ,IAAI1D,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,uBAAuB,EACvD0D,mBAAmB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;oBAC9D,QAAQ;wBAAC1D,MAAM,KAAK;qBAAC;gBACvB;gBAEF,MAAM2D,0BAA0B,CAAC,CAACD;gBAElC,MAAME,cAAc5D,MAAM,MAAM;gBAChC,MAAM6D,oBAAoB,QAAAX,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CAACU;gBAC3D,MAAME,aAAaX,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,KAAK;gBAEzD,MAAMY,mBAAmBJ,0BACrB,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACA5D,MAAM,SAAS;gBAErB,MAAMiE,eAAe,CAAC,CAACF;gBAEvB,MAAMG,kBACJ,AAACP,2BAA4BM,eAEzBjD,SADAmD,qBAAqBnE,OAAOsC,UAAU,IAAI;gBAEhD,MAAM8B,cAAc,CAAC,CAACF;gBAEtB,IAAIG;gBACJ,IAAI,CAACV,2BAA2B,CAACM,gBAAgB,CAACG,aAChD,IAAI;oBACFf,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCrD,OACA;wBACE,SAASsC;oBACX,GACAjC;oBAEFiD,UAAUD,aAAa,IAAI;oBAC3BgB,sBAAsBhB,aAAa,OAAO;gBAC5C,EAAE,OAAOT,OAAO;oBACd,IAAIA,iBAAiB0B,cACnBhB,UAAUV,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAGF,MAAM2B,UACJb,oBACAK,oBACAG,mBACAG;gBAEF,IAAIG;gBACJ,IACED,WACA,IAAI,CAAC,SAAS,IACd,CAACN,gBACDjE,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EACpC,IAAI;oBACF,MAAMyE,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CACtDF,QAAQ,IAAI,EACZA,AAA6BvD,WAA7BuD,QAAQ,gBAAgB,GACpB;wBAAE,iBAAiBA,QAAQ,gBAAgB;oBAAC,IAC5CvD;oBAEN,IAAIyD,WAAWC,OAAO,IAAI,CAACD,SAAS,MAAM,GAAG,GAAG;wBAC9C5E,MACE,uCACA+D,aACAa;wBAEFD,oBAAoBC;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQb;4BACR,OAAOa;wBACT,GACAZ;oBAEJ,OACEhE,MACE,yDACA+D;gBAGN,EAAE,OAAOhB,OAAO;oBACd/C,MAAM,kCAAkC+C;gBAC1C;qBAEA/C,MAAM;gBAIV,IAAI,CAAC0E,SAAS;oBACZ,IAAInB,YACF,MAAM,IAAIkB,aACR,CAAC,mBAAmB,EAAEtE,MAAM,MAAM,EAAE,EACpCoD;oBAGJ,MAAM,IAAI1B,MAAM,CAAC,mBAAmB,EAAE1B,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAI2E;gBAEJ,IAAIhB,yBACFgB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO3E,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIiE,cACTU,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPb;wBACA,aAAaU;oBACf;gBACF;qBACK,IAAIJ,aACTO,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,IAAIT,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,EAAE;wBACvB,MAAMA,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,IAAI;oBAC7B;gBACF;gBAGFnB,QAAAA,YAAAA,SAAWwB;gBAEX,OAAO;oBACL,QAAQ;wBACNA;oBACF;oBACAI;gBACF;YACF;QACF;QAEA,OAAO1B;IACT;IArdA,YAAY,EAAE2B,iBAAiB,EAAEC,OAAO,EAAEC,SAAS,EAAmB,CAAE;QANxE,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAGE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;IACnB;AAkdF"}