@midscene/core 0.30.5 → 1.0.1-beta-20251021060907.0

package/dist/es/agent/task-builder.mjs

@@ -0,0 +1,303 @@
+import { InsightError } from "../types.mjs";
+import { sleep } from "../utils.mjs";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { findAllMidsceneLocatorField, parseActionParam } from "../ai-model/index.mjs";
+import { matchElementFromCache, matchElementFromPlan } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('agent:task-builder');
+function locatePlanForLocate(param) {
+    const locate = 'string' == typeof param ? {
+        prompt: param
+    } : param;
+    const locatePlan = {
+        type: 'Locate',
+        locate,
+        param: locate,
+        thought: ''
+    };
+    return locatePlan;
+}
+class TaskBuilder {
+    async build(plans, modelConfig, options) {
+        const tasks = [];
+        const cacheable = null == options ? void 0 : options.cacheable;
+        const taskForLocatePlan = (plan, detailedLocateParam, onResult)=>{
+            if ('string' == typeof detailedLocateParam) detailedLocateParam = {
+                prompt: detailedLocateParam
+            };
+            if (void 0 !== cacheable) detailedLocateParam = {
+                ...detailedLocateParam,
+                cacheable
+            };
+            const taskFind = {
+                type: 'Insight',
+                subType: 'Locate',
+                param: detailedLocateParam,
+                thought: plan.thought,
+                executor: async (param, taskContext)=>{
+                    var _this_taskCache, _locateCacheRecord_cacheContent;
+                    const { task } = taskContext;
+                    let { uiContext } = taskContext;
+                    assert((null == param ? void 0 : param.prompt) || (null == param ? void 0 : param.id) || (null == param ? void 0 : param.bbox), `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
+                    if (!uiContext) uiContext = await this.insight.contextRetrieverFn();
+                    assert(uiContext, 'uiContext is required for Insight task');
+                    let locateDump;
+                    let locateResult;
+                    const applyDump = (dump)=>{
+                        var _dump_taskInfo, _dump_taskInfo1;
+                        if (!dump) return;
+                        locateDump = dump;
+                        task.log = {
+                            dump
+                        };
+                        task.usage = null == (_dump_taskInfo = dump.taskInfo) ? void 0 : _dump_taskInfo.usage;
+                        if (null == (_dump_taskInfo1 = dump.taskInfo) ? void 0 : _dump_taskInfo1.searchAreaUsage) task.searchAreaUsage = dump.taskInfo.searchAreaUsage;
+                    };
+                    const elementFromXpath = param.xpath && this.interface.getElementInfoByXpath ? await this.interface.getElementInfoByXpath(param.xpath) : void 0;
+                    const userExpectedPathHitFlag = !!elementFromXpath;
+                    const cachePrompt = param.prompt;
+                    const locateCacheRecord = null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchLocateCache(cachePrompt);
+                    const cacheEntry = null == locateCacheRecord ? void 0 : null == (_locateCacheRecord_cacheContent = locateCacheRecord.cacheContent) ? void 0 : _locateCacheRecord_cacheContent.cache;
+                    const elementFromCache = userExpectedPathHitFlag ? null : await matchElementFromCache({
+                        taskCache: this.taskCache,
+                        interfaceInstance: this.interface
+                    }, cacheEntry, cachePrompt, param.cacheable);
+                    const cacheHitFlag = !!elementFromCache;
+                    const elementFromPlan = userExpectedPathHitFlag || cacheHitFlag ? void 0 : matchElementFromPlan(param, uiContext.tree);
+                    const planHitFlag = !!elementFromPlan;
+                    let elementFromAiLocate;
+                    if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) try {
+                        locateResult = await this.insight.locate(param, {
+                            context: uiContext
+                        }, modelConfig);
+                        applyDump(locateResult.dump);
+                        elementFromAiLocate = locateResult.element;
+                    } catch (error) {
+                        if (error instanceof InsightError) applyDump(error.dump);
+                        throw error;
+                    }
+                    const aiLocateHitFlag = !!elementFromAiLocate;
+                    const element = elementFromXpath || elementFromCache || elementFromPlan || elementFromAiLocate;
+                    let currentCacheEntry;
+                    if (element && this.taskCache && !cacheHitFlag && (null == param ? void 0 : param.cacheable) !== false) if (this.interface.cacheFeatureForRect) try {
+                        const feature = await this.interface.cacheFeatureForRect(element.rect, void 0 !== element.isOrderSensitive ? {
+                            _orderSensitive: element.isOrderSensitive
+                        } : void 0);
+                        if (feature && Object.keys(feature).length > 0) {
+                            debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
+                            currentCacheEntry = feature;
+                            this.taskCache.updateOrAppendCacheRecord({
+                                type: 'locate',
+                                prompt: cachePrompt,
+                                cache: feature
+                            }, locateCacheRecord);
+                        } else debug('no cache data returned, skip cache update, prompt: %s', cachePrompt);
+                    } catch (error) {
+                        debug('cacheFeatureForRect failed: %s', error);
+                    }
+                    else debug('cacheFeatureForRect is not supported, skip cache update');
+                    if (!element) {
+                        if (locateDump) throw new InsightError(`Element not found: ${param.prompt}`, locateDump);
+                        throw new Error(`Element not found: ${param.prompt}`);
+                    }
+                    let hitBy;
+                    if (userExpectedPathHitFlag) hitBy = {
+                        from: 'User expected path',
+                        context: {
+                            xpath: param.xpath
+                        }
+                    };
+                    else if (cacheHitFlag) hitBy = {
+                        from: 'Cache',
+                        context: {
+                            cacheEntry,
+                            cacheToSave: currentCacheEntry
+                        }
+                    };
+                    else if (planHitFlag) hitBy = {
+                        from: 'Planning',
+                        context: {
+                            id: null == elementFromPlan ? void 0 : elementFromPlan.id,
+                            bbox: null == elementFromPlan ? void 0 : elementFromPlan.bbox
+                        }
+                    };
+                    else if (aiLocateHitFlag) hitBy = {
+                        from: 'AI model',
+                        context: {
+                            prompt: param.prompt
+                        }
+                    };
+                    null == onResult || onResult(element);
+                    return {
+                        output: {
+                            element
+                        },
+                        uiContext,
+                        hitBy
+                    };
+                }
+            };
+            return taskFind;
+        };
+        for (const plan of plans){
+            if ('Locate' === plan.type) {
+                if (!plan.locate || null === plan.locate) {
+                    debug('Locate action with id is null, will be ignored', plan);
+                    continue;
+                }
+                const taskLocate = taskForLocatePlan(plan, plan.locate);
+                tasks.push(taskLocate);
+                continue;
+            }
+            if ('Error' === plan.type) {
+                var _plan_param;
+                const taskActionError = {
+                    type: 'Action',
+                    subType: 'Error',
+                    param: plan.param,
+                    thought: plan.thought || (null == (_plan_param = plan.param) ? void 0 : _plan_param.thought),
+                    locate: plan.locate,
+                    executor: async ()=>{
+                        var _plan_param;
+                        throw new Error((null == plan ? void 0 : plan.thought) || (null == (_plan_param = plan.param) ? void 0 : _plan_param.thought) || 'error without thought');
+                    }
+                };
+                tasks.push(taskActionError);
+                continue;
+            }
+            if ('Finished' === plan.type) {
+                const taskActionFinished = {
+                    type: 'Action',
+                    subType: 'Finished',
+                    param: null,
+                    thought: plan.thought,
+                    locate: plan.locate,
+                    executor: async ()=>{}
+                };
+                tasks.push(taskActionFinished);
+                continue;
+            }
+            if ('Sleep' === plan.type) {
+                const taskActionSleep = {
+                    type: 'Action',
+                    subType: 'Sleep',
+                    param: plan.param,
+                    thought: plan.thought,
+                    locate: plan.locate,
+                    executor: async (taskParam)=>{
+                        await sleep((null == taskParam ? void 0 : taskParam.timeMs) || 3000);
+                    }
+                };
+                tasks.push(taskActionSleep);
+                continue;
+            }
+            const planType = plan.type;
+            const actionSpace = await this.interface.actionSpace();
+            const action = actionSpace.find((item)=>item.name === planType);
+            const param = plan.param;
+            if (!action) throw new Error(`Action type '${planType}' not found`);
+            const locateFields = action ? findAllMidsceneLocatorField(action.paramSchema) : [];
+            const requiredLocateFields = action ? findAllMidsceneLocatorField(action.paramSchema, true) : [];
+            locateFields.forEach((field)=>{
+                if (param[field]) {
+                    const locatePlan = locatePlanForLocate(param[field]);
+                    debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`);
+                    const locateTask = taskForLocatePlan(locatePlan, param[field], (result)=>{
+                        param[field] = result;
+                    });
+                    tasks.push(locateTask);
+                } else {
+                    assert(!requiredLocateFields.includes(field), `Required locate field '${field}' is not provided for action ${planType}`);
+                    debug(`field '${field}' is not provided for action ${planType}`);
+                }
+            });
+            const task = {
+                type: 'Action',
+                subType: planType,
+                thought: plan.thought,
+                param: plan.param,
+                executor: async (param, taskContext)=>{
+                    var _taskContext_element;
+                    debug('executing action', planType, param, `taskContext.element.center: ${null == (_taskContext_element = taskContext.element) ? void 0 : _taskContext_element.center}`);
+                    const uiContext = taskContext.uiContext;
+                    assert(uiContext, 'uiContext is required for Action task');
+                    requiredLocateFields.forEach((field)=>{
+                        assert(param[field], `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`);
+                    });
+                    try {
+                        await Promise.all([
+                            (async ()=>{
+                                if (this.interface.beforeInvokeAction) {
+                                    debug('will call "beforeInvokeAction" for interface');
+                                    await this.interface.beforeInvokeAction(action.name, param);
+                                    debug('called "beforeInvokeAction" for interface');
+                                }
+                            })(),
+                            sleep(200)
+                        ]);
+                    } catch (originalError) {
+                        const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
+                        throw new Error(`error in running beforeInvokeAction for ${action.name}: ${originalMessage}`, {
+                            cause: originalError
+                        });
+                    }
+                    if (action.paramSchema) try {
+                        param = parseActionParam(param, action.paramSchema);
+                    } catch (error) {
+                        throw new Error(`Invalid parameters for action ${action.name}: ${error.message}\nParameters: ${JSON.stringify(param)}`, {
+                            cause: error
+                        });
+                    }
+                    debug('calling action', action.name);
+                    const actionFn = action.call.bind(this.interface);
+                    await actionFn(param, taskContext);
+                    debug('called action', action.name);
+                    try {
+                        if (this.interface.afterInvokeAction) {
+                            debug('will call "afterInvokeAction" for interface');
+                            await this.interface.afterInvokeAction(action.name, param);
+                            debug('called "afterInvokeAction" for interface');
+                        }
+                    } catch (originalError) {
+                        const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
+                        throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalMessage}`, {
+                            cause: originalError
+                        });
+                    }
+                    return {
+                        output: {
+                            success: true,
+                            action: planType,
+                            param: param
+                        }
+                    };
+                }
+            };
+            tasks.push(task);
+        }
+        return {
+            tasks
+        };
+    }
+    constructor({ interfaceInstance, insight, taskCache }){
+        _define_property(this, "interface", void 0);
+        _define_property(this, "insight", void 0);
+        _define_property(this, "taskCache", void 0);
+        this.interface = interfaceInstance;
+        this.insight = insight;
+        this.taskCache = taskCache;
+    }
+}
+export { TaskBuilder, locatePlanForLocate };
+
+//# sourceMappingURL=task-builder.mjs.map

package/dist/es/agent/task-builder.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/task-builder.mjs","sources":["webpack://@midscene/core/./src/agent/task-builder.ts"],"sourcesContent":["import type { AbstractInterface } from '@/device';\nimport type Insight from '@/insight';\nimport type {\n  DetailedLocateParam,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskInsightLocateApply,\n  InsightDump,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningActionParamError,\n  PlanningActionParamSleep,\n  PlanningLocateParam,\n} from '@/types';\nimport { InsightError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  findAllMidsceneLocatorField,\n  parseActionParam,\n} from '@/ai-model';\nimport {\n  matchElementFromCache,\n  matchElementFromPlan,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    locate,\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  insight: Insight;\n  taskCache?: TaskCache;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly insight: Insight;\n\n  private readonly taskCache?: TaskCache;\n\n  constructor({ interfaceInstance, insight, taskCache }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.insight = insight;\n    this.taskCache = taskCache;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfig: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const taskForLocatePlan = (\n      plan: PlanningAction<PlanningLocateParam>,\n      detailedLocateParam: DetailedLocateParam | string,\n      onResult?: (result: LocateResultElement) => void,\n    ): ExecutionTaskInsightLocateApply => {\n      if (typeof detailedLocateParam === 'string') {\n        detailedLocateParam = {\n          prompt: detailedLocateParam,\n        };\n      }\n\n      if (cacheable !== undefined) {\n        detailedLocateParam = {\n          ...detailedLocateParam,\n          cacheable,\n        };\n      }\n\n      const taskFind: ExecutionTaskInsightLocateApply = {\n        type: 'Insight',\n        subType: 'Locate',\n        param: detailedLocateParam,\n        thought: plan.thought,\n        executor: async (param, taskContext) => {\n          const { task } = taskContext;\n          let { uiContext } = taskContext;\n\n          assert(\n            param?.prompt || param?.id || param?.bbox,\n            `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n              param,\n            )}`,\n          );\n\n          if (!uiContext) {\n            uiContext = await this.insight.contextRetrieverFn();\n          }\n\n          assert(uiContext, 'uiContext is required for Insight task');\n\n          let locateDump: InsightDump | undefined;\n          let locateResult: LocateResultWithDump | undefined;\n\n          const applyDump = (dump?: InsightDump) => {\n            if (!dump) {\n              return;\n            }\n            locateDump = dump;\n            task.log = {\n              dump,\n            };\n            task.usage = dump.taskInfo?.usage;\n            if (dump.taskInfo?.searchAreaUsage) {\n              task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n            }\n          };\n\n          const elementFromXpath =\n            param.xpath && (this.interface as any).getElementInfoByXpath\n              ? await (this.interface as any).getElementInfoByXpath(param.xpath)\n              : undefined;\n          const userExpectedPathHitFlag = !!elementFromXpath;\n\n          const cachePrompt = param.prompt;\n          const locateCacheRecord = this.taskCache?.matchLocateCache(\n            cachePrompt,\n          );\n          const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n          const elementFromCache = userExpectedPathHitFlag\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n          const cacheHitFlag = !!elementFromCache;\n\n          const elementFromPlan =\n            !userExpectedPathHitFlag && !cacheHitFlag\n              ? matchElementFromPlan(param, uiContext.tree)\n              : undefined;\n          const planHitFlag = !!elementFromPlan;\n\n          let elementFromAiLocate: LocateResultElement | null | undefined;\n          if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) {\n            try {\n              locateResult = await this.insight.locate(\n                param,\n                {\n                  context: uiContext,\n                },\n                modelConfig,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n            } catch (error) {\n              if (error instanceof InsightError) {\n                applyDump(error.dump);\n              }\n              throw error;\n            }\n          }\n          const aiLocateHitFlag = !!elementFromAiLocate;\n\n          const element =\n            elementFromXpath ||\n            elementFromCache ||\n            elementFromPlan ||\n            elementFromAiLocate;\n\n          let currentCacheEntry: ElementCacheFeature | undefined;\n          if (\n            element &&\n            this.taskCache &&\n            !cacheHitFlag &&\n            param?.cacheable !== false\n          ) {\n            if (this.interface.cacheFeatureForRect) {\n              try {\n                const feature = await this.interface.cacheFeatureForRect(\n                  element.rect,\n                  element.isOrderSensitive !== undefined\n                    ? { _orderSensitive: element.isOrderSensitive }\n                    : undefined,\n                );\n                if (feature && Object.keys(feature).length > 0) {\n                  debug(\n                    'update cache, prompt: %s, cache: %o',\n                    cachePrompt,\n                    feature,\n                  );\n                  currentCacheEntry = feature;\n                  this.taskCache.updateOrAppendCacheRecord(\n                    {\n                      type: 'locate',\n                      prompt: cachePrompt,\n                      cache: feature,\n                    },\n                    locateCacheRecord,\n                  );\n                } else {\n                  debug(\n                    'no cache data returned, skip cache update, prompt: %s',\n                    cachePrompt,\n                  );\n                }\n              } catch (error) {\n                debug('cacheFeatureForRect failed: %s', error);\n              }\n            } else {\n              debug('cacheFeatureForRect is not supported, skip cache update');\n            }\n          }\n\n          if (!element) {\n            if (locateDump) {\n              throw new InsightError(\n                `Element not found: ${param.prompt}`,\n                locateDump,\n              );\n            }\n            throw new Error(`Element not found: ${param.prompt}`);\n          }\n\n          let hitBy: ExecutionTaskHitBy | undefined;\n\n          if (userExpectedPathHitFlag) {\n            hitBy = {\n              from: 'User expected path',\n              context: {\n                xpath: param.xpath,\n              },\n            };\n          } else if (cacheHitFlag) {\n            hitBy = {\n              from: 'Cache',\n              context: {\n                cacheEntry,\n                cacheToSave: currentCacheEntry,\n              },\n            };\n          } else if (planHitFlag) {\n            hitBy = {\n              from: 'Planning',\n              context: {\n                id: elementFromPlan?.id,\n                bbox: elementFromPlan?.bbox,\n              },\n            };\n          } else if (aiLocateHitFlag) {\n            hitBy = {\n              from: 'AI model',\n              context: {\n                prompt: param.prompt,\n              },\n            };\n          }\n\n          onResult?.(element);\n\n          return {\n            output: {\n              element,\n            },\n            uiContext,\n            hitBy,\n          };\n        },\n      };\n\n      return taskFind;\n    };\n\n    for (const plan of plans) {\n      if (plan.type === 'Locate') {\n        if (!plan.locate || plan.locate === null) {\n          debug('Locate action with id is null, will be ignored', plan);\n          continue;\n        }\n        const taskLocate = taskForLocatePlan(plan, plan.locate);\n        tasks.push(taskLocate);\n        continue;\n      }\n\n      if (plan.type === 'Error') {\n        const taskActionError: ExecutionTaskActionApply<PlanningActionParamError> =\n          {\n            type: 'Action',\n            subType: 'Error',\n            param: plan.param,\n            thought: plan.thought || plan.param?.thought,\n            locate: plan.locate,\n            executor: async () => {\n              throw new Error(\n                plan?.thought || plan.param?.thought || 'error without thought',\n              );\n            },\n          };\n        tasks.push(taskActionError);\n        continue;\n      }\n\n      if (plan.type === 'Finished') {\n        const taskActionFinished: ExecutionTaskActionApply<null> = {\n          type: 'Action',\n          subType: 'Finished',\n          param: null,\n          thought: plan.thought,\n          locate: plan.locate,\n          executor: async () => {},\n        };\n        tasks.push(taskActionFinished);\n        continue;\n      }\n\n      if (plan.type === 'Sleep') {\n        const taskActionSleep: ExecutionTaskActionApply<PlanningActionParamSleep> =\n          {\n            type: 'Action',\n            subType: 'Sleep',\n            param: plan.param,\n            thought: plan.thought,\n            locate: plan.locate,\n            executor: async (taskParam) => {\n              await sleep(taskParam?.timeMs || 3000);\n            },\n          };\n        tasks.push(taskActionSleep);\n        continue;\n      }\n\n      const planType = plan.type;\n      const actionSpace = await this.interface.actionSpace();\n      const action = actionSpace.find((item) => item.name === planType);\n      const param = plan.param;\n\n      if (!action) {\n        throw new Error(`Action type '${planType}' not found`);\n      }\n\n      const locateFields = action\n        ? findAllMidsceneLocatorField(action.paramSchema)\n        : [];\n\n      const requiredLocateFields = action\n        ? findAllMidsceneLocatorField(action.paramSchema, true)\n        : [];\n\n      locateFields.forEach((field) => {\n        if (param[field]) {\n          const locatePlan = locatePlanForLocate(param[field]);\n          debug(\n            'will prepend locate param for field',\n            `action.type=${planType}`,\n            `param=${JSON.stringify(param[field])}`,\n            `locatePlan=${JSON.stringify(locatePlan)}`,\n          );\n          const locateTask = taskForLocatePlan(\n            locatePlan,\n            param[field],\n            (result) => {\n              param[field] = result;\n            },\n          );\n          tasks.push(locateTask);\n        } else {\n          assert(\n            !requiredLocateFields.includes(field),\n            `Required locate field '${field}' is not provided for action ${planType}`,\n          );\n          debug(`field '${field}' is not provided for action ${planType}`);\n        }\n      });\n\n      const task: ExecutionTaskApply<\n        'Action',\n        any,\n        { success: boolean; action: string; param: any },\n        void\n      > = {\n        type: 'Action',\n        subType: planType,\n        thought: plan.thought,\n        param: plan.param,\n        executor: async (param, taskContext) => {\n          debug(\n            'executing action',\n            planType,\n            param,\n            `taskContext.element.center: ${taskContext.element?.center}`,\n          );\n\n          const uiContext = taskContext.uiContext;\n          assert(uiContext, 'uiContext is required for Action task');\n\n          requiredLocateFields.forEach((field) => {\n            assert(\n              param[field],\n              `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n            );\n          });\n\n          try {\n            await Promise.all([\n              (async () => {\n                if (this.interface.beforeInvokeAction) {\n                  debug('will call \"beforeInvokeAction\" for interface');\n                  await this.interface.beforeInvokeAction(action.name, param);\n                  debug('called \"beforeInvokeAction\" for interface');\n                }\n              })(),\n              sleep(200),\n            ]);\n          } catch (originalError: any) {\n            const originalMessage =\n              originalError?.message || String(originalError);\n            throw new Error(\n              `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n              { cause: originalError },\n            );\n          }\n\n          if (action.paramSchema) {\n            try {\n              param = parseActionParam(param, action.paramSchema);\n            } catch (error: any) {\n              throw new Error(\n                `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n                { cause: error },\n              );\n            }\n          }\n\n          debug('calling action', action.name);\n          const actionFn = action.call.bind(this.interface);\n          await actionFn(param, taskContext);\n          debug('called action', action.name);\n\n          try {\n            if (this.interface.afterInvokeAction) {\n              debug('will call \"afterInvokeAction\" for interface');\n              await this.interface.afterInvokeAction(action.name, param);\n              debug('called \"afterInvokeAction\" for interface');\n            }\n          } catch (originalError: any) {\n            const originalMessage =\n              originalError?.message || String(originalError);\n            throw new Error(\n              `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n              { cause: originalError },\n            );\n          }\n\n          return {\n            output: {\n              success: true,\n              action: planType,\n              param: param,\n            },\n          };\n        },\n      };\n\n      tasks.push(task);\n    }\n\n    return {\n      tasks,\n    };\n  }\n}\n"],"names":["debug","getDebug","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfig","options","tasks","cacheable","taskForLocatePlan","plan","detailedLocateParam","onResult","undefined","taskFind","taskContext","_this_taskCache","_locateCacheRecord_cacheContent","task","uiContext","assert","JSON","locateDump","locateResult","applyDump","dump","_dump_taskInfo","_dump_taskInfo1","elementFromXpath","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","cacheEntry","elementFromCache","matchElementFromCache","cacheHitFlag","elementFromPlan","matchElementFromPlan","planHitFlag","elementFromAiLocate","error","InsightError","aiLocateHitFlag","element","currentCacheEntry","feature","Object","Error","hitBy","taskLocate","_plan_param","taskActionError","taskActionFinished","taskActionSleep","taskParam","sleep","planType","actionSpace","action","item","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","locateTask","result","_taskContext_element","Promise","originalError","originalMessage","String","parseActionParam","actionFn","interfaceInstance","insight","taskCache"],"mappings":";;;;;;;;;;;;;;;;AAgCA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACND;QACA,OAAOA;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAYO,MAAMC;IAaX,MAAa,MACXC,KAAuB,EACvBC,WAAyB,EACzBC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,SAAS;QAEpC,MAAMG,oBAAoB,CACxBC,MACAC,qBACAC;YAEA,IAAI,AAA+B,YAA/B,OAAOD,qBACTA,sBAAsB;gBACpB,QAAQA;YACV;YAGF,IAAIH,AAAcK,WAAdL,WACFG,sBAAsB;gBACpB,GAAGA,mBAAmB;gBACtBH;YACF;YAGF,MAAMM,WAA4C;gBAChD,MAAM;gBACN,SAAS;gBACT,OAAOH;gBACP,SAASD,KAAK,OAAO;gBACrB,UAAU,OAAOV,OAAOe;wBAyCIC,iBAGPC;oBA3CnB,MAAM,EAAEC,IAAI,EAAE,GAAGH;oBACjB,IAAI,EAAEI,SAAS,EAAE,GAAGJ;oBAEpBK,OACEpB,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,MAAM,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,EAAE,AAAD,KAAKA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,IAAI,AAAD,GACxC,CAAC,qDAAqD,EAAEqB,KAAK,SAAS,CACpErB,QACC;oBAGL,IAAI,CAACmB,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;oBAGnDC,OAAOD,WAAW;oBAElB,IAAIG;oBACJ,IAAIC;oBAEJ,MAAMC,YAAY,CAACC;4BAQJC,gBACTC;wBARJ,IAAI,CAACF,MACH;wBAEFH,aAAaG;wBACbP,KAAK,GAAG,GAAG;4BACTO;wBACF;wBACAP,KAAK,KAAK,GAAG,QAAAQ,CAAAA,iBAAAA,KAAK,QAAQ,AAAD,IAAZA,KAAAA,IAAAA,eAAe,KAAK;wBACjC,IAAI,QAAAC,CAAAA,kBAAAA,KAAK,QAAQ,AAAD,IAAZA,KAAAA,IAAAA,gBAAe,eAAe,EAChCT,KAAK,eAAe,GAAGO,KAAK,QAAQ,CAAC,eAAe;oBAExD;oBAEA,MAAMG,mBACJ5B,MAAM,KAAK,IAAK,IAAI,CAAC,SAAS,CAAS,qBAAqB,GACxD,MAAO,IAAI,CAAC,SAAS,CAAS,qBAAqB,CAACA,MAAM,KAAK,IAC/Da;oBACN,MAAMgB,0BAA0B,CAAC,CAACD;oBAElC,MAAME,cAAc9B,MAAM,MAAM;oBAChC,MAAM+B,oBAAoB,QAAAf,CAAAA,kBAAAA,IAAI,CAAC,SAAS,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,gBAAgB,CACxDc;oBAEF,MAAME,aAAaf,QAAAA,oBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,kCAAAA,kBAAmB,YAAY,AAAD,IAA9BA,KAAAA,IAAAA,gCAAiC,KAAK;oBAEzD,MAAMgB,mBAAmBJ,0BACrB,OACA,MAAMK,sBACJ;wBACE,WAAW,IAAI,CAAC,SAAS;wBACzB,mBAAmB,IAAI,CAAC,SAAS;oBACnC,GACAF,YACAF,aACA9B,MAAM,SAAS;oBAErB,MAAMmC,eAAe,CAAC,CAACF;oBAEvB,MAAMG,kBACJ,AAACP,2BAA4BM,eAEzBtB,SADAwB,qBAAqBrC,OAAOmB,UAAU,IAAI;oBAEhD,MAAMmB,cAAc,CAAC,CAACF;oBAEtB,IAAIG;oBACJ,IAAI,CAACV,2BAA2B,CAACM,gBAAgB,CAACG,aAChD,IAAI;wBACFf,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCvB,OACA;4BACE,SAASmB;wBACX,GACAd;wBAEFmB,UAAUD,aAAa,IAAI;wBAC3BgB,sBAAsBhB,aAAa,OAAO;oBAC5C,EAAE,OAAOiB,OAAO;wBACd,IAAIA,iBAAiBC,cACnBjB,UAAUgB,MAAM,IAAI;wBAEtB,MAAMA;oBACR;oBAEF,MAAME,kBAAkB,CAAC,CAACH;oBAE1B,MAAMI,UACJf,oBACAK,oBACAG,mBACAG;oBAEF,IAAIK;oBACJ,IACED,WACA,IAAI,CAAC,SAAS,IACd,CAACR,gBACDnC,AAAAA,CAAAA,QAAAA,QAAAA,KAAAA,IAAAA,MAAO,SAAS,AAAD,MAAM,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EACpC,IAAI;wBACF,MAAM6C,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CACtDF,QAAQ,IAAI,EACZA,AAA6B9B,WAA7B8B,QAAQ,gBAAgB,GACpB;4BAAE,iBAAiBA,QAAQ,gBAAgB;wBAAC,IAC5C9B;wBAEN,IAAIgC,WAAWC,OAAO,IAAI,CAACD,SAAS,MAAM,GAAG,GAAG;4BAC9ChD,MACE,uCACAiC,aACAe;4BAEFD,oBAAoBC;4BACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;gCACE,MAAM;gCACN,QAAQf;gCACR,OAAOe;4BACT,GACAd;wBAEJ,OACElC,MACE,yDACAiC;oBAGN,EAAE,OAAOU,OAAO;wBACd3C,MAAM,kCAAkC2C;oBAC1C;yBAEA3C,MAAM;oBAIV,IAAI,CAAC8C,SAAS;wBACZ,IAAIrB,YACF,MAAM,IAAImB,aACR,CAAC,mBAAmB,EAAEzC,MAAM,MAAM,EAAE,EACpCsB;wBAGJ,MAAM,IAAIyB,MAAM,CAAC,mBAAmB,EAAE/C,MAAM,MAAM,EAAE;oBACtD;oBAEA,IAAIgD;oBAEJ,IAAInB,yBACFmB,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,OAAOhD,MAAM,KAAK;wBACpB;oBACF;yBACK,IAAImC,cACTa,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACPhB;4BACA,aAAaY;wBACf;oBACF;yBACK,IAAIN,aACTU,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,IAAIZ,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,EAAE;4BACvB,MAAMA,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,IAAI;wBAC7B;oBACF;yBACK,IAAIM,iBACTM,QAAQ;wBACN,MAAM;wBACN,SAAS;4BACP,QAAQhD,MAAM,MAAM;wBACtB;oBACF;oBAGFY,QAAAA,YAAAA,SAAW+B;oBAEX,OAAO;wBACL,QAAQ;4BACNA;wBACF;wBACAxB;wBACA6B;oBACF;gBACF;YACF;YAEA,OAAOlC;QACT;QAEA,KAAK,MAAMJ,QAAQN,MAAO;YACxB,IAAIM,AAAc,aAAdA,KAAK,IAAI,EAAe;gBAC1B,IAAI,CAACA,KAAK,MAAM,IAAIA,AAAgB,SAAhBA,KAAK,MAAM,EAAW;oBACxCb,MAAM,kDAAkDa;oBACxD;gBACF;gBACA,MAAMuC,aAAaxC,kBAAkBC,MAAMA,KAAK,MAAM;gBACtDH,MAAM,IAAI,CAAC0C;gBACX;YACF;YAEA,IAAIvC,AAAc,YAAdA,KAAK,IAAI,EAAc;oBAMIwC;gBAL7B,MAAMC,kBACJ;oBACE,MAAM;oBACN,SAAS;oBACT,OAAOzC,KAAK,KAAK;oBACjB,SAASA,KAAK,OAAO,aAAIwC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD;oBAC3C,QAAQxC,KAAK,MAAM;oBACnB,UAAU;4BAEWwC;wBADnB,MAAM,IAAIH,MACRrC,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,OAAO,AAAD,KAAC,SAAIwC,CAAAA,cAAAA,KAAK,KAAK,AAAD,IAATA,KAAAA,IAAAA,YAAY,OAAO,AAAD,KAAK;oBAE5C;gBACF;gBACF3C,MAAM,IAAI,CAAC4C;gBACX;YACF;YAEA,IAAIzC,AAAc,eAAdA,KAAK,IAAI,EAAiB;gBAC5B,MAAM0C,qBAAqD;oBACzD,MAAM;oBACN,SAAS;oBACT,OAAO;oBACP,SAAS1C,KAAK,OAAO;oBACrB,QAAQA,KAAK,MAAM;oBACnB,UAAU,WAAa;gBACzB;gBACAH,MAAM,IAAI,CAAC6C;gBACX;YACF;YAEA,IAAI1C,AAAc,YAAdA,KAAK,IAAI,EAAc;gBACzB,MAAM2C,kBACJ;oBACE,MAAM;oBACN,SAAS;oBACT,OAAO3C,KAAK,KAAK;oBACjB,SAASA,KAAK,OAAO;oBACrB,QAAQA,KAAK,MAAM;oBACnB,UAAU,OAAO4C;wBACf,MAAMC,MAAMD,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,MAAM,AAAD,KAAK;oBACnC;gBACF;gBACF/C,MAAM,IAAI,CAAC8C;gBACX;YACF;YAEA,MAAMG,WAAW9C,KAAK,IAAI;YAC1B,MAAM+C,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;YACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;YACxD,MAAMxD,QAAQU,KAAK,KAAK;YAExB,IAAI,CAACgD,QACH,MAAM,IAAIX,MAAM,CAAC,aAAa,EAAES,SAAS,WAAW,CAAC;YAGvD,MAAMI,eAAeF,SACjBG,4BAA4BH,OAAO,WAAW,IAC9C,EAAE;YAEN,MAAMI,uBAAuBJ,SACzBG,4BAA4BH,OAAO,WAAW,EAAE,QAChD,EAAE;YAENE,aAAa,OAAO,CAAC,CAACG;gBACpB,IAAI/D,KAAK,CAAC+D,MAAM,EAAE;oBAChB,MAAM7D,aAAaH,oBAAoBC,KAAK,CAAC+D,MAAM;oBACnDlE,MACE,uCACA,CAAC,YAAY,EAAE2D,UAAU,EACzB,CAAC,MAAM,EAAEnC,KAAK,SAAS,CAACrB,KAAK,CAAC+D,MAAM,GAAG,EACvC,CAAC,WAAW,EAAE1C,KAAK,SAAS,CAACnB,aAAa;oBAE5C,MAAM8D,aAAavD,kBACjBP,YACAF,KAAK,CAAC+D,MAAM,EACZ,CAACE;wBACCjE,KAAK,CAAC+D,MAAM,GAAGE;oBACjB;oBAEF1D,MAAM,IAAI,CAACyD;gBACb,OAAO;oBACL5C,OACE,CAAC0C,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEP,UAAU;oBAE3E3D,MAAM,CAAC,OAAO,EAAEkE,MAAM,6BAA6B,EAAEP,UAAU;gBACjE;YACF;YAEA,MAAMtC,OAKF;gBACF,MAAM;gBACN,SAASsC;gBACT,SAAS9C,KAAK,OAAO;gBACrB,OAAOA,KAAK,KAAK;gBACjB,UAAU,OAAOV,OAAOe;wBAKWmD;oBAJjCrE,MACE,oBACA2D,UACAxD,OACA,CAAC,4BAA4B,EAAE,QAAAkE,CAAAA,uBAAAA,YAAY,OAAO,AAAD,IAAlBA,KAAAA,IAAAA,qBAAqB,MAAM,EAAE;oBAG9D,MAAM/C,YAAYJ,YAAY,SAAS;oBACvCK,OAAOD,WAAW;oBAElB2C,qBAAqB,OAAO,CAAC,CAACC;wBAC5B3C,OACEpB,KAAK,CAAC+D,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEP,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;oBAE9G;oBAEA,IAAI;wBACF,MAAMW,QAAQ,GAAG,CAAC;4BACf;gCACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;oCACrCtE,MAAM;oCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC6D,OAAO,IAAI,EAAE1D;oCACrDH,MAAM;gCACR;4BACF;4BACA0D,MAAM;yBACP;oBACH,EAAE,OAAOa,eAAoB;wBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;wBACnC,MAAM,IAAIrB,MACR,CAAC,wCAAwC,EAAEW,OAAO,IAAI,CAAC,EAAE,EAAEW,iBAAiB,EAC5E;4BAAE,OAAOD;wBAAc;oBAE3B;oBAEA,IAAIV,OAAO,WAAW,EACpB,IAAI;wBACF1D,QAAQuE,iBAAiBvE,OAAO0D,OAAO,WAAW;oBACpD,EAAE,OAAOlB,OAAY;wBACnB,MAAM,IAAIO,MACR,CAAC,8BAA8B,EAAEW,OAAO,IAAI,CAAC,EAAE,EAAElB,MAAM,OAAO,CAAC,cAAc,EAAEnB,KAAK,SAAS,CAACrB,QAAQ,EACtG;4BAAE,OAAOwC;wBAAM;oBAEnB;oBAGF3C,MAAM,kBAAkB6D,OAAO,IAAI;oBACnC,MAAMc,WAAWd,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;oBAChD,MAAMc,SAASxE,OAAOe;oBACtBlB,MAAM,iBAAiB6D,OAAO,IAAI;oBAElC,IAAI;wBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;4BACpC7D,MAAM;4BACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC6D,OAAO,IAAI,EAAE1D;4BACpDH,MAAM;wBACR;oBACF,EAAE,OAAOuE,eAAoB;wBAC3B,MAAMC,kBACJD,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,OAAO,AAAD,KAAKE,OAAOF;wBACnC,MAAM,IAAIrB,MACR,CAAC,uCAAuC,EAAEW,OAAO,IAAI,CAAC,EAAE,EAAEW,iBAAiB,EAC3E;4BAAE,OAAOD;wBAAc;oBAE3B;oBAEA,OAAO;wBACL,QAAQ;4BACN,SAAS;4BACT,QAAQZ;4BACR,OAAOxD;wBACT;oBACF;gBACF;YACF;YAEAO,MAAM,IAAI,CAACW;QACb;QAEA,OAAO;YACLX;QACF;IACF;IA5aA,YAAY,EAAEkE,iBAAiB,EAAEC,OAAO,EAAEC,SAAS,EAAmB,CAAE;QANxE,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAGE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;IACnB;AAyaF"}