@midscene/core 1.14.0 → 1.15.0

package/dist/lib/agent/task-builder.js

@@ -151,26 +151,6 @@ class TaskBuilder {
                 const uiContext = taskContext.uiContext;
                 (0, utils_namespaceObject.assert)(uiContext, 'uiContext is required for Action task');
                 (0, external_common_js_namespaceObject.validateRequiredLocateFields)(param, action.paramSchema);
-                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'beforeInvokeActionHookStart');
-                const delayBeforeRunner = action.delayBeforeRunner ?? 200;
-                try {
-                    await Promise.all([
-                        (async ()=>{
-                            if (this.interface.beforeInvokeAction) {
-                                debug(`will call "beforeInvokeAction" for interface with action name ${action.name}`);
-                                await this.interface.beforeInvokeAction(action.name, param);
-                                debug(`called "beforeInvokeAction" for interface with action name ${action.name}`);
-                            }
-                        })(),
-                        delayBeforeRunner > 0 ? (0, external_utils_js_namespaceObject.sleep)(delayBeforeRunner) : Promise.resolve()
-                    ]);
-                } catch (originalError) {
-                    const originalMessage = originalError?.message || String(originalError);
-                    throw new Error(`error in running beforeInvokeAction for ${action.name}: ${originalMessage}`, {
-                        cause: originalError
-                    });
-                }
-                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'beforeInvokeActionHookEnd');
                 const { shrunkShotToLogicalRatio } = uiContext;
                 if (void 0 === shrunkShotToLogicalRatio) throw new Error('shrunkShotToLogicalRatio is not defined in Action task');
                 const parsedParam = (()=>{
@@ -185,28 +165,58 @@ class TaskBuilder {
                         });
                     }
                 })();
+                const useDefaultWait = !this.actionReadiness;
+                taskContext.task.actionReadiness = useDefaultWait ? 'default' : 'custom';
+                context.abortSignal?.throwIfAborted();
+                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'beforeInvokeActionHookStart');
+                const delayBeforeRunner = useDefaultWait ? action.delayBeforeRunner ?? 200 : 0;
+                try {
+                    await Promise.all([
+                        this.interface.beforeInvokeAction?.(action.name, param),
+                        delayBeforeRunner > 0 ? (0, external_utils_js_namespaceObject.sleep)(delayBeforeRunner) : Promise.resolve()
+                    ]);
+                } catch (originalError) {
+                    throw new Error(`error in running beforeInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`, {
+                        cause: originalError
+                    });
+                } finally{
+                    (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'beforeInvokeActionHookEnd');
+                }
+                context.abortSignal?.throwIfAborted();
                 (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'callActionStart');
                 debug('calling action', action.name);
-                const actionFn = action.call.bind(this.interface);
-                const actionResult = await actionFn(parsedParam, taskContext);
+                const actionResult = await action.call.call(this.interface, parsedParam, useDefaultWait ? taskContext : {
+                    ...taskContext,
+                    skipDefaultWait: true
+                });
                 (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'callActionEnd');
                 debug('called action', action.name, 'result:', actionResult);
+                context.abortSignal?.throwIfAborted();
                 (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'afterInvokeActionHookStart');
-                const delayAfterRunner = action.delayAfterRunner ?? this.waitAfterAction ?? 300;
-                if (delayAfterRunner > 0) await (0, external_utils_js_namespaceObject.sleep)(delayAfterRunner);
+                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'waitForActionReadyStart');
                 try {
-                    if (this.interface.afterInvokeAction) {
-                        debug(`will call "afterInvokeAction" for interface with action name ${action.name}`);
-                        await this.interface.afterInvokeAction(action.name, parsedParam);
-                        debug(`called "afterInvokeAction" for interface with action name ${action.name}`);
-                    }
+                    if (useDefaultWait) {
+                        const delayAfterRunner = action.delayAfterRunner ?? this.waitAfterAction ?? 300;
+                        if (delayAfterRunner > 0) await (0, external_utils_js_namespaceObject.sleep)(delayAfterRunner);
+                        await this.interface.defaultActionWait?.(action.name, parsedParam);
+                    } else await this.actionReadiness.wait({
+                        id: taskContext.task.taskId,
+                        name: action.name,
+                        param: parsedParam
+                    }, context.abortSignal);
+                } finally{
+                    (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'waitForActionReadyEnd');
+                }
+                context.abortSignal?.throwIfAborted();
+                try {
+                    await this.interface.afterInvokeAction?.(action.name, parsedParam);
                 } catch (originalError) {
-                    const originalMessage = originalError?.message || String(originalError);
-                    throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalMessage}`, {
+                    throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`, {
                         cause: originalError
                     });
+                } finally{
+                    (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'afterInvokeActionHookEnd');
                 }
-                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'afterInvokeActionHookEnd');
                 return {
                     output: actionResult
                 };
@@ -379,17 +389,19 @@ class TaskBuilder {
         };
         return taskLocator;
     }
-    constructor({ interfaceInstance, service, taskCache, actionSpace, waitAfterAction }){
+    constructor({ interfaceInstance, service, taskCache, actionSpace, waitAfterAction, actionReadiness }){
         _define_property(this, "interface", void 0);
         _define_property(this, "service", void 0);
         _define_property(this, "taskCache", void 0);
         _define_property(this, "actionSpace", void 0);
         _define_property(this, "waitAfterAction", void 0);
+        _define_property(this, "actionReadiness", void 0);
         this.interface = interfaceInstance;
         this.service = service;
         this.taskCache = taskCache;
         this.actionSpace = actionSpace;
         this.waitAfterAction = waitAfterAction;
+        this.actionReadiness = actionReadiness;
     }
 }
 exports.TaskBuilder = __webpack_exports__.TaskBuilder;

package/dist/lib/agent/task-builder.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/task-builder.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport {\n  findActionInActionSpaceOrThrow,\n  validateRequiredLocateFields,\n} from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifLocateParamHasLocatedPixelResult,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [element.center?.[0], element.center?.[1]];\n  if (\n    element.center?.length !== 2 ||\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<DetailedLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<DetailedLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<DetailedLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    validateRequiredLocateFields(param, action.paramSchema);\n\n    locateFields.forEach((field) => {\n      if (param?.[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelResult is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      planningFeedbackMaxLength: action.planningFeedbackMaxLength,\n      executor: async (taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        validateRequiredLocateFields(param, action.paramSchema);\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        const parsedParam = (() => {\n          if (!action.paramSchema) {\n            return param;\n          }\n\n          try {\n            return parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        })();\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(parsedParam, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, parsedParam);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<DetailedLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(\n          locateParam,\n        )\n          ? locateParam\n          : undefined;\n\n        assert(\n          locateParam?.prompt || paramWithLocatedPixelResult,\n          `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(\n            locateParam,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            // The deepLocate search-area call belongs to the same locate task,\n            // so it shares the companion usage intent.\n            task.searchAreaUsage = withUsageIntent(\n              dump.taskInfo.searchAreaUsage,\n              'default',\n            );\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelResult\n          ? matchElementFromPlan(paramWithLocatedPixelResult)\n          : undefined;\n\n        // from locatedPixelResult (direct plan hit)\n        // when deepLocate is enabled, locatedPixelResult should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = locateParam.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          locateParam.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [locateParam.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? transformLogicalElementToScreenshot(\n              {\n                center: [\n                  rectFromXpath.left + rectFromXpath.width / 2,\n                  rectFromXpath.top + rectFromXpath.height / 2,\n                ],\n                description:\n                  typeof locateParam.prompt === 'string'\n                    ? locateParam.prompt\n                    : locateParam.prompt?.prompt || '',\n                rect: rectFromXpath,\n              },\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = locateParam.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                locateParam.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              locateParam,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          locateParam.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              const pointForCache: [number, number] = [\n                Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                Math.round(element.center[1] / shrunkShotToLogicalRatio),\n              ];\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof locateParam.prompt === 'string'\n                      ? locateParam.prompt\n                      : locateParam.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${locateParam.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${locateParam.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelResult) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelResult:\n                paramWithLocatedPixelResult.locatedPixelResult,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: locateParam.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        const promptDisplay = locateParam.promptDisplay;\n        const elementForAction = promptDisplay\n          ? {\n              ...element,\n              description: promptDisplay,\n            }\n          : element;\n\n        if (promptDisplay && locateDump?.matchedElement) {\n          locateDump.matchedElement = locateDump.matchedElement.map(\n            (matchedElement) => ({\n              ...matchedElement,\n              description: promptDisplay,\n            }),\n          );\n        }\n\n        onResult?.(elementForAction);\n\n        return {\n          output: {\n            element: {\n              ...elementForAction,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","hasNonEmptyCache","cache","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","validateRequiredLocateFields","field","ifLocateParamHasLocatedPixelResult","locateTask","result","task","taskContext","timing","uiContext","assert","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","Error","shrunkShotToLogicalRatio","parsedParam","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelResult","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","transformLogicalElementToScreenshot","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","promptDisplay","elementForAction","matchedElement","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;AC+BA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPP,OAAO,IAAI,CAACO,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASC,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QAACD,QAAQ,MAAM,EAAE,CAAC,EAAE;QAAEA,QAAQ,MAAM,EAAE,CAAC,EAAE;KAAC;IACzD,IACEA,QAAQ,MAAM,EAAE,WAAW,KAC3BC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;AAG1E;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BH,OAAO,WAAW;QAEnEI,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6B7B,OAAOyB,OAAO,WAAW;QAEtDE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,OAAO,CAAC8B,MAAM,EAAE;gBAGlB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnDzC,MACE,uCACA,CAAC,YAAY,EAAEkC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,sBAAsB,EAAEwB,AAAAA,IAAAA,oCAAAA,kCAAAA,AAAAA,EAAmC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OACE3C,MAAM,CAAC,OAAO,EAAEyC,MAAM,6BAA6B,EAAEP,UAAU;QAEnE;QAEA,MAAMW,OAKF;YACF,MAAM;YACN,SAASX;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,2BAA2BO,OAAO,yBAAyB;YAC3D,UAAU,OAAOU;gBACf,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtC9C,MACE,oBACAkC,UACAvB,OACA,CAAC,4BAA4B,EAAEmC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElBR,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6B7B,OAAOyB,OAAO,WAAW;gBAEtDc,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBH,QAAQ;gBAC3B,MAAMI,oBAAoBf,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMgB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCpD,MACE,CAAC,8DAA8D,EAAEoC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDX,MACE,CAAC,2DAA2D,EAAEoC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAe,oBAAoB,IAChBE,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,wCAAwC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBH,QAAQ;gBAE3B,MAAM,EAAEW,wBAAwB,EAAE,GAAGV;gBACrC,IAAIU,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,MAAME,cAAe,AAAC;oBACpB,IAAI,CAACvB,OAAO,WAAW,EACrB,OAAOzB;oBAGT,IAAI;wBACF,OAAOiD,AAAAA,IAAAA,yBAAAA,gBAAAA,AAAAA,EAAiBjD,OAAOyB,OAAO,WAAW,EAAE;4BACjDsB;wBACF;oBACF,EAAE,OAAOG,OAAY;wBACnB,MAAM,IAAIJ,MACR,CAAC,8BAA8B,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAEpD,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAOkD;wBAAM;oBAEnB;gBACF;gBAEAX,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBH,QAAQ;gBAE3B/C,MAAM,kBAAkBoC,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAASH,aAAab;gBACjDI,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBH,QAAQ;gBAC3B/C,MAAM,iBAAiBoC,OAAO,IAAI,EAAE,WAAW2B;gBAE/Cb,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBH,QAAQ;gBAE3B,MAAMiB,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMX,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMW;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpChE,MACE,CAAC,6DAA6D,EAAEoC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEuB;wBACpD3D,MACE,CAAC,0DAA0D,EAAEoC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOkB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,uCAAuC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBH,QAAQ;gBAE3B,OAAO;oBACL,QAAQgB;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACmB;IACrB;IAEQ,iBACNhB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEqD,WAAW,EAAE,GAAGzC;QAE7D,IAAI0C,cAAc1D,qBAAqBuD;QAEvC,IAAIxC,AAAcV,WAAdU,WACF2C,cAAc;YACZ,GAAGA,WAAW;YACd3C;QACF;QAGF,IAAIX,cAAc,CAACsD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASvC,KAAK,OAAO;YACrB,UAAU,OAAOiB;gBACf,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMwB,8BAA8B5B,AAAAA,IAAAA,oCAAAA,kCAAAA,AAAAA,EAClC0B,eAEEA,cACArD;gBAEJkC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEmB,aAAa,UAAUE,6BACvB,CAAC,mEAAmE,EAAE7D,KAAK,SAAS,CAClF2D,cACC;gBAGL,IAAI,CAACpB,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElB,MAAM,EAAEU,wBAAwB,EAAE,GAAGV;gBAErC,IAAIU,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIc;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb7B,KAAK,GAAG,GAAG;wBACT6B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA7B,KAAK,KAAK,GAAG8B,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD7B,KAAK,UAAU,GAAG6B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjB7B,KAAK,eAAe,GAAG8B,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjB7B,KAAK,iBAAiB,GAAG6B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,8BACvBO,AAAAA,IAAAA,oCAAAA,oBAAAA,AAAAA,EAAqBP,+BACrBvD;gBAKJ,MAAM+D,kBAAkBV,YAAY,UAAU,GAC1CrD,SACA6D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDX,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACZ,YAAY,KAAK;yBAAC;oBAC7B;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMa,mBAAmBD,gBACrBE,AAAAA,IAAAA,oCAAAA,mCAAAA,AAAAA,EACE;oBACE,QAAQ;wBACNF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;wBAC3CA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;qBAC5C;oBACD,aACE,AAA8B,YAA9B,OAAOZ,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU;oBACpC,MAAMY;gBACR,GACAtB,4BAEF3C;gBAEJ,MAAMoE,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAchB,YAAY,MAAM;gBACtC,MAAMiB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJR,mBAAmBI,aACf,OACA,MAAMK,AAAAA,IAAAA,oCAAAA,qBAAAA,AAAAA,EACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAhB,YAAY,SAAS;gBAI7B,MAAMqB,mBAAmBF,yBACrBL,AAAAA,IAAAA,oCAAAA,mCAAAA,AAAAA,EACEK,wBACA7B,4BAEF3C;gBAEJ,MAAM2E,aAAa,CAAC,CAACD;gBAErB,IAAIE;gBACJ,MAAM5C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACqC,cAAc,CAACO,cAAc,CAACX,iBACjC,IAAI;oBACF7B,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBH,QAAQ;oBAC3ByB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCJ,aACA;wBACE,SAASpB;wBACT4B;oBACF,GACAtD,cACA6C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BmB,sBAAsBnB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiB+B,kCAAAA,YAAYA,EAC/BnB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRX,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBH,QAAQ;gBAC7B;gBAGF,MAAM1C,UACJyE,mBACAG,oBACAQ,oBACAE;gBAEF,IAAItF,SAAS;oBACX,MAAMwF,uBAAuBzF,2BAA2BC;oBACxD,IAAIwF,sBAAsB;wBACxB,IAAItB,YACF,MAAM,IAAIqB,kCAAAA,YAAYA,CAACC,sBAAsBtB;wBAE/C,MAAM,IAAId,MAAMoC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B5F,iBAC/BmF,mBAAmB,cAAc;gBAGnC,IAAIU;gBAOJ,IACE1F,WACA,IAAI,CAAC,SAAS,IACd,CAACqF,cACA,EAACX,mBAAmB,CAACe,wBAAuB,KAC7C1B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,MAAM4B,gBAAkC;wBACtCC,KAAK,KAAK,CAAC5F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;wBAC/BuC,KAAK,KAAK,CAAC5F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;qBAChC;oBAED,MAAMwC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO5B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAc9C;oBAChB;oBAEF,IAAIpB,iBAAiBgG,UAAU;wBAC7BlG,MACE,uCACAoF,aACAc;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQd;4BACR,OAAOc;wBACT,GACAb;oBAEJ,OACErF,MACE,yDACAoF;gBAGN,EAAE,OAAOvB,OAAO;oBACd7D,MAAM,mCAAmC6D;gBAC3C;qBAEA7D,MAAM;gBAIV,IAAI,CAACK,SAAS;oBACZ,IAAIkE,YACF,MAAM,IAAIqB,kCAAAA,YAAYA,CACpB,CAAC,oBAAoB,EAAExB,YAAY,MAAM,EAAE,EAC3CG;oBAGJ,MAAM,IAAId,MAAM,CAAC,mBAAmB,EAAEW,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAI+B;gBAEJ,IAAIpB,mBAAmBT,6BACrB6B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,oBACE7B,4BAA4B,kBAAkB;oBAClD;gBACF;qBACK,IAAIa,YACTgB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO/B,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIsB,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPb;wBACA,aAAaS;oBACf;gBACF;gBAGF,MAAMK,gBAAgBhC,YAAY,aAAa;gBAC/C,MAAMiC,mBAAmBD,gBACrB;oBACE,GAAG/F,OAAO;oBACV,aAAa+F;gBACf,IACA/F;gBAEJ,IAAI+F,iBAAiB7B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAAC+B,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJlC,WAAWmC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKrD,UAAU,aAAa;wBAC9B;oBACF;oBACAmD;gBACF;YACF;QACF;QAEA,OAAO9B;IACT;IA3kBA,YAAY,EACVkC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTtE,WAAW,EACXuE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGtE;QACnB,IAAI,CAAC,eAAe,GAAGuE;IACzB;AAgkBF"}
+{"version":3,"file":"agent/task-builder.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/task-builder.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport {\n  findActionInActionSpaceOrThrow,\n  validateRequiredLocateFields,\n} from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ActionReadiness } from './action-readiness';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifLocateParamHasLocatedPixelResult,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [element.center?.[0], element.center?.[1]];\n  if (\n    element.center?.length !== 2 ||\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<DetailedLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n  actionReadiness?: ActionReadiness;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n  private readonly actionReadiness?: ActionReadiness;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n    actionReadiness,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n    this.actionReadiness = actionReadiness;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<DetailedLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<DetailedLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    validateRequiredLocateFields(param, action.paramSchema);\n\n    locateFields.forEach((field) => {\n      if (param?.[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelResult is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      planningFeedbackMaxLength: action.planningFeedbackMaxLength,\n      executor: async (taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        validateRequiredLocateFields(param, action.paramSchema);\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n        const parsedParam = (() => {\n          if (!action.paramSchema) return param;\n          try {\n            return parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        })();\n\n        const useDefaultWait = !this.actionReadiness;\n        taskContext.task.actionReadiness = useDefaultWait\n          ? 'default'\n          : 'custom';\n        context.abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = useDefaultWait\n          ? (action.delayBeforeRunner ?? 200)\n          : 0;\n        try {\n          await Promise.all([\n            this.interface.beforeInvokeAction?.(action.name, param),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n        }\n\n        context.abortSignal?.throwIfAborted();\n        setTimingFieldOnce(timing, 'callActionStart');\n        debug('calling action', action.name);\n        const actionResult = await action.call.call(\n          this.interface,\n          parsedParam,\n          useDefaultWait\n            ? taskContext\n            : { ...taskContext, skipDefaultWait: true },\n        );\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n        context.abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n        setTimingFieldOnce(timing, 'waitForActionReadyStart');\n        try {\n          if (useDefaultWait) {\n            const delayAfterRunner =\n              action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n            if (delayAfterRunner > 0) await sleep(delayAfterRunner);\n            await this.interface.defaultActionWait?.(action.name, parsedParam);\n          } else {\n            await this.actionReadiness!.wait(\n              {\n                id: taskContext.task.taskId,\n                name: action.name,\n                param: parsedParam,\n              },\n              context.abortSignal,\n            );\n          }\n        } finally {\n          setTimingFieldOnce(timing, 'waitForActionReadyEnd');\n        }\n        context.abortSignal?.throwIfAborted();\n        try {\n          await this.interface.afterInvokeAction?.(action.name, parsedParam);\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n        }\n        return { output: actionResult };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<DetailedLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(\n          locateParam,\n        )\n          ? locateParam\n          : undefined;\n\n        assert(\n          locateParam?.prompt || paramWithLocatedPixelResult,\n          `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(\n            locateParam,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            // The deepLocate search-area call belongs to the same locate task,\n            // so it shares the companion usage intent.\n            task.searchAreaUsage = withUsageIntent(\n              dump.taskInfo.searchAreaUsage,\n              'default',\n            );\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelResult\n          ? matchElementFromPlan(paramWithLocatedPixelResult)\n          : undefined;\n\n        // from locatedPixelResult (direct plan hit)\n        // when deepLocate is enabled, locatedPixelResult should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = locateParam.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          locateParam.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [locateParam.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? transformLogicalElementToScreenshot(\n              {\n                center: [\n                  rectFromXpath.left + rectFromXpath.width / 2,\n                  rectFromXpath.top + rectFromXpath.height / 2,\n                ],\n                description:\n                  typeof locateParam.prompt === 'string'\n                    ? locateParam.prompt\n                    : locateParam.prompt?.prompt || '',\n                rect: rectFromXpath,\n              },\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = locateParam.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                locateParam.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              locateParam,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          locateParam.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              const pointForCache: [number, number] = [\n                Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                Math.round(element.center[1] / shrunkShotToLogicalRatio),\n              ];\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof locateParam.prompt === 'string'\n                      ? locateParam.prompt\n                      : locateParam.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${locateParam.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${locateParam.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelResult) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelResult:\n                paramWithLocatedPixelResult.locatedPixelResult,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: locateParam.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        const promptDisplay = locateParam.promptDisplay;\n        const elementForAction = promptDisplay\n          ? {\n              ...element,\n              description: promptDisplay,\n            }\n          : element;\n\n        if (promptDisplay && locateDump?.matchedElement) {\n          locateDump.matchedElement = locateDump.matchedElement.map(\n            (matchedElement) => ({\n              ...matchedElement,\n              description: promptDisplay,\n            }),\n          );\n        }\n\n        onResult?.(elementForAction);\n\n        return {\n          output: {\n            element: {\n              ...elementForAction,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","hasNonEmptyCache","cache","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","validateRequiredLocateFields","field","ifLocateParamHasLocatedPixelResult","locateTask","result","task","taskContext","timing","uiContext","assert","shrunkShotToLogicalRatio","Error","parsedParam","parseActionParam","error","useDefaultWait","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","String","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelResult","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","transformLogicalElementToScreenshot","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","promptDisplay","elementForAction","matchedElement","interfaceInstance","service","taskCache","waitAfterAction","actionReadiness"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;ACgCA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPP,OAAO,IAAI,CAACO,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASC,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QAACD,QAAQ,MAAM,EAAE,CAAC,EAAE;QAAEA,QAAQ,MAAM,EAAE,CAAC,EAAE;KAAC;IACzD,IACEA,QAAQ,MAAM,EAAE,WAAW,KAC3BC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;AAG1E;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AA0BO,MAAMC;IA4BX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,AAAAA,IAAAA,mCAAAA,8BAAAA,AAAAA,EAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BH,OAAO,WAAW;QAEnEI,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6B7B,OAAOyB,OAAO,WAAW;QAEtDE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,OAAO,CAAC8B,MAAM,EAAE;gBAGlB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnDzC,MACE,uCACA,CAAC,YAAY,EAAEkC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,sBAAsB,EAAEwB,AAAAA,IAAAA,oCAAAA,kCAAAA,AAAAA,EAAmC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OACE3C,MAAM,CAAC,OAAO,EAAEyC,MAAM,6BAA6B,EAAEP,UAAU;QAEnE;QAEA,MAAMW,OAKF;YACF,MAAM;YACN,SAASX;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,2BAA2BO,OAAO,yBAAyB;YAC3D,UAAU,OAAOU;gBACf,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtC9C,MACE,oBACAkC,UACAvB,OACA,CAAC,4BAA4B,EAAEmC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCG,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElBR,IAAAA,mCAAAA,4BAAAA,AAAAA,EAA6B7B,OAAOyB,OAAO,WAAW;gBAEtD,MAAM,EAAEc,wBAAwB,EAAE,GAAGF;gBACrC,IAAIE,AAA6BnC,WAA7BmC,0BACF,MAAM,IAAIC,MACR;gBAGJ,MAAMC,cAAe,AAAC;oBACpB,IAAI,CAAChB,OAAO,WAAW,EAAE,OAAOzB;oBAChC,IAAI;wBACF,OAAO0C,AAAAA,IAAAA,yBAAAA,gBAAAA,AAAAA,EAAiB1C,OAAOyB,OAAO,WAAW,EAAE;4BACjDc;wBACF;oBACF,EAAE,OAAOI,OAAY;wBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAEf,OAAO,IAAI,CAAC,EAAE,EAAEkB,MAAM,OAAO,CAAC,cAAc,EAAE7C,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAO2C;wBAAM;oBAEnB;gBACF;gBAEA,MAAMC,iBAAiB,CAAC,IAAI,CAAC,eAAe;gBAC5CT,YAAY,IAAI,CAAC,eAAe,GAAGS,iBAC/B,YACA;gBACJ7B,QAAQ,WAAW,EAAE;gBAErB8B,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;gBAC3B,MAAMU,oBAAoBF,iBACrBnB,OAAO,iBAAiB,IAAI,MAC7B;gBACJ,IAAI;oBACF,MAAMsB,QAAQ,GAAG,CAAC;wBAChB,IAAI,CAAC,SAAS,CAAC,kBAAkB,GAAGtB,OAAO,IAAI,EAAEzB;wBACjD8C,oBAAoB,IAChBE,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAM,IAAIT,MACR,CAAC,wCAAwC,EAAEf,OAAO,IAAI,CAAC,EAAE,EAAEwB,eAAe,WAAWC,OAAOD,gBAAgB,EAC5G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRJ,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;gBAC7B;gBAEArB,QAAQ,WAAW,EAAE;gBACrB8B,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;gBAC3B/C,MAAM,kBAAkBoC,OAAO,IAAI;gBACnC,MAAM0B,eAAe,MAAM1B,OAAO,IAAI,CAAC,IAAI,CACzC,IAAI,CAAC,SAAS,EACdgB,aACAG,iBACIT,cACA;oBAAE,GAAGA,WAAW;oBAAE,iBAAiB;gBAAK;gBAE9CU,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;gBAC3B/C,MAAM,iBAAiBoC,OAAO,IAAI,EAAE,WAAW0B;gBAC/CpC,QAAQ,WAAW,EAAE;gBAErB8B,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;gBAC3BS,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;gBAC3B,IAAI;oBACF,IAAIQ,gBAAgB;wBAClB,MAAMQ,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;wBACrD,IAAI2B,mBAAmB,GAAG,MAAMJ,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMI;wBACtC,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GAAG3B,OAAO,IAAI,EAAEgB;oBACxD,OACE,MAAM,IAAI,CAAC,eAAe,CAAE,IAAI,CAC9B;wBACE,IAAIN,YAAY,IAAI,CAAC,MAAM;wBAC3B,MAAMV,OAAO,IAAI;wBACjB,OAAOgB;oBACT,GACA1B,QAAQ,WAAW;gBAGzB,SAAU;oBACR8B,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;gBAC7B;gBACArB,QAAQ,WAAW,EAAE;gBACrB,IAAI;oBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GAAGU,OAAO,IAAI,EAAEgB;gBACxD,EAAE,OAAOQ,eAAoB;oBAC3B,MAAM,IAAIT,MACR,CAAC,uCAAuC,EAAEf,OAAO,IAAI,CAAC,EAAE,EAAEwB,eAAe,WAAWC,OAAOD,gBAAgB,EAC3G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRJ,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;gBAC7B;gBACA,OAAO;oBAAE,QAAQe;gBAAa;YAChC;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACmB;IACrB;IAEQ,iBACNhB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOiB;gBACf,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMuB,8BAA8B3B,AAAAA,IAAAA,oCAAAA,kCAAAA,AAAAA,EAClCyB,eAEEA,cACApD;gBAEJkC,IAAAA,sBAAAA,MAAAA,AAAAA,EACEkB,aAAa,UAAUE,6BACvB,CAAC,mEAAmE,EAAE5D,KAAK,SAAS,CAClF0D,cACC;gBAGL,IAAI,CAACnB,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;gBAElB,MAAM,EAAEE,wBAAwB,EAAE,GAAGF;gBAErC,IAAIE,AAA6BnC,WAA7BmC,0BACF,MAAM,IAAIC,MACR;gBAIJ,IAAImB;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb5B,KAAK,GAAG,GAAG;wBACT4B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA5B,KAAK,KAAK,GAAG6B,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD5B,KAAK,UAAU,GAAG4B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjB5B,KAAK,eAAe,GAAG6B,AAAAA,IAAAA,yCAAAA,eAAAA,AAAAA,EACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjB5B,KAAK,iBAAiB,GAAG4B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,8BACvBO,AAAAA,IAAAA,oCAAAA,oBAAAA,AAAAA,EAAqBP,+BACrBtD;gBAKJ,MAAM8D,kBAAkBV,YAAY,UAAU,GAC1CpD,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDX,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACZ,YAAY,KAAK;yBAAC;oBAC7B;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMa,mBAAmBD,gBACrBE,AAAAA,IAAAA,oCAAAA,mCAAAA,AAAAA,EACE;oBACE,QAAQ;wBACNF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;wBAC3CA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;qBAC5C;oBACD,aACE,AAA8B,YAA9B,OAAOZ,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU;oBACpC,MAAMY;gBACR,GACA7B,4BAEFnC;gBAEJ,MAAMmE,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAchB,YAAY,MAAM;gBACtC,MAAMiB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJR,mBAAmBI,aACf,OACA,MAAMK,AAAAA,IAAAA,oCAAAA,qBAAAA,AAAAA,EACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAhB,YAAY,SAAS;gBAI7B,MAAMqB,mBAAmBF,yBACrBL,AAAAA,IAAAA,oCAAAA,mCAAAA,AAAAA,EACEK,wBACApC,4BAEFnC;gBAEJ,MAAM0E,aAAa,CAAC,CAACD;gBAErB,IAAIE;gBACJ,MAAM3C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACO,cAAc,CAACX,iBACjC,IAAI;oBACFtB,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;oBAC3BwB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCJ,aACA;wBACE,SAASnB;wBACT2B;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BmB,sBAAsBnB,aAAa,OAAO;gBAC5C,EAAE,OAAOjB,OAAO;oBACd,IAAIA,iBAAiBqC,kCAAAA,YAAYA,EAC/BnB,UAAUlB,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRE,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBT,QAAQ;gBAC7B;gBAGF,MAAM1C,UACJwE,mBACAG,oBACAQ,oBACAE;gBAEF,IAAIrF,SAAS;oBACX,MAAMuF,uBAAuBxF,2BAA2BC;oBACxD,IAAIuF,sBAAsB;wBACxB,IAAItB,YACF,MAAM,IAAIqB,kCAAAA,YAAYA,CAACC,sBAAsBtB;wBAE/C,MAAM,IAAInB,MAAMyC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B3F,iBAC/BkF,mBAAmB,cAAc;gBAGnC,IAAIU;gBAOJ,IACEzF,WACA,IAAI,CAAC,SAAS,IACd,CAACoF,cACA,EAACX,mBAAmB,CAACe,wBAAuB,KAC7C1B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,MAAM4B,gBAAkC;wBACtCC,KAAK,KAAK,CAAC3F,QAAQ,MAAM,CAAC,EAAE,GAAG6C;wBAC/B8C,KAAK,KAAK,CAAC3F,QAAQ,MAAM,CAAC,EAAE,GAAG6C;qBAChC;oBAED,MAAM+C,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO5B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAc7C;oBAChB;oBAEF,IAAIpB,iBAAiB+F,UAAU;wBAC7BjG,MACE,uCACAmF,aACAc;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQd;4BACR,OAAOc;wBACT,GACAb;oBAEJ,OACEpF,MACE,yDACAmF;gBAGN,EAAE,OAAO7B,OAAO;oBACdtD,MAAM,mCAAmCsD;gBAC3C;qBAEAtD,MAAM;gBAIV,IAAI,CAACK,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIqB,kCAAAA,YAAYA,CACpB,CAAC,oBAAoB,EAAExB,YAAY,MAAM,EAAE,EAC3CG;oBAGJ,MAAM,IAAInB,MAAM,CAAC,mBAAmB,EAAEgB,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAI+B;gBAEJ,IAAIpB,mBAAmBT,6BACrB6B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,oBACE7B,4BAA4B,kBAAkB;oBAClD;gBACF;qBACK,IAAIa,YACTgB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO/B,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIsB,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPb;wBACA,aAAaS;oBACf;gBACF;gBAGF,MAAMK,gBAAgBhC,YAAY,aAAa;gBAC/C,MAAMiC,mBAAmBD,gBACrB;oBACE,GAAG9F,OAAO;oBACV,aAAa8F;gBACf,IACA9F;gBAEJ,IAAI8F,iBAAiB7B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAAC+B,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJlC,WAAWmC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKpD,UAAU,aAAa;wBAC9B;oBACF;oBACAkD;gBACF;YACF;QACF;QAEA,OAAO9B;IACT;IA7kBA,YAAY,EACVkC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTrE,WAAW,EACXsE,eAAe,EACfC,eAAe,EACC,CAAE;QAlBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QACA,uBAAiB,mBAAjB;QAUE,IAAI,CAAC,SAAS,GAAGJ;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGrE;QACnB,IAAI,CAAC,eAAe,GAAGsE;QACvB,IAAI,CAAC,eAAe,GAAGC;IACzB;AAgkBF"}

package/dist/lib/agent/tasks.js

@@ -40,6 +40,7 @@ const external_task_runner_js_namespaceObject = require("../task-runner.js");
 const external_types_js_namespaceObject = require("../types.js");
 const logger_namespaceObject = require("@midscene/shared/logger");
 const utils_namespaceObject = require("@midscene/shared/utils");
+const external_action_readiness_js_namespaceObject = require("./action-readiness.js");
 const external_execution_session_js_namespaceObject = require("./execution-session.js");
 const external_file_chooser_js_namespaceObject = require("./file-chooser.js");
 const external_progress_index_js_namespaceObject = require("./progress/index.js");
@@ -195,7 +196,9 @@ class TaskExecutor {
             ...defaultModel,
             executionId: runner.id
         };
-        const { tasks } = await this.convertPlanToExecutable(plans, executionPlanningModel, executionDefaultModel);
+        const { tasks } = await this.convertPlanToExecutable(plans, executionPlanningModel, executionDefaultModel, {
+            abortSignal: options?.abortSignal
+        });
         const result = await session.appendAndRun(tasks);
         const { output } = result ?? {};
         return {
@@ -204,6 +207,7 @@ class TaskExecutor {
         };
     }
     async action(userPrompt, planningModel, defaultModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', fileChooserAccept, deepLocate, abortSignal, reportOptions) {
+        abortSignal?.throwIfAborted();
         return (0, external_file_chooser_js_namespaceObject.withFileChooser)(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, defaultModel, aiActContext, cacheable, replanningCycleLimitOverride, effort, deepLocate, abortSignal, reportOptions));
     }
     invalidateFailedCacheHitLocates(runner, fromIndex) {
@@ -426,6 +430,7 @@ class TaskExecutor {
                 await session.appendAndRun(executables.tasks);
                 this.setPendingFeedbackMessage(conversationHistory, initialTimeString, this.collectPlanningFeedback(runner.tasks.slice(taskCountBeforeRun)));
             } catch (error) {
+                if ((0, external_action_readiness_js_namespaceObject.isActionReadinessError)(error)) throw error;
                 errorCountInOnePlanningLoop++;
                 const timeString = await this.getTimeString();
                 this.setPendingFeedbackMessage(conversationHistory, timeString, `Error executing running tasks: ${error?.message || String(error)}`);
@@ -547,6 +552,7 @@ class TaskExecutor {
         return queryTask;
     }
     async createTypeQueryExecution(type, demand, modelRuntime, opt, multimodalPrompt, executionOptions) {
+        executionOptions?.abortSignal?.throwIfAborted();
         const session = this.createExecutionSession((0, external_ui_utils_js_namespaceObject.taskTitleStr)(type, 'string' == typeof demand ? demand : JSON.stringify(demand)), {
             ...executionOptions?.uiContext ? {
                 uiContext: executionOptions.uiContext
@@ -646,7 +652,8 @@ class TaskExecutor {
             service,
             taskCache: opts.taskCache,
             actionSpace: this.getActionSpace(),
-            waitAfterAction: opts.waitAfterAction
+            waitAfterAction: opts.waitAfterAction,
+            actionReadiness: opts.actionReadiness
         });
     }
 }