@midscene/core 1.15.2-beta-20261012123219.0 → 1.15.2

package/dist/es/agent/task-builder.mjs

@@ -1,8 +1,8 @@
 import { findAllMidsceneLocatorField, parseActionParam } from "../ai-model/index.mjs";
 import { findActionInActionSpaceOrThrow, validateRequiredLocateFields } from "../common.mjs";
+import { defineActionSleep } from "../device/index.mjs";
 import { setTimingFieldOnce } from "../task-timing.mjs";
 import { ServiceError } from "../types.mjs";
-import { sleep } from "../utils.mjs";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { withUsageIntent } from "./usage-intent.mjs";
@@ -72,6 +72,7 @@ class TaskBuilder {
         ]);
         const defaultHandler = (plan)=>this.handleActionPlan(plan, context);
         for (const plan of plans){
+            context.abortSignal?.throwIfAborted();
             const handler = planHandlers.get(plan.type) ?? defaultHandler;
             await handler(plan);
         }
@@ -117,6 +118,7 @@ class TaskBuilder {
             param: plan.param,
             planningFeedbackMaxLength: action.planningFeedbackMaxLength,
             executor: async (taskContext)=>{
+                const abortSignal = context.abortSignal ?? taskContext.signal;
                 const timing = taskContext.task.timing;
                 debug('executing action', planType, param, `taskContext.element.center: ${taskContext.element?.center}`);
                 const uiContext = taskContext.uiContext;
@@ -138,14 +140,18 @@ class TaskBuilder {
                 })();
                 const useDefaultWait = !this.actionReadiness;
                 taskContext.task.actionReadiness = useDefaultWait ? 'default' : 'custom';
-                context.abortSignal?.throwIfAborted();
+                abortSignal?.throwIfAborted();
                 setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');
                 const delayBeforeRunner = useDefaultWait ? action.delayBeforeRunner ?? 200 : 0;
                 try {
-                    await Promise.all([
-                        this.interface.beforeInvokeAction?.(action.name, param),
-                        delayBeforeRunner > 0 ? sleep(delayBeforeRunner) : Promise.resolve()
+                    const preparation = await Promise.allSettled([
+                        this.interface.beforeInvokeAction?.(action.name, param, abortSignal),
+                        delayBeforeRunner > 0 ? defineActionSleep(abortSignal).call({
+                            timeMs: delayBeforeRunner
+                        }) : Promise.resolve()
                     ]);
+                    const rejected = preparation.find((item)=>'rejected' === item.status);
+                    if (rejected?.status === 'rejected') throw rejected.reason;
                 } catch (originalError) {
                     throw new Error(`error in running beforeInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`, {
                         cause: originalError
@@ -153,34 +159,40 @@ class TaskBuilder {
                 } finally{
                     setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');
                 }
-                context.abortSignal?.throwIfAborted();
+                abortSignal?.throwIfAborted();
                 setTimingFieldOnce(timing, 'callActionStart');
                 debug('calling action', action.name);
-                const actionResult = await action.call.call(this.interface, parsedParam, useDefaultWait ? taskContext : {
+                const actionResult = await action.call.call(this.interface, parsedParam, useDefaultWait ? {
                     ...taskContext,
+                    signal: abortSignal
+                } : {
+                    ...taskContext,
+                    signal: abortSignal,
                     skipDefaultWait: true
                 });
                 setTimingFieldOnce(timing, 'callActionEnd');
                 debug('called action', action.name, 'result:', actionResult);
-                context.abortSignal?.throwIfAborted();
+                abortSignal?.throwIfAborted();
                 setTimingFieldOnce(timing, 'afterInvokeActionHookStart');
                 setTimingFieldOnce(timing, 'waitForActionReadyStart');
                 try {
                     if (useDefaultWait) {
                         const delayAfterRunner = action.delayAfterRunner ?? this.waitAfterAction ?? 300;
-                        if (delayAfterRunner > 0) await sleep(delayAfterRunner);
-                        await this.interface.defaultActionWait?.(action.name, parsedParam);
+                        if (delayAfterRunner > 0) await defineActionSleep(abortSignal).call({
+                            timeMs: delayAfterRunner
+                        });
+                        await this.interface.defaultActionWait?.(action.name, parsedParam, abortSignal);
                     } else await this.actionReadiness.wait({
                         id: taskContext.task.taskId,
                         name: action.name,
                         param: parsedParam
-                    }, context.abortSignal);
+                    }, abortSignal, this.operationResource);
                 } finally{
                     setTimingFieldOnce(timing, 'waitForActionReadyEnd');
                 }
-                context.abortSignal?.throwIfAborted();
+                abortSignal?.throwIfAborted();
                 try {
-                    await this.interface.afterInvokeAction?.(action.name, parsedParam);
+                    await this.interface.afterInvokeAction?.(action.name, parsedParam, abortSignal);
                 } catch (originalError) {
                     throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`, {
                         cause: originalError
@@ -196,7 +208,7 @@ class TaskBuilder {
         context.tasks.push(task);
     }
     createLocateTask(plan, detailedLocateParam, context, onResult) {
-        const { cacheable, defaultModel, deepLocate, abortSignal } = context;
+        const { cacheable, defaultModel, deepLocate } = context;
         let locateParam = normalizeLocateParam(detailedLocateParam);
         if (void 0 !== cacheable) locateParam = {
             ...locateParam,
@@ -212,11 +224,16 @@ class TaskBuilder {
             param: locateParam,
             thought: plan.thought,
             executor: async (taskContext)=>{
+                const abortSignal = context.abortSignal ?? taskContext.signal;
+                abortSignal?.throwIfAborted();
                 const { task } = taskContext;
                 let { uiContext } = taskContext;
                 const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(locateParam) ? locateParam : void 0;
                 assert(locateParam?.prompt || paramWithLocatedPixelResult, `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(locateParam)}`);
-                if (!uiContext) uiContext = await this.service.contextRetrieverFn();
+                if (!uiContext) {
+                    uiContext = await this.service.contextRetrieverFn(abortSignal);
+                    abortSignal?.throwIfAborted();
+                }
                 assert(uiContext, 'uiContext is required for Service task');
                 const { shrunkShotToLogicalRatio } = uiContext;
                 if (void 0 === shrunkShotToLogicalRatio) throw new Error('shrunkShotToLogicalRatio is not defined in locate task');
@@ -360,19 +377,21 @@ class TaskBuilder {
         };
         return taskLocator;
     }
-    constructor({ interfaceInstance, service, taskCache, actionSpace, waitAfterAction, actionReadiness }){
+    constructor({ interfaceInstance, service, taskCache, actionSpace, waitAfterAction, actionReadiness, operationResource }){
         _define_property(this, "interface", void 0);
         _define_property(this, "service", void 0);
         _define_property(this, "taskCache", void 0);
         _define_property(this, "actionSpace", void 0);
         _define_property(this, "waitAfterAction", void 0);
         _define_property(this, "actionReadiness", void 0);
+        _define_property(this, "operationResource", void 0);
         this.interface = interfaceInstance;
         this.service = service;
         this.taskCache = taskCache;
         this.actionSpace = actionSpace;
         this.waitAfterAction = waitAfterAction;
         this.actionReadiness = actionReadiness;
+        this.operationResource = operationResource ?? interfaceInstance;
     }
 }
 export { TaskBuilder, locatePlanForLocate };

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport {\n  findActionInActionSpaceOrThrow,\n  validateRequiredLocateFields,\n} from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ActionReadiness } from './action-readiness';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifLocateParamHasLocatedPixelResult,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [element.center?.[0], element.center?.[1]];\n  if (\n    element.center?.length !== 2 ||\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<DetailedLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n  actionReadiness?: ActionReadiness;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n  private readonly actionReadiness?: ActionReadiness;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n    actionReadiness,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n    this.actionReadiness = actionReadiness;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<DetailedLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<DetailedLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    validateRequiredLocateFields(param, action.paramSchema);\n\n    locateFields.forEach((field) => {\n      if (param?.[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelResult is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      planningFeedbackMaxLength: action.planningFeedbackMaxLength,\n      executor: async (taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        validateRequiredLocateFields(param, action.paramSchema);\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n        const parsedParam = (() => {\n          if (!action.paramSchema) return param;\n          try {\n            return parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        })();\n\n        const useDefaultWait = !this.actionReadiness;\n        taskContext.task.actionReadiness = useDefaultWait\n          ? 'default'\n          : 'custom';\n        context.abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = useDefaultWait\n          ? (action.delayBeforeRunner ?? 200)\n          : 0;\n        try {\n          await Promise.all([\n            this.interface.beforeInvokeAction?.(action.name, param),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n        }\n\n        context.abortSignal?.throwIfAborted();\n        setTimingFieldOnce(timing, 'callActionStart');\n        debug('calling action', action.name);\n        const actionResult = await action.call.call(\n          this.interface,\n          parsedParam,\n          useDefaultWait\n            ? taskContext\n            : { ...taskContext, skipDefaultWait: true },\n        );\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n        context.abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n        setTimingFieldOnce(timing, 'waitForActionReadyStart');\n        try {\n          if (useDefaultWait) {\n            const delayAfterRunner =\n              action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n            if (delayAfterRunner > 0) await sleep(delayAfterRunner);\n            await this.interface.defaultActionWait?.(action.name, parsedParam);\n          } else {\n            await this.actionReadiness!.wait(\n              {\n                id: taskContext.task.taskId,\n                name: action.name,\n                param: parsedParam,\n              },\n              context.abortSignal,\n            );\n          }\n        } finally {\n          setTimingFieldOnce(timing, 'waitForActionReadyEnd');\n        }\n        context.abortSignal?.throwIfAborted();\n        try {\n          await this.interface.afterInvokeAction?.(action.name, parsedParam);\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n        }\n        return { output: actionResult };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<DetailedLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(\n          locateParam,\n        )\n          ? locateParam\n          : undefined;\n\n        assert(\n          locateParam?.prompt || paramWithLocatedPixelResult,\n          `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(\n            locateParam,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            // The deepLocate search-area call belongs to the same locate task,\n            // so it shares the companion usage intent.\n            task.searchAreaUsage = withUsageIntent(\n              dump.taskInfo.searchAreaUsage,\n              'default',\n            );\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelResult\n          ? matchElementFromPlan(paramWithLocatedPixelResult)\n          : undefined;\n\n        // from locatedPixelResult (direct plan hit)\n        // when deepLocate is enabled, locatedPixelResult should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = locateParam.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          locateParam.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [locateParam.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? transformLogicalElementToScreenshot(\n              {\n                center: [\n                  rectFromXpath.left + rectFromXpath.width / 2,\n                  rectFromXpath.top + rectFromXpath.height / 2,\n                ],\n                description:\n                  typeof locateParam.prompt === 'string'\n                    ? locateParam.prompt\n                    : locateParam.prompt?.prompt || '',\n                rect: rectFromXpath,\n              },\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = locateParam.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                locateParam.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              locateParam,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          locateParam.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              const pointForCache: [number, number] = [\n                Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                Math.round(element.center[1] / shrunkShotToLogicalRatio),\n              ];\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof locateParam.prompt === 'string'\n                      ? locateParam.prompt\n                      : locateParam.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${locateParam.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${locateParam.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelResult) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelResult:\n                paramWithLocatedPixelResult.locatedPixelResult,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: locateParam.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        const promptDisplay = locateParam.promptDisplay;\n        const elementForAction = promptDisplay\n          ? {\n              ...element,\n              description: promptDisplay,\n            }\n          : element;\n\n        if (promptDisplay && locateDump?.matchedElement) {\n          locateDump.matchedElement = locateDump.matchedElement.map(\n            (matchedElement) => ({\n              ...matchedElement,\n              description: promptDisplay,\n            }),\n          );\n        }\n\n        onResult?.(elementForAction);\n\n        return {\n          output: {\n            element: {\n              ...elementForAction,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","validateRequiredLocateFields","field","ifLocateParamHasLocatedPixelResult","locateTask","result","task","taskContext","timing","uiContext","assert","shrunkShotToLogicalRatio","Error","parsedParam","parseActionParam","error","useDefaultWait","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","String","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelResult","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","transformLogicalElementToScreenshot","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","promptDisplay","elementForAction","matchedElement","interfaceInstance","service","taskCache","waitAfterAction","actionReadiness"],"mappings":";;;;;;;;;;;;;;;;;;;AAsCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QAACD,QAAQ,MAAM,EAAE,CAAC,EAAE;QAAEA,QAAQ,MAAM,EAAE,CAAC,EAAE;KAAC;IACzD,IACEA,QAAQ,MAAM,EAAE,WAAW,KAC3BC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;AAG1E;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AA0BO,MAAMC;IA4BX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnEI,6BAA6B7B,OAAOyB,OAAO,WAAW;QAEtDE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,OAAO,CAAC8B,MAAM,EAAE;gBAGlB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,sBAAsB,EAAEwB,mCAAmC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OACE5C,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;QAEnE;QAEA,MAAMW,OAKF;YACF,MAAM;YACN,SAASX;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,2BAA2BO,OAAO,yBAAyB;YAC3D,UAAU,OAAOU;gBACf,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtC/C,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEmC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCG,OAAOD,WAAW;gBAElBR,6BAA6B7B,OAAOyB,OAAO,WAAW;gBAEtD,MAAM,EAAEc,wBAAwB,EAAE,GAAGF;gBACrC,IAAIE,AAA6BnC,WAA7BmC,0BACF,MAAM,IAAIC,MACR;gBAGJ,MAAMC,cAAe,AAAC;oBACpB,IAAI,CAAChB,OAAO,WAAW,EAAE,OAAOzB;oBAChC,IAAI;wBACF,OAAO0C,iBAAiB1C,OAAOyB,OAAO,WAAW,EAAE;4BACjDc;wBACF;oBACF,EAAE,OAAOI,OAAY;wBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAEf,OAAO,IAAI,CAAC,EAAE,EAAEkB,MAAM,OAAO,CAAC,cAAc,EAAE7C,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAO2C;wBAAM;oBAEnB;gBACF;gBAEA,MAAMC,iBAAiB,CAAC,IAAI,CAAC,eAAe;gBAC5CT,YAAY,IAAI,CAAC,eAAe,GAAGS,iBAC/B,YACA;gBACJ7B,QAAQ,WAAW,EAAE;gBAErB8B,mBAAmBT,QAAQ;gBAC3B,MAAMU,oBAAoBF,iBACrBnB,OAAO,iBAAiB,IAAI,MAC7B;gBACJ,IAAI;oBACF,MAAMsB,QAAQ,GAAG,CAAC;wBAChB,IAAI,CAAC,SAAS,CAAC,kBAAkB,GAAGtB,OAAO,IAAI,EAAEzB;wBACjD8C,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAM,IAAIT,MACR,CAAC,wCAAwC,EAAEf,OAAO,IAAI,CAAC,EAAE,EAAEwB,eAAe,WAAWC,OAAOD,gBAAgB,EAC5G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRJ,mBAAmBT,QAAQ;gBAC7B;gBAEArB,QAAQ,WAAW,EAAE;gBACrB8B,mBAAmBT,QAAQ;gBAC3BhD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAM0B,eAAe,MAAM1B,OAAO,IAAI,CAAC,IAAI,CACzC,IAAI,CAAC,SAAS,EACdgB,aACAG,iBACIT,cACA;oBAAE,GAAGA,WAAW;oBAAE,iBAAiB;gBAAK;gBAE9CU,mBAAmBT,QAAQ;gBAC3BhD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW0B;gBAC/CpC,QAAQ,WAAW,EAAE;gBAErB8B,mBAAmBT,QAAQ;gBAC3BS,mBAAmBT,QAAQ;gBAC3B,IAAI;oBACF,IAAIQ,gBAAgB;wBAClB,MAAMQ,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;wBACrD,IAAI2B,mBAAmB,GAAG,MAAMJ,MAAMI;wBACtC,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GAAG3B,OAAO,IAAI,EAAEgB;oBACxD,OACE,MAAM,IAAI,CAAC,eAAe,CAAE,IAAI,CAC9B;wBACE,IAAIN,YAAY,IAAI,CAAC,MAAM;wBAC3B,MAAMV,OAAO,IAAI;wBACjB,OAAOgB;oBACT,GACA1B,QAAQ,WAAW;gBAGzB,SAAU;oBACR8B,mBAAmBT,QAAQ;gBAC7B;gBACArB,QAAQ,WAAW,EAAE;gBACrB,IAAI;oBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GAAGU,OAAO,IAAI,EAAEgB;gBACxD,EAAE,OAAOQ,eAAoB;oBAC3B,MAAM,IAAIT,MACR,CAAC,uCAAuC,EAAEf,OAAO,IAAI,CAAC,EAAE,EAAEwB,eAAe,WAAWC,OAAOD,gBAAgB,EAC3G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRJ,mBAAmBT,QAAQ;gBAC7B;gBACA,OAAO;oBAAE,QAAQe;gBAAa;YAChC;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACmB;IACrB;IAEQ,iBACNhB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOiB;gBACf,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMuB,8BAA8B3B,mCAClCyB,eAEEA,cACApD;gBAEJkC,OACEkB,aAAa,UAAUE,6BACvB,CAAC,mEAAmE,EAAE5D,KAAK,SAAS,CAClF0D,cACC;gBAGL,IAAI,CAACnB,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDC,OAAOD,WAAW;gBAElB,MAAM,EAAEE,wBAAwB,EAAE,GAAGF;gBAErC,IAAIE,AAA6BnC,WAA7BmC,0BACF,MAAM,IAAIC,MACR;gBAIJ,IAAImB;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb5B,KAAK,GAAG,GAAG;wBACT4B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA5B,KAAK,KAAK,GAAG6B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD5B,KAAK,UAAU,GAAG4B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjB5B,KAAK,eAAe,GAAG6B,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjB5B,KAAK,iBAAiB,GAAG4B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,8BACvBO,qBAAqBP,+BACrBtD;gBAKJ,MAAM8D,kBAAkBV,YAAY,UAAU,GAC1CpD,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDX,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACZ,YAAY,KAAK;yBAAC;oBAC7B;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMa,mBAAmBD,gBACrBE,oCACE;oBACE,QAAQ;wBACNF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;wBAC3CA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;qBAC5C;oBACD,aACE,AAA8B,YAA9B,OAAOZ,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU;oBACpC,MAAMY;gBACR,GACA7B,4BAEFnC;gBAEJ,MAAMmE,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAchB,YAAY,MAAM;gBACtC,MAAMiB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJR,mBAAmBI,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAhB,YAAY,SAAS;gBAI7B,MAAMqB,mBAAmBF,yBACrBL,oCACEK,wBACApC,4BAEFnC;gBAEJ,MAAM0E,aAAa,CAAC,CAACD;gBAErB,IAAIE;gBACJ,MAAM3C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACO,cAAc,CAACX,iBACjC,IAAI;oBACFtB,mBAAmBT,QAAQ;oBAC3BwB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCJ,aACA;wBACE,SAASnB;wBACT2B;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BmB,sBAAsBnB,aAAa,OAAO;gBAC5C,EAAE,OAAOjB,OAAO;oBACd,IAAIA,iBAAiBqC,cACnBnB,UAAUlB,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRE,mBAAmBT,QAAQ;gBAC7B;gBAGF,MAAM1C,UACJwE,mBACAG,oBACAQ,oBACAE;gBAEF,IAAIrF,SAAS;oBACX,MAAMuF,uBAAuBxF,2BAA2BC;oBACxD,IAAIuF,sBAAsB;wBACxB,IAAItB,YACF,MAAM,IAAIqB,aAAaC,sBAAsBtB;wBAE/C,MAAM,IAAInB,MAAMyC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B5F,iBAC/BmF,mBAAmB,cAAc;gBAGnC,IAAIU;gBAOJ,IACEzF,WACA,IAAI,CAAC,SAAS,IACd,CAACoF,cACA,EAACX,mBAAmB,CAACe,wBAAuB,KAC7C1B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,MAAM4B,gBAAkC;wBACtCC,KAAK,KAAK,CAAC3F,QAAQ,MAAM,CAAC,EAAE,GAAG6C;wBAC/B8C,KAAK,KAAK,CAAC3F,QAAQ,MAAM,CAAC,EAAE,GAAG6C;qBAChC;oBAED,MAAM+C,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO5B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAc7C;oBAChB;oBAEF,IAAIrB,iBAAiBgG,UAAU;wBAC7BlG,MACE,uCACAoF,aACAc;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQd;4BACR,OAAOc;wBACT,GACAb;oBAEJ,OACErF,MACE,yDACAoF;gBAGN,EAAE,OAAO7B,OAAO;oBACdvD,MAAM,mCAAmCuD;gBAC3C;qBAEAvD,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIqB,aACR,CAAC,oBAAoB,EAAExB,YAAY,MAAM,EAAE,EAC3CG;oBAGJ,MAAM,IAAInB,MAAM,CAAC,mBAAmB,EAAEgB,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAI+B;gBAEJ,IAAIpB,mBAAmBT,6BACrB6B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,oBACE7B,4BAA4B,kBAAkB;oBAClD;gBACF;qBACK,IAAIa,YACTgB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO/B,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIsB,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPb;wBACA,aAAaS;oBACf;gBACF;gBAGF,MAAMK,gBAAgBhC,YAAY,aAAa;gBAC/C,MAAMiC,mBAAmBD,gBACrB;oBACE,GAAG9F,OAAO;oBACV,aAAa8F;gBACf,IACA9F;gBAEJ,IAAI8F,iBAAiB7B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAAC+B,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJlC,WAAWmC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKpD,UAAU,aAAa;wBAC9B;oBACF;oBACAkD;gBACF;YACF;QACF;QAEA,OAAO9B;IACT;IA7kBA,YAAY,EACVkC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTrE,WAAW,EACXsE,eAAe,EACfC,eAAe,EACC,CAAE;QAlBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QACA,uBAAiB,mBAAjB;QAUE,IAAI,CAAC,SAAS,GAAGJ;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGrE;QACnB,IAAI,CAAC,eAAe,GAAGsE;QACvB,IAAI,CAAC,eAAe,GAAGC;IACzB;AAgkBF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport {\n  findActionInActionSpaceOrThrow,\n  validateRequiredLocateFields,\n} from '@/common';\nimport { type AbstractInterface, defineActionSleep } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ActionReadiness } from './action-readiness';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifLocateParamHasLocatedPixelResult,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [element.center?.[0], element.center?.[1]];\n  if (\n    element.center?.length !== 2 ||\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<DetailedLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n  actionReadiness?: ActionReadiness;\n  operationResource?: object;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n  private readonly actionReadiness?: ActionReadiness;\n  private readonly operationResource: object;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n    actionReadiness,\n    operationResource,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n    this.actionReadiness = actionReadiness;\n    this.operationResource = operationResource ?? interfaceInstance;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<DetailedLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      context.abortSignal?.throwIfAborted();\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<DetailedLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    validateRequiredLocateFields(param, action.paramSchema);\n\n    locateFields.forEach((field) => {\n      if (param?.[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelResult is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      planningFeedbackMaxLength: action.planningFeedbackMaxLength,\n      executor: async (taskContext) => {\n        const abortSignal = context.abortSignal ?? taskContext.signal;\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        validateRequiredLocateFields(param, action.paramSchema);\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n        const parsedParam = (() => {\n          if (!action.paramSchema) return param;\n          try {\n            return parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        })();\n\n        const useDefaultWait = !this.actionReadiness;\n        taskContext.task.actionReadiness = useDefaultWait\n          ? 'default'\n          : 'custom';\n        abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = useDefaultWait\n          ? (action.delayBeforeRunner ?? 200)\n          : 0;\n        try {\n          const preparation = await Promise.allSettled([\n            this.interface.beforeInvokeAction?.(\n              action.name,\n              param,\n              abortSignal,\n            ),\n            delayBeforeRunner > 0\n              ? defineActionSleep(abortSignal).call({\n                  timeMs: delayBeforeRunner,\n                })\n              : Promise.resolve(),\n          ]);\n          const rejected = preparation.find(\n            (item) => item.status === 'rejected',\n          );\n          if (rejected?.status === 'rejected') throw rejected.reason;\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n        }\n\n        abortSignal?.throwIfAborted();\n        setTimingFieldOnce(timing, 'callActionStart');\n        debug('calling action', action.name);\n        const actionResult = await action.call.call(\n          this.interface,\n          parsedParam,\n          useDefaultWait\n            ? { ...taskContext, signal: abortSignal }\n            : { ...taskContext, signal: abortSignal, skipDefaultWait: true },\n        );\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n        abortSignal?.throwIfAborted();\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n        setTimingFieldOnce(timing, 'waitForActionReadyStart');\n        try {\n          if (useDefaultWait) {\n            const delayAfterRunner =\n              action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n            if (delayAfterRunner > 0)\n              await defineActionSleep(abortSignal).call({\n                timeMs: delayAfterRunner,\n              });\n            await this.interface.defaultActionWait?.(\n              action.name,\n              parsedParam,\n              abortSignal,\n            );\n          } else {\n            await this.actionReadiness!.wait(\n              {\n                id: taskContext.task.taskId,\n                name: action.name,\n                param: parsedParam,\n              },\n              abortSignal,\n              this.operationResource,\n            );\n          }\n        } finally {\n          setTimingFieldOnce(timing, 'waitForActionReadyEnd');\n        }\n        abortSignal?.throwIfAborted();\n        try {\n          await this.interface.afterInvokeAction?.(\n            action.name,\n            parsedParam,\n            abortSignal,\n          );\n        } catch (originalError: any) {\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalError?.message || String(originalError)}`,\n            { cause: originalError },\n          );\n        } finally {\n          setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n        }\n        return { output: actionResult };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<DetailedLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (taskContext) => {\n        const abortSignal = context.abortSignal ?? taskContext.signal;\n        abortSignal?.throwIfAborted();\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(\n          locateParam,\n        )\n          ? locateParam\n          : undefined;\n\n        assert(\n          locateParam?.prompt || paramWithLocatedPixelResult,\n          `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(\n            locateParam,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn(abortSignal);\n          abortSignal?.throwIfAborted();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawAssistantOutput: dump.taskInfo?.rawAssistantOutput,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            // The deepLocate search-area call belongs to the same locate task,\n            // so it shares the companion usage intent.\n            task.searchAreaUsage = withUsageIntent(\n              dump.taskInfo.searchAreaUsage,\n              'default',\n            );\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelResult\n          ? matchElementFromPlan(paramWithLocatedPixelResult)\n          : undefined;\n\n        // from locatedPixelResult (direct plan hit)\n        // when deepLocate is enabled, locatedPixelResult should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = locateParam.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          locateParam.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [locateParam.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? transformLogicalElementToScreenshot(\n              {\n                center: [\n                  rectFromXpath.left + rectFromXpath.width / 2,\n                  rectFromXpath.top + rectFromXpath.height / 2,\n                ],\n                description:\n                  typeof locateParam.prompt === 'string'\n                    ? locateParam.prompt\n                    : locateParam.prompt?.prompt || '',\n                rect: rectFromXpath,\n              },\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = locateParam.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                locateParam.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              locateParam,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          locateParam.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              const pointForCache: [number, number] = [\n                Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                Math.round(element.center[1] / shrunkShotToLogicalRatio),\n              ];\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof locateParam.prompt === 'string'\n                      ? locateParam.prompt\n                      : locateParam.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${locateParam.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${locateParam.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelResult) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelResult:\n                paramWithLocatedPixelResult.locatedPixelResult,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: locateParam.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        const promptDisplay = locateParam.promptDisplay;\n        const elementForAction = promptDisplay\n          ? {\n              ...element,\n              description: promptDisplay,\n            }\n          : element;\n\n        if (promptDisplay && locateDump?.matchedElement) {\n          locateDump.matchedElement = locateDump.matchedElement.map(\n            (matchedElement) => ({\n              ...matchedElement,\n              description: promptDisplay,\n            }),\n          );\n        }\n\n        onResult?.(elementForAction);\n\n        return {\n          output: {\n            element: {\n              ...elementForAction,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","validateRequiredLocateFields","field","ifLocateParamHasLocatedPixelResult","locateTask","result","task","taskContext","abortSignal","timing","uiContext","assert","shrunkShotToLogicalRatio","Error","parsedParam","parseActionParam","error","useDefaultWait","setTimingFieldOnce","delayBeforeRunner","preparation","Promise","defineActionSleep","rejected","item","originalError","String","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","taskLocator","paramWithLocatedPixelResult","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","transformLogicalElementToScreenshot","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","promptDisplay","elementForAction","matchedElement","interfaceInstance","service","taskCache","waitAfterAction","actionReadiness","operationResource"],"mappings":";;;;;;;;;;;;;;;;;;;AAqCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QAACD,QAAQ,MAAM,EAAE,CAAC,EAAE;QAAEA,QAAQ,MAAM,EAAE,CAAC,EAAE;KAAC;IACzD,IACEA,QAAQ,MAAM,EAAE,WAAW,KAC3BC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;AAG1E;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AA2BO,MAAMC;IA+BX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxBM,QAAQ,WAAW,EAAE;YACrB,MAAMK,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnEI,6BAA6B7B,OAAOyB,OAAO,WAAW;QAEtDE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,OAAO,CAAC8B,MAAM,EAAE;gBAGlB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,sBAAsB,EAAEwB,mCAAmC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OACE5C,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;QAEnE;QAEA,MAAMW,OAKF;YACF,MAAM;YACN,SAASX;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,2BAA2BO,OAAO,yBAAyB;YAC3D,UAAU,OAAOU;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7D,MAAME,SAASF,YAAY,IAAI,CAAC,MAAM;gBAEtC/C,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEmC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAMG,YAAYH,YAAY,SAAS;gBACvCI,OAAOD,WAAW;gBAElBT,6BAA6B7B,OAAOyB,OAAO,WAAW;gBAEtD,MAAM,EAAEe,wBAAwB,EAAE,GAAGF;gBACrC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAGJ,MAAMC,cAAe,AAAC;oBACpB,IAAI,CAACjB,OAAO,WAAW,EAAE,OAAOzB;oBAChC,IAAI;wBACF,OAAO2C,iBAAiB3C,OAAOyB,OAAO,WAAW,EAAE;4BACjDe;wBACF;oBACF,EAAE,OAAOI,OAAY;wBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAEmB,MAAM,OAAO,CAAC,cAAc,EAAE9C,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAO4C;wBAAM;oBAEnB;gBACF;gBAEA,MAAMC,iBAAiB,CAAC,IAAI,CAAC,eAAe;gBAC5CV,YAAY,IAAI,CAAC,eAAe,GAAGU,iBAC/B,YACA;gBACJT,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3B,MAAMU,oBAAoBF,iBACrBpB,OAAO,iBAAiB,IAAI,MAC7B;gBACJ,IAAI;oBACF,MAAMuB,cAAc,MAAMC,QAAQ,UAAU,CAAC;wBAC3C,IAAI,CAAC,SAAS,CAAC,kBAAkB,GAC/BxB,OAAO,IAAI,EACXzB,OACAoC;wBAEFW,oBAAoB,IAChBG,kBAAkBd,aAAa,IAAI,CAAC;4BAClC,QAAQW;wBACV,KACAE,QAAQ,OAAO;qBACpB;oBACD,MAAME,WAAWH,YAAY,IAAI,CAC/B,CAACI,OAASA,AAAgB,eAAhBA,KAAK,MAAM;oBAEvB,IAAID,UAAU,WAAW,YAAY,MAAMA,SAAS,MAAM;gBAC5D,EAAE,OAAOE,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,wCAAwC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC5G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBAEAD,aAAa;gBACbU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAM8B,eAAe,MAAM9B,OAAO,IAAI,CAAC,IAAI,CACzC,IAAI,CAAC,SAAS,EACdiB,aACAG,iBACI;oBAAE,GAAGV,WAAW;oBAAE,QAAQC;gBAAY,IACtC;oBAAE,GAAGD,WAAW;oBAAE,QAAQC;oBAAa,iBAAiB;gBAAK;gBAEnEU,mBAAmBT,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW8B;gBAC/CnB,aAAa;gBAEbU,mBAAmBT,QAAQ;gBAC3BS,mBAAmBT,QAAQ;gBAC3B,IAAI;oBACF,IAAIQ,gBAAgB;wBAClB,MAAMW,mBACJ/B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;wBACrD,IAAI+B,mBAAmB,GACrB,MAAMN,kBAAkBd,aAAa,IAAI,CAAC;4BACxC,QAAQoB;wBACV;wBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpC/B,OAAO,IAAI,EACXiB,aACAN;oBAEJ,OACE,MAAM,IAAI,CAAC,eAAe,CAAE,IAAI,CAC9B;wBACE,IAAID,YAAY,IAAI,CAAC,MAAM;wBAC3B,MAAMV,OAAO,IAAI;wBACjB,OAAOiB;oBACT,GACAN,aACA,IAAI,CAAC,iBAAiB;gBAG5B,SAAU;oBACRU,mBAAmBT,QAAQ;gBAC7B;gBACAD,aAAa;gBACb,IAAI;oBACF,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,GACpCX,OAAO,IAAI,EACXiB,aACAN;gBAEJ,EAAE,OAAOiB,eAAoB;oBAC3B,MAAM,IAAIZ,MACR,CAAC,uCAAuC,EAAEhB,OAAO,IAAI,CAAC,EAAE,EAAE4B,eAAe,WAAWC,OAAOD,gBAAgB,EAC3G;wBAAE,OAAOA;oBAAc;gBAE3B,SAAU;oBACRP,mBAAmBT,QAAQ;gBAC7B;gBACA,OAAO;oBAAE,QAAQkB;gBAAa;YAChC;QACF;QAEAxC,QAAQ,KAAK,CAAC,IAAI,CAACmB;IACrB;IAEQ,iBACNhB,IAAyC,EACzCuC,mBAAiD,EACjD1C,OAAyB,EACzB2C,QAAgD,EACd;QAClC,MAAM,EAAE5C,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAE,GAAGY;QAEhD,IAAI4C,cAAc5D,qBAAqB0D;QAEvC,IAAI3C,AAAcV,WAAdU,WACF6C,cAAc;YACZ,GAAGA,WAAW;YACd7C;QACF;QAGF,IAAIX,cAAc,CAACwD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASzC,KAAK,OAAO;YACrB,UAAU,OAAOiB;gBACf,MAAMC,cAAcrB,QAAQ,WAAW,IAAIoB,YAAY,MAAM;gBAC7DC,aAAa;gBACb,MAAM,EAAEF,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEG,SAAS,EAAE,GAAGH;gBACpB,MAAM0B,8BAA8B9B,mCAClC4B,eAEEA,cACAvD;gBAEJmC,OACEoB,aAAa,UAAUE,6BACvB,CAAC,mEAAmE,EAAE/D,KAAK,SAAS,CAClF6D,cACC;gBAGL,IAAI,CAACrB,WAAW;oBACdA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAACF;oBAClDA,aAAa;gBACf;gBAEAG,OAAOD,WAAW;gBAElB,MAAM,EAAEE,wBAAwB,EAAE,GAAGF;gBAErC,IAAIE,AAA6BpC,WAA7BoC,0BACF,MAAM,IAAIC,MACR;gBAIJ,IAAIqB;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb/B,KAAK,GAAG,GAAG;wBACT+B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,oBAAoBA,KAAK,QAAQ,EAAE;wBACnC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA/B,KAAK,KAAK,GAAGgC,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD/B,KAAK,UAAU,GAAG+B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjB/B,KAAK,eAAe,GAAGgC,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjB/B,KAAK,iBAAiB,GAAG+B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,8BACvBO,qBAAqBP,+BACrBzD;gBAKJ,MAAMiE,kBAAkBV,YAAY,UAAU,GAC1CvD,SACA+D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDX,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACZ,YAAY,KAAK;yBAAC;oBAC7B;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMa,mBAAmBD,gBACrBE,oCACE;oBACE,QAAQ;wBACNF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;wBAC3CA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;qBAC5C;oBACD,aACE,AAA8B,YAA9B,OAAOZ,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU;oBACpC,MAAMY;gBACR,GACA/B,4BAEFpC;gBAEJ,MAAMsE,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAchB,YAAY,MAAM;gBACtC,MAAMiB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJR,mBAAmBI,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAhB,YAAY,SAAS;gBAI7B,MAAMqB,mBAAmBF,yBACrBL,oCACEK,wBACAtC,4BAEFpC;gBAEJ,MAAM6E,aAAa,CAAC,CAACD;gBAErB,IAAIE;gBACJ,MAAM7C,SAASF,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACuC,cAAc,CAACO,cAAc,CAACX,iBACjC,IAAI;oBACFxB,mBAAmBT,QAAQ;oBAC3B0B,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCJ,aACA;wBACE,SAASrB;wBACT6B;oBACF,GACAxD,cACAyB;oBAEF4B,UAAUD,aAAa,IAAI;oBAC3BmB,sBAAsBnB,aAAa,OAAO;gBAC5C,EAAE,OAAOnB,OAAO;oBACd,IAAIA,iBAAiBuC,cACnBnB,UAAUpB,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRE,mBAAmBT,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJ2E,mBACAG,oBACAQ,oBACAE;gBAEF,IAAIxF,SAAS;oBACX,MAAM0F,uBAAuB3F,2BAA2BC;oBACxD,IAAI0F,sBAAsB;wBACxB,IAAItB,YACF,MAAM,IAAIqB,aAAaC,sBAAsBtB;wBAE/C,MAAM,IAAIrB,MAAM2C;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B/F,iBAC/BsF,mBAAmB,cAAc;gBAGnC,IAAIU;gBAOJ,IACE5F,WACA,IAAI,CAAC,SAAS,IACd,CAACuF,cACA,EAACX,mBAAmB,CAACe,wBAAuB,KAC7C1B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,MAAM4B,gBAAkC;wBACtCC,KAAK,KAAK,CAAC9F,QAAQ,MAAM,CAAC,EAAE,GAAG8C;wBAC/BgD,KAAK,KAAK,CAAC9F,QAAQ,MAAM,CAAC,EAAE,GAAG8C;qBAChC;oBAED,MAAMiD,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO5B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAchD;oBAChB;oBAEF,IAAIrB,iBAAiBmG,UAAU;wBAC7BrG,MACE,uCACAuF,aACAc;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQd;4BACR,OAAOc;wBACT,GACAb;oBAEJ,OACExF,MACE,yDACAuF;gBAGN,EAAE,OAAO/B,OAAO;oBACdxD,MAAM,mCAAmCwD;gBAC3C;qBAEAxD,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIoE,YACF,MAAM,IAAIqB,aACR,CAAC,oBAAoB,EAAExB,YAAY,MAAM,EAAE,EAC3CG;oBAGJ,MAAM,IAAIrB,MAAM,CAAC,mBAAmB,EAAEkB,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAI+B;gBAEJ,IAAIpB,mBAAmBT,6BACrB6B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,oBACE7B,4BAA4B,kBAAkB;oBAClD;gBACF;qBACK,IAAIa,YACTgB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO/B,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIsB,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPb;wBACA,aAAaS;oBACf;gBACF;gBAGF,MAAMK,gBAAgBhC,YAAY,aAAa;gBAC/C,MAAMiC,mBAAmBD,gBACrB;oBACE,GAAGjG,OAAO;oBACV,aAAaiG;gBACf,IACAjG;gBAEJ,IAAIiG,iBAAiB7B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAAC+B,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJjC,WAAWkC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKtD,UAAU,aAAa;wBAC9B;oBACF;oBACAoD;gBACF;YACF;QACF;QAEA,OAAO9B;IACT;IA1mBA,YAAY,EACVkC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTxE,WAAW,EACXyE,eAAe,EACfC,eAAe,EACfC,iBAAiB,EACD,CAAE;QApBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QACA,uBAAiB,mBAAjB;QACA,uBAAiB,qBAAjB;QAWE,IAAI,CAAC,SAAS,GAAGL;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGxE;QACnB,IAAI,CAAC,eAAe,GAAGyE;QACvB,IAAI,CAAC,eAAe,GAAGC;QACvB,IAAI,CAAC,iBAAiB,GAAGC,qBAAqBL;IAChD;AA2lBF"}

package/dist/es/agent/tasks.mjs

@@ -47,10 +47,10 @@ class TaskExecutor {
         return this.interface;
     }
     createExecutionSession(title, options) {
-        return new ExecutionSession(title, ()=>options?.uiContext ? Promise.resolve(options.uiContext) : Promise.resolve(this.service.contextRetrieverFn()), {
+        return new ExecutionSession(title, (signal)=>options?.uiContext ? Promise.resolve(options.uiContext) : Promise.resolve(this.service.contextRetrieverFn(signal)), {
+            abortSignal: options?.abortSignal ?? this.getAbortSignal?.(),
             onTaskStart: this.onTaskStartCallback,
             tasks: options?.tasks,
-            abortSignal: options?.abortSignal,
             referenceImages: options?.referenceImages,
             onSnapshotChange: async (runner, error)=>{
                 await this.hooks?.onSnapshotChange?.(runner, error);
@@ -133,6 +133,7 @@ class TaskExecutor {
         };
     }
     async sleep(ms, abortSignal) {
+        abortSignal ??= this.getAbortSignal?.();
         const session = this.createExecutionSession('Sleep', {
             abortSignal
         });
@@ -157,7 +158,11 @@ class TaskExecutor {
         });
     }
     async runPlans(title, plans, planningModel, defaultModel, options) {
-        const session = this.createExecutionSession(title, options);
+        const abortSignal = options?.abortSignal ?? this.getAbortSignal?.();
+        const session = this.createExecutionSession(title, {
+            ...options,
+            abortSignal
+        });
         const runner = session.getRunner();
         const executionPlanningModel = {
             ...planningModel,
@@ -168,7 +173,7 @@ class TaskExecutor {
             executionId: runner.id
         };
         const { tasks } = await this.convertPlanToExecutable(plans, executionPlanningModel, executionDefaultModel, {
-            abortSignal: options?.abortSignal
+            abortSignal
         });
         const result = await session.appendAndRun(tasks);
         const { output } = result ?? {};
@@ -192,6 +197,7 @@ class TaskExecutor {
         }
     }
     async runAction(userPrompt, planningModel, defaultModel, aiActContext, cacheable, replanningCycleLimitOverride, effort = 'balance', deepLocate, abortSignal, reportOptions) {
+        abortSignal ??= this.getAbortSignal?.();
         const conversationHistory = new ConversationHistory();
         const promptDisplay = reportOptions?.prompt || userPromptToString(userPrompt);
         let activeActionReporter;
@@ -497,7 +503,10 @@ class TaskExecutor {
                     extraPageDescription = await descriptionOfTree(tree, 200, false, opt?.domIncluded === 'visible-only');
                 }
                 try {
-                    extractResult = await this.service.extract(demandInput, modelRuntime, opt, extraPageDescription, multimodalPrompt, uiContext, executionOptions);
+                    extractResult = await this.service.extract(demandInput, modelRuntime, opt, extraPageDescription, multimodalPrompt, uiContext, {
+                        ...executionOptions,
+                        abortSignal: executionOptions?.abortSignal ?? taskContext.signal
+                    });
                 } catch (error) {
                     if (error instanceof ServiceError) applyDump(error.dump);
                     throw error;
@@ -524,20 +533,24 @@ class TaskExecutor {
         return queryTask;
     }
     async createTypeQueryExecution(type, demand, modelRuntime, opt, multimodalPrompt, executionOptions) {
-        executionOptions?.abortSignal?.throwIfAborted();
+        const abortSignal = executionOptions?.abortSignal ?? this.getAbortSignal?.();
+        abortSignal?.throwIfAborted();
         const session = this.createExecutionSession(taskTitleStr(type, 'string' == typeof demand ? demand : JSON.stringify(demand)), {
             ...executionOptions?.uiContext ? {
                 uiContext: executionOptions.uiContext
             } : {},
             referenceImages: multimodalPrompt?.images,
-            abortSignal: executionOptions?.abortSignal
+            abortSignal
         });
         const runner = session.getRunner();
         const executionModelRuntime = {
             ...modelRuntime,
             executionId: runner.id
         };
-        const queryTask = await this.createTypeQueryTask(type, demand, executionModelRuntime, opt, multimodalPrompt, executionOptions);
+        const queryTask = await this.createTypeQueryTask(type, demand, executionModelRuntime, opt, multimodalPrompt, {
+            ...executionOptions,
+            abortSignal
+        });
         const result = await session.appendAndRun(queryTask);
         if (!result) throw new Error('result of taskExecutor.flush() is undefined in function createTypeQueryTask');
         const { output, thought } = result;
@@ -548,10 +561,12 @@ class TaskExecutor {
         };
     }
     async waitFor(assertion, opt, modelRuntime) {
+        const abortSignal = this.getAbortSignal?.();
         const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
         const description = `waitFor: ${textPrompt}`;
         const session = this.createExecutionSession(taskTitleStr('WaitFor', description), {
-            referenceImages: multimodalPrompt?.images
+            referenceImages: multimodalPrompt?.images,
+            abortSignal
         });
         const runner = session.getRunner();
         const executionModelRuntime = {
@@ -574,7 +589,10 @@ class TaskExecutor {
         while(lastCheckStart - overallStartTime <= timeoutMs){
             const currentCheckStart = Date.now();
             lastCheckStart = currentCheckStart;
-            const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt);
+            abortSignal?.throwIfAborted();
+            const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, executionModelRuntime, serviceExtractOpt, multimodalPrompt, {
+                abortSignal
+            });
             const result = await session.appendAndRun(queryTask);
             if (result?.output) return {
                 output: void 0,
@@ -594,7 +612,9 @@ class TaskExecutor {
                         },
                         thought
                     }
-                ], modelRuntime, modelRuntime);
+                ], modelRuntime, modelRuntime, {
+                    abortSignal
+                });
                 if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
             }
         }
@@ -608,6 +628,7 @@ class TaskExecutor {
         _define_property(this, "taskBuilder", void 0);
         _define_property(this, "onTaskStartCallback", void 0);
         _define_property(this, "hooks", void 0);
+        _define_property(this, "getAbortSignal", void 0);
         _define_property(this, "replanningCycleLimit", void 0);
         _define_property(this, "waitAfterAction", void 0);
         _define_property(this, "useDeviceTime", void 0);
@@ -619,6 +640,7 @@ class TaskExecutor {
         this.waitAfterAction = opts.waitAfterAction;
         this.useDeviceTime = opts.useDeviceTime;
         this.hooks = opts.hooks;
+        this.getAbortSignal = opts.getAbortSignal;
         this.providedActionSpace = opts.actionSpace;
         this.taskBuilder = new TaskBuilder({
             interfaceInstance,
@@ -626,7 +648,8 @@ class TaskExecutor {
             taskCache: opts.taskCache,
             actionSpace: this.getActionSpace(),
             waitAfterAction: opts.waitAfterAction,
-            actionReadiness: opts.actionReadiness
+            actionReadiness: opts.actionReadiness,
+            operationResource: opts.operationResource
         });
     }
 }