@midscene/core 1.7.11-beta-20260512144905.0 → 1.8.0

package/dist/es/agent/task-builder.mjs

@@ -17,29 +17,9 @@ function _define_property(obj, key, value) {
     return obj;
 }
 const debug = getDebug('agent:task-builder');
-const EXPERIMENTAL_DEEP_LOCATE_ZOOM_RATIO = 4;
-const EXPERIMENTAL_DEEP_LOCATE_ZOOM_DURATION_MS = 500;
-const UI_CONTEXT_CACHE_TTL_BYPASS_MS = 350;
 function hasNonEmptyCache(cache) {
     return null != cache && 'object' == typeof cache && Object.keys(cache).length > 0;
 }
-function transformZoomedScreenshotElementToScreenshot(element, scale) {
-    if (1 === scale) return element;
-    return {
-        ...element,
-        center: [
-            Math.round(element.center[0] / scale),
-            Math.round(element.center[1] / scale)
-        ],
-        rect: {
-            ...element.rect,
-            left: Math.round(element.rect.left / scale),
-            top: Math.round(element.rect.top / scale),
-            width: Math.round(element.rect.width / scale),
-            height: Math.round(element.rect.height / scale)
-        }
-    };
-}
 function locatePlanForLocate(param) {
     const locate = 'string' == typeof param ? {
         prompt: param
@@ -52,43 +32,6 @@ function locatePlanForLocate(param) {
     return locatePlan;
 }
 class TaskBuilder {
-    getWebPinchApi() {
-        const pinch = this.interface.pinch;
-        if ('function' != typeof pinch) return;
-        return pinch.bind(this.interface);
-    }
-    async activateWebDeepLocateZoom(element, shrunkShotToLogicalRatio) {
-        const pinch = this.getWebPinchApi();
-        if (!pinch || this.activeWebDeepLocateZoom) return;
-        const { width, height } = await this.interface.size();
-        const baseDistance = Math.round(Math.min(width, height) / 4);
-        const logicalCenterX = Math.round(element.center[0] / shrunkShotToLogicalRatio);
-        const logicalCenterY = Math.round(element.center[1] / shrunkShotToLogicalRatio);
-        const zoomState = {
-            centerX: logicalCenterX,
-            centerY: logicalCenterY,
-            startDistance: baseDistance,
-            endDistance: Math.round(baseDistance * EXPERIMENTAL_DEEP_LOCATE_ZOOM_RATIO),
-            duration: EXPERIMENTAL_DEEP_LOCATE_ZOOM_DURATION_MS
-        };
-        debug('activate experimental web deepLocate pinch zoom', zoomState);
-        await pinch(zoomState.centerX, zoomState.centerY, zoomState.startDistance, zoomState.endDistance, zoomState.duration);
-        this.activeWebDeepLocateZoom = zoomState;
-    }
-    async restoreWebDeepLocateZoom() {
-        const pinch = this.getWebPinchApi();
-        const zoomState = this.activeWebDeepLocateZoom;
-        this.activeWebDeepLocateZoom = void 0;
-        if (!pinch || !zoomState) return;
-        debug('restore experimental web deepLocate pinch zoom', zoomState);
-        await pinch(zoomState.centerX, zoomState.centerY, zoomState.endDistance, zoomState.startDistance, zoomState.duration);
-    }
-    async getVisualViewportScale() {
-        const evaluateJavaScript = this.interface.evaluateJavaScript;
-        if (!this.activeWebDeepLocateZoom || 'function' != typeof evaluateJavaScript) return 1;
-        const scale = await evaluateJavaScript.call(this.interface, 'window.visualViewport?.scale ?? 1');
-        return 'number' == typeof scale && Number.isFinite(scale) ? scale : 1;
-    }
     async build(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options) {
         const tasks = [];
         const cacheable = options?.cacheable;
@@ -198,44 +141,28 @@ class TaskBuilder {
                         cause: error
                     });
                 }
-                let actionResult;
-                let pendingError;
-                let restoreError;
+                setTimingFieldOnce(timing, 'callActionStart');
+                debug('calling action', action.name);
+                const actionFn = action.call.bind(this.interface);
+                const actionResult = await actionFn(param, taskContext);
+                setTimingFieldOnce(timing, 'callActionEnd');
+                debug('called action', action.name, 'result:', actionResult);
+                setTimingFieldOnce(timing, 'afterInvokeActionHookStart');
+                const delayAfterRunner = action.delayAfterRunner ?? this.waitAfterAction ?? 300;
+                if (delayAfterRunner > 0) await sleep(delayAfterRunner);
                 try {
-                    setTimingFieldOnce(timing, 'callActionStart');
-                    debug('calling action', action.name);
-                    const actionFn = action.call.bind(this.interface);
-                    actionResult = await actionFn(param, taskContext);
-                    setTimingFieldOnce(timing, 'callActionEnd');
-                    debug('called action', action.name, 'result:', actionResult);
-                    setTimingFieldOnce(timing, 'afterInvokeActionHookStart');
-                    const delayAfterRunner = action.delayAfterRunner ?? this.waitAfterAction ?? 300;
-                    if (delayAfterRunner > 0) await sleep(delayAfterRunner);
-                    try {
-                        if (this.interface.afterInvokeAction) {
-                            debug(`will call "afterInvokeAction" for interface with action name ${action.name}`);
-                            await this.interface.afterInvokeAction(action.name, param);
-                            debug(`called "afterInvokeAction" for interface with action name ${action.name}`);
-                        }
-                    } catch (originalError) {
-                        const originalMessage = originalError?.message || String(originalError);
-                        throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalMessage}`, {
-                            cause: originalError
-                        });
-                    }
-                    setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');
-                } catch (error) {
-                    pendingError = error;
-                } finally{
-                    if (this.activeWebDeepLocateZoom) try {
-                        await this.restoreWebDeepLocateZoom();
-                    } catch (error) {
-                        restoreError = error;
-                        if (pendingError) console.warn('[Midscene] failed to restore experimental deepLocate zoom:', error);
+                    if (this.interface.afterInvokeAction) {
+                        debug(`will call "afterInvokeAction" for interface with action name ${action.name}`);
+                        await this.interface.afterInvokeAction(action.name, param);
+                        debug(`called "afterInvokeAction" for interface with action name ${action.name}`);
                     }
+                } catch (originalError) {
+                    const originalMessage = originalError?.message || String(originalError);
+                    throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalMessage}`, {
+                        cause: originalError
+                    });
                 }
-                if (pendingError) throw pendingError;
-                if (restoreError) throw restoreError;
+                setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');
                 return {
                     output: actionResult
                 };
@@ -309,31 +236,12 @@ class TaskBuilder {
                 const timing = taskContext.task.timing;
                 if (!isXpathHit && !isCacheHit && !isPlanHit) try {
                     setTimingFieldOnce(timing, 'callAiStart');
-                    const experimentalPinchDeepLocate = !!param.deepLocate && !!this.getWebPinchApi();
-                    const locateParamForAi = experimentalPinchDeepLocate ? {
-                        ...param,
-                        deepLocate: false
-                    } : param;
-                    locateResult = await this.service.locate(locateParamForAi, {
+                    locateResult = await this.service.locate(param, {
                         context: uiContext,
                         planLocatedElement
                     }, modelConfigForDefaultIntent, abortSignal);
                     applyDump(locateResult.dump);
                     elementFromAiLocate = locateResult.element;
-                    if (experimentalPinchDeepLocate && elementFromAiLocate && !this.activeWebDeepLocateZoom) {
-                        await this.activateWebDeepLocateZoom(elementFromAiLocate, shrunkShotToLogicalRatio);
-                        await sleep(UI_CONTEXT_CACHE_TTL_BYPASS_MS);
-                        uiContext = await this.service.contextRetrieverFn();
-                        locateResult = await this.service.locate(locateParamForAi, {
-                            context: uiContext
-                        }, modelConfigForDefaultIntent, abortSignal);
-                        applyDump(locateResult.dump);
-                        elementFromAiLocate = locateResult.element;
-                        if (elementFromAiLocate) {
-                            const zoomScale = await this.getVisualViewportScale();
-                            elementFromAiLocate = transformZoomedScreenshotElementToScreenshot(elementFromAiLocate, zoomScale);
-                        }
-                    }
                 } catch (error) {
                     if (error instanceof ServiceError) applyDump(error.dump);
                     throw error;
@@ -413,7 +321,6 @@ class TaskBuilder {
         _define_property(this, "taskCache", void 0);
         _define_property(this, "actionSpace", void 0);
         _define_property(this, "waitAfterAction", void 0);
-        _define_property(this, "activeWebDeepLocateZoom", void 0);
         this.interface = interfaceInstance;
         this.service = service;
         this.taskCache = taskCache;

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\nconst EXPERIMENTAL_DEEP_LOCATE_ZOOM_RATIO = 4;\nconst EXPERIMENTAL_DEEP_LOCATE_ZOOM_DURATION_MS = 500;\nconst UI_CONTEXT_CACHE_TTL_BYPASS_MS = 350;\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction transformZoomedScreenshotElementToScreenshot(\n  element: LocateResultElement,\n  scale: number,\n): LocateResultElement {\n  if (scale === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] / scale),\n      Math.round(element.center[1] / scale),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left / scale),\n      top: Math.round(element.rect.top / scale),\n      width: Math.round(element.rect.width / scale),\n      height: Math.round(element.rect.height / scale),\n    },\n  };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface WebDeepLocateZoomState {\n  centerX: number;\n  centerY: number;\n  startDistance: number;\n  endDistance: number;\n  duration: number;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  private activeWebDeepLocateZoom?: WebDeepLocateZoomState;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  private getWebPinchApi():\n    | ((\n        centerX: number,\n        centerY: number,\n        startDistance: number,\n        endDistance: number,\n        duration?: number,\n      ) => Promise<void>)\n    | undefined {\n    const pinch = (this.interface as any).pinch;\n    if (typeof pinch !== 'function') {\n      return undefined;\n    }\n    return pinch.bind(this.interface);\n  }\n\n  private async activateWebDeepLocateZoom(\n    element: LocateResultElement,\n    shrunkShotToLogicalRatio: number,\n  ): Promise<void> {\n    const pinch = this.getWebPinchApi();\n    if (!pinch || this.activeWebDeepLocateZoom) {\n      return;\n    }\n\n    const { width, height } = await this.interface.size();\n    const baseDistance = Math.round(Math.min(width, height) / 4);\n    const logicalCenterX = Math.round(\n      element.center[0] / shrunkShotToLogicalRatio,\n    );\n    const logicalCenterY = Math.round(\n      element.center[1] / shrunkShotToLogicalRatio,\n    );\n    const zoomState: WebDeepLocateZoomState = {\n      centerX: logicalCenterX,\n      centerY: logicalCenterY,\n      startDistance: baseDistance,\n      endDistance: Math.round(\n        baseDistance * EXPERIMENTAL_DEEP_LOCATE_ZOOM_RATIO,\n      ),\n      duration: EXPERIMENTAL_DEEP_LOCATE_ZOOM_DURATION_MS,\n    };\n\n    debug('activate experimental web deepLocate pinch zoom', zoomState);\n    await pinch(\n      zoomState.centerX,\n      zoomState.centerY,\n      zoomState.startDistance,\n      zoomState.endDistance,\n      zoomState.duration,\n    );\n    this.activeWebDeepLocateZoom = zoomState;\n  }\n\n  private async restoreWebDeepLocateZoom(): Promise<void> {\n    const pinch = this.getWebPinchApi();\n    const zoomState = this.activeWebDeepLocateZoom;\n    this.activeWebDeepLocateZoom = undefined;\n    if (!pinch || !zoomState) {\n      return;\n    }\n\n    debug('restore experimental web deepLocate pinch zoom', zoomState);\n    await pinch(\n      zoomState.centerX,\n      zoomState.centerY,\n      zoomState.endDistance,\n      zoomState.startDistance,\n      zoomState.duration,\n    );\n  }\n\n  private async getVisualViewportScale(): Promise<number> {\n    const evaluateJavaScript = this.interface.evaluateJavaScript;\n    if (\n      !this.activeWebDeepLocateZoom ||\n      typeof evaluateJavaScript !== 'function'\n    ) {\n      return 1;\n    }\n\n    const scale = (await evaluateJavaScript.call(\n      this.interface,\n      'window.visualViewport?.scale ?? 1',\n    )) as unknown;\n\n    return typeof scale === 'number' && Number.isFinite(scale) ? scale : 1;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        let actionResult;\n        let pendingError: unknown;\n        let restoreError: unknown;\n\n        try {\n          setTimingFieldOnce(timing, 'callActionStart');\n\n          debug('calling action', action.name);\n          const actionFn = action.call.bind(this.interface);\n          actionResult = await actionFn(param, taskContext);\n          setTimingFieldOnce(timing, 'callActionEnd');\n          debug('called action', action.name, 'result:', actionResult);\n\n          setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n          const delayAfterRunner =\n            action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n          if (delayAfterRunner > 0) {\n            await sleep(delayAfterRunner);\n          }\n\n          try {\n            if (this.interface.afterInvokeAction) {\n              debug(\n                `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n              );\n              await this.interface.afterInvokeAction(action.name, param);\n              debug(\n                `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n              );\n            }\n          } catch (originalError: any) {\n            const originalMessage =\n              originalError?.message || String(originalError);\n            throw new Error(\n              `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n              { cause: originalError },\n            );\n          }\n\n          setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n        } catch (error) {\n          pendingError = error;\n        } finally {\n          if (this.activeWebDeepLocateZoom) {\n            try {\n              await this.restoreWebDeepLocateZoom();\n            } catch (error) {\n              restoreError = error;\n              if (pendingError) {\n                console.warn(\n                  '[Midscene] failed to restore experimental deepLocate zoom:',\n                  error,\n                );\n              }\n            }\n          }\n        }\n\n        if (pendingError) {\n          throw pendingError;\n        }\n\n        if (restoreError) {\n          throw restoreError;\n        }\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n\n        // from bbox (plan hit)\n        // when deepLocate is enabled, bbox should be used as search area hint,\n        // not as a final direct hit\n        const elementFromBbox = param.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            const experimentalPinchDeepLocate =\n              !!param.deepLocate && !!this.getWebPinchApi();\n            const locateParamForAi = experimentalPinchDeepLocate\n              ? {\n                  ...param,\n                  deepLocate: false,\n                }\n              : param;\n\n            locateResult = await this.service.locate(\n              locateParamForAi,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              modelConfigForDefaultIntent,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n\n            if (\n              experimentalPinchDeepLocate &&\n              elementFromAiLocate &&\n              !this.activeWebDeepLocateZoom\n            ) {\n              await this.activateWebDeepLocateZoom(\n                elementFromAiLocate,\n                shrunkShotToLogicalRatio,\n              );\n              await sleep(UI_CONTEXT_CACHE_TTL_BYPASS_MS);\n              uiContext = await this.service.contextRetrieverFn();\n              locateResult = await this.service.locate(\n                locateParamForAi,\n                {\n                  context: uiContext,\n                },\n                modelConfigForDefaultIntent,\n                abortSignal,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n              if (elementFromAiLocate) {\n                const zoomScale = await this.getVisualViewportScale();\n                elementFromAiLocate =\n                  transformZoomedScreenshotElementToScreenshot(\n                    elementFromAiLocate,\n                    zoomScale,\n                  );\n              }\n            }\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","EXPERIMENTAL_DEEP_LOCATE_ZOOM_RATIO","EXPERIMENTAL_DEEP_LOCATE_ZOOM_DURATION_MS","UI_CONTEXT_CACHE_TTL_BYPASS_MS","hasNonEmptyCache","cache","Object","transformZoomedScreenshotElementToScreenshot","element","scale","Math","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","pinch","shrunkShotToLogicalRatio","width","height","baseDistance","logicalCenterX","logicalCenterY","zoomState","undefined","evaluateJavaScript","Number","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","parseActionParam","error","actionResult","pendingError","restoreError","actionFn","delayAfterRunner","console","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","planLocatedElement","matchElementFromPlan","elementFromBbox","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","experimentalPinchDeepLocate","locateParamForAi","zoomScale","ServiceError","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;AAkCA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,sCAAsC;AAC5C,MAAMC,4CAA4C;AAClD,MAAMC,iCAAiC;AAKvC,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,6CACPC,OAA4B,EAC5BC,KAAa;IAEb,IAAIA,AAAU,MAAVA,OACF,OAAOD;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNE,KAAK,KAAK,CAACF,QAAQ,MAAM,CAAC,EAAE,GAAGC;YAC/BC,KAAK,KAAK,CAACF,QAAQ,MAAM,CAAC,EAAE,GAAGC;SAChC;QACD,MAAM;YACJ,GAAGD,QAAQ,IAAI;YACf,MAAME,KAAK,KAAK,CAACF,QAAQ,IAAI,CAAC,IAAI,GAAGC;YACrC,KAAKC,KAAK,KAAK,CAACF,QAAQ,IAAI,CAAC,GAAG,GAAGC;YACnC,OAAOC,KAAK,KAAK,CAACF,QAAQ,IAAI,CAAC,KAAK,GAAGC;YACvC,QAAQC,KAAK,KAAK,CAACF,QAAQ,IAAI,CAAC,MAAM,GAAGC;QAC3C;IACF;AACF;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAiCO,MAAMC;IA2BH,iBAQM;QACZ,MAAMC,QAAS,IAAI,CAAC,SAAS,CAAS,KAAK;QAC3C,IAAI,AAAiB,cAAjB,OAAOA,OACT;QAEF,OAAOA,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS;IAClC;IAEA,MAAc,0BACZR,OAA4B,EAC5BS,wBAAgC,EACjB;QACf,MAAMD,QAAQ,IAAI,CAAC,cAAc;QACjC,IAAI,CAACA,SAAS,IAAI,CAAC,uBAAuB,EACxC;QAGF,MAAM,EAAEE,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI;QACnD,MAAMC,eAAeV,KAAK,KAAK,CAACA,KAAK,GAAG,CAACQ,OAAOC,UAAU;QAC1D,MAAME,iBAAiBX,KAAK,KAAK,CAC/BF,QAAQ,MAAM,CAAC,EAAE,GAAGS;QAEtB,MAAMK,iBAAiBZ,KAAK,KAAK,CAC/BF,QAAQ,MAAM,CAAC,EAAE,GAAGS;QAEtB,MAAMM,YAAoC;YACxC,SAASF;YACT,SAASC;YACT,eAAeF;YACf,aAAaV,KAAK,KAAK,CACrBU,eAAenB;YAEjB,UAAUC;QACZ;QAEAH,MAAM,mDAAmDwB;QACzD,MAAMP,MACJO,UAAU,OAAO,EACjBA,UAAU,OAAO,EACjBA,UAAU,aAAa,EACvBA,UAAU,WAAW,EACrBA,UAAU,QAAQ;QAEpB,IAAI,CAAC,uBAAuB,GAAGA;IACjC;IAEA,MAAc,2BAA0C;QACtD,MAAMP,QAAQ,IAAI,CAAC,cAAc;QACjC,MAAMO,YAAY,IAAI,CAAC,uBAAuB;QAC9C,IAAI,CAAC,uBAAuB,GAAGC;QAC/B,IAAI,CAACR,SAAS,CAACO,WACb;QAGFxB,MAAM,kDAAkDwB;QACxD,MAAMP,MACJO,UAAU,OAAO,EACjBA,UAAU,OAAO,EACjBA,UAAU,WAAW,EACrBA,UAAU,aAAa,EACvBA,UAAU,QAAQ;IAEtB;IAEA,MAAc,yBAA0C;QACtD,MAAME,qBAAqB,IAAI,CAAC,SAAS,CAAC,kBAAkB;QAC5D,IACE,CAAC,IAAI,CAAC,uBAAuB,IAC7B,AAA8B,cAA9B,OAAOA,oBAEP,OAAO;QAGT,MAAMhB,QAAS,MAAMgB,mBAAmB,IAAI,CAC1C,IAAI,CAAC,SAAS,EACd;QAGF,OAAO,AAAiB,YAAjB,OAAOhB,SAAsBiB,OAAO,QAAQ,CAACjB,SAASA,QAAQ;IACvE;IAEA,MAAa,MACXkB,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAM7B,QAAQwB,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAIrC,KAAK,CAACqC,MAAM,EAAE;gBAGhB,MAAMnC,aAAaH,oBAAoBC,KAAK,CAACqC,MAAM;gBACnDlD,MACE,uCACA,CAAC,YAAY,EAAE0C,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAACtC,KAAK,CAACqC,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACpC,aAAa,EAC1C,CAAC,QAAQ,EAAEqC,wBAAwBvC,KAAK,CAACqC,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtCtC,YACAF,KAAK,CAACqC,MAAM,EACZhB,SACA,CAACoB;oBACCzC,KAAK,CAACqC,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3E1C,MAAM,CAAC,OAAO,EAAEkD,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOxB,OAAO4C;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCzD,MACE,oBACA0C,UACA7B,OACA,CAAC,4BAA4B,EAAE4C,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE1C,KAAK,CAACqC,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBjB,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMkB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC9D,MACE,CAAC,8DAA8D,EAAE4C,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAE/B;gCACrDb,MACE,CAAC,2DAA2D,EAAE4C,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAiB,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIlB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEqB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAExC,wBAAwB,EAAE,GAAGyC;gBACrC,IAAIzC,AAA6BO,WAA7BP,0BACF,MAAM,IAAI4B,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACF/B,QAAQsD,iBAAiBtD,OAAO+B,OAAO,WAAW,EAAE;wBAClD1B;oBACF;gBACF,EAAE,OAAOkD,OAAY;oBACnB,MAAM,IAAItB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEwB,MAAM,OAAO,CAAC,cAAc,EAAEjB,KAAK,SAAS,CAACtC,QAAQ,EACtG;wBAAE,OAAOuD;oBAAM;gBAEnB;gBAGF,IAAIC;gBACJ,IAAIC;gBACJ,IAAIC;gBAEJ,IAAI;oBACFX,mBAAmBF,QAAQ;oBAE3B1D,MAAM,kBAAkB4C,OAAO,IAAI;oBACnC,MAAM4B,WAAW5B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;oBAChDyB,eAAe,MAAMG,SAAS3D,OAAO4C;oBACrCG,mBAAmBF,QAAQ;oBAC3B1D,MAAM,iBAAiB4C,OAAO,IAAI,EAAE,WAAWyB;oBAE/CT,mBAAmBF,QAAQ;oBAE3B,MAAMe,mBACJ7B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;oBACrD,IAAI6B,mBAAmB,GACrB,MAAMV,MAAMU;oBAGd,IAAI;wBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;4BACpCzE,MACE,CAAC,6DAA6D,EAAE4C,OAAO,IAAI,EAAE;4BAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAE/B;4BACpDb,MACE,CAAC,0DAA0D,EAAE4C,OAAO,IAAI,EAAE;wBAE9E;oBACF,EAAE,OAAOoB,eAAoB;wBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;wBACnC,MAAM,IAAIlB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEqB,iBAAiB,EAC3E;4BAAE,OAAOD;wBAAc;oBAE3B;oBAEAJ,mBAAmBF,QAAQ;gBAC7B,EAAE,OAAOU,OAAO;oBACdE,eAAeF;gBACjB,SAAU;oBACR,IAAI,IAAI,CAAC,uBAAuB,EAC9B,IAAI;wBACF,MAAM,IAAI,CAAC,wBAAwB;oBACrC,EAAE,OAAOA,OAAO;wBACdG,eAAeH;wBACf,IAAIE,cACFI,QAAQ,IAAI,CACV,8DACAN;oBAGN;gBAEJ;gBAEA,IAAIE,cACF,MAAMA;gBAGR,IAAIC,cACF,MAAMA;gBAGR,OAAO;oBACL,QAAQF;gBACV;YACF;QACF;QAEAnC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCsC,mBAAiD,EACjDzC,OAAyB,EACzB0C,QAAgD,EACd;QAClC,MAAM,EAAE3C,SAAS,EAAEH,2BAA2B,EAAE+C,UAAU,EAAEC,WAAW,EAAE,GACvE5C;QAEF,IAAI6C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI9C,AAAcR,WAAdQ,WACF8C,cAAc;YACZ,GAAGA,WAAW;YACd9C;QACF;QAGF,IAAI4C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAS1C,KAAK,OAAO;YACrB,UAAU,OAAOxB,OAAO4C;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE1C,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAEsC,KAAK,SAAS,CACpEtC,QACC;gBAGL,IAAI,CAAC8C,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEzC,wBAAwB,EAAE,GAAGyC;gBAErC,IAAIzC,AAA6BO,WAA7BP,0BACF,MAAM,IAAI4B,MACR;gBAIJ,IAAImC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb5B,KAAK,GAAG,GAAG;wBACT4B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA5B,KAAK,KAAK,GAAG4B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB5B,KAAK,eAAe,GAAG4B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB5B,KAAK,iBAAiB,GAAG4B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAMC,qBAAqBjC,wBAAwBvC,SAC/CyE,qBAAqBzE,SACrBY;gBAKJ,MAAM8D,kBAAkB1E,MAAM,UAAU,GACpCY,SACA4D;gBACJ,MAAMG,YAAY,CAAC,CAACD;gBAGpB,IAAIE;gBACJ,IACE,CAACD,aACD3E,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF4E,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC5E,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM6E,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACAvE,2BAEF,AAAwB,YAAxB,OAAOL,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BY;gBAEJ,MAAMoE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcjF,MAAM,MAAM;gBAChC,MAAMkF,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAjF,MAAM,SAAS;gBAIvB,MAAMsF,mBAAmBF,yBACrBG,oCACEH,wBACA/E,4BAEFO;gBAEJ,MAAM4E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM5C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACQ,cAAc,CAACb,WACjC,IAAI;oBACF5B,mBAAmBF,QAAQ;oBAC3B,MAAM6C,8BACJ,CAAC,CAAC1F,MAAM,UAAU,IAAI,CAAC,CAAC,IAAI,CAAC,cAAc;oBAC7C,MAAM2F,mBAAmBD,8BACrB;wBACE,GAAG1F,KAAK;wBACR,YAAY;oBACd,IACAA;oBAEJqE,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCsB,kBACA;wBACE,SAAS7C;wBACT0B;oBACF,GACAvD,6BACAgD;oBAEFK,UAAUD,aAAa,IAAI;oBAC3BoB,sBAAsBpB,aAAa,OAAO;oBAE1C,IACEqB,+BACAD,uBACA,CAAC,IAAI,CAAC,uBAAuB,EAC7B;wBACA,MAAM,IAAI,CAAC,yBAAyB,CAClCA,qBACApF;wBAEF,MAAM6C,MAAM3D;wBACZuD,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;wBACjDuB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCsB,kBACA;4BACE,SAAS7C;wBACX,GACA7B,6BACAgD;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BoB,sBAAsBpB,aAAa,OAAO;wBAC1C,IAAIoB,qBAAqB;4BACvB,MAAMG,YAAY,MAAM,IAAI,CAAC,sBAAsB;4BACnDH,sBACE9F,6CACE8F,qBACAG;wBAEN;oBACF;gBACF,EAAE,OAAOrC,OAAO;oBACd,IAAIA,iBAAiBsC,cACnBvB,UAAUf,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRR,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAMjD,UACJ8E,mBACAG,oBACAS,oBACAG;gBAGF,MAAMK,2BAA2BtG,iBAC/B0F,mBAAmB,cAAc;gBAGnC,IAAIa;gBAOJ,IACEnG,WACA,IAAI,CAAC,SAAS,IACd,CAAC4F,cACA,EAACb,aAAa,CAACmB,wBAAuB,KACvC9F,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIgG,gBAAkCpG,QAAQ,MAAM;oBACpD,IAAIS,AAA6B,MAA7BA,0BAAgC;wBAClC2F,gBAAgB;4BACdlG,KAAK,KAAK,CAACF,QAAQ,MAAM,CAAC,EAAE,GAAGS;4BAC/BP,KAAK,KAAK,CAACF,QAAQ,MAAM,CAAC,EAAE,GAAGS;yBAChC;wBACDlB,MACE,8DACAS,QAAQ,MAAM,EACdoG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOhG,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaiB;oBACf;oBAEF,IAAIzB,iBAAiByG,UAAU;wBAC7B9G,MACE,uCACA8F,aACAgB;wBAEFF,oBAAoBE;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQhB;4BACR,OAAOgB;wBACT,GACAf;oBAEJ,OACE/F,MACE,yDACA8F;gBAGN,EAAE,OAAO1B,OAAO;oBACdpE,MAAM,mCAAmCoE;gBAC3C;qBAEApE,MAAM;gBAIV,IAAI,CAACS,SAAS;oBACZ,IAAIwE,YACF,MAAM,IAAIyB,aACR,CAAC,oBAAoB,EAAE7F,MAAM,MAAM,EAAE,EACrCoE;oBAGJ,MAAM,IAAInC,MAAM,CAAC,mBAAmB,EAAEjC,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIkG;gBAEJ,IAAIvB,WACFuB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMlG,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIgF,YACTkB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOlG,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIwF,YACTU,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPf;wBACA,aAAaY;oBACf;gBACF;gBAGFhC,WAAWnE;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAKkD,UAAU,aAAa;wBAC9B;oBACF;oBACAoD;gBACF;YACF;QACF;QAEA,OAAO/B;IACT;IAvtBA,YAAY,EACVgC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTvE,WAAW,EACXwE,eAAe,EACC,CAAE;QAlBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QAEA,uBAAQ,2BAAR;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGvE;QACnB,IAAI,CAAC,eAAe,GAAGwE;IACzB;AA4sBF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n\n        // from bbox (plan hit)\n        // when deepLocate is enabled, bbox should be used as search area hint,\n        // not as a final direct hit\n        const elementFromBbox = param.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              modelConfigForDefaultIntent,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","planLocatedElement","matchElementFromPlan","elementFromBbox","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;AAkCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBjB,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMkB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC5C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAiB,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIlB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEqB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAES,wBAAwB,EAAE,GAAGR;gBACrC,IAAIQ,AAA6BC,WAA7BD,0BACF,MAAM,IAAIrB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ6C,iBAAiB7C,OAAOoB,OAAO,WAAW,EAAE;wBAClDuB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIxB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAE0B,MAAM,OAAO,CAAC,cAAc,EAAEnB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO8C;oBAAM;gBAEnB;gBAGFV,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM2B,WAAW3B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM4B,eAAe,MAAMD,SAAS/C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW4B;gBAE/CZ,mBAAmBF,QAAQ;gBAE3B,MAAMe,mBACJ7B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI6B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCvD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOoB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIlB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEqB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQc;gBACV;YACF;QACF;QAEAtC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCqC,mBAAiD,EACjDxC,OAAyB,EACzByC,QAAgD,EACd;QAClC,MAAM,EAAE1C,SAAS,EAAEH,2BAA2B,EAAE8C,UAAU,EAAEC,WAAW,EAAE,GACvE3C;QAEF,IAAI4C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI7C,AAAcmC,WAAdnC,WACF6C,cAAc;YACZ,GAAGA,WAAW;YACd7C;QACF;QAGF,IAAI2C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASzC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEQ,wBAAwB,EAAE,GAAGR;gBAErC,IAAIQ,AAA6BC,WAA7BD,0BACF,MAAM,IAAIrB,MACR;gBAIJ,IAAIkC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb3B,KAAK,GAAG,GAAG;wBACT2B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA3B,KAAK,KAAK,GAAG2B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB3B,KAAK,eAAe,GAAG2B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB3B,KAAK,iBAAiB,GAAG2B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAMC,qBAAqBhC,wBAAwB5B,SAC/C6D,qBAAqB7D,SACrB4C;gBAKJ,MAAMkB,kBAAkB9D,MAAM,UAAU,GACpC4C,SACAgB;gBACJ,MAAMG,YAAY,CAAC,CAACD;gBAGpB,IAAIE;gBACJ,IACE,CAACD,aACD/D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFgE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAChE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMiE,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACArB,2BAEF,AAAwB,YAAxB,OAAO3C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B4C;gBAEJ,MAAMwB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcrE,MAAM,MAAM;gBAChC,MAAMsE,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACArE,MAAM,SAAS;gBAIvB,MAAM0E,mBAAmBF,yBACrBG,oCACEH,wBACA7B,4BAEFC;gBAEJ,MAAMgC,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM3C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACmC,cAAc,CAACQ,cAAc,CAACb,WACjC,IAAI;oBACF3B,mBAAmBF,QAAQ;oBAC3BuB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCzD,OACA;wBACE,SAASmC;wBACTyB;oBACF,GACAtD,6BACA+C;oBAEFK,UAAUD,aAAa,IAAI;oBAC3BoB,sBAAsBpB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBgC,cACnBpB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRV,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM6C,UACJjB,mBACAG,oBACAS,oBACAG;gBAGF,MAAMG,2BAA2BpF,iBAC/B0E,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACd,CAACH,cACA,EAACb,aAAa,CAACiB,wBAAuB,KACvChF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIkF,gBAAkCH,QAAQ,MAAM;oBACpD,IAAIpC,AAA6B,MAA7BA,0BAAgC;wBAClCuC,gBAAgB;4BACdC,KAAK,KAAK,CAACJ,QAAQ,MAAM,CAAC,EAAE,GAAGpC;4BAC/BwC,KAAK,KAAK,CAACJ,QAAQ,MAAM,CAAC,EAAE,GAAGpC;yBAChC;wBACDjD,MACE,8DACAqF,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOlF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiBwF,UAAU;wBAC7B1F,MACE,uCACA2E,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACE5E,MACE,yDACA2E;gBAGN,EAAE,OAAOvB,OAAO;oBACdpD,MAAM,mCAAmCoD;gBAC3C;qBAEApD,MAAM;gBAIV,IAAI,CAACqF,SAAS;oBACZ,IAAIvB,YACF,MAAM,IAAIsB,aACR,CAAC,oBAAoB,EAAE9E,MAAM,MAAM,EAAE,EACrCwD;oBAGJ,MAAM,IAAIlC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIqF;gBAEJ,IAAItB,WACFsB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMrF,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIoE,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOrF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI4E,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF9B,WAAW4B;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK5C,UAAU,aAAa;wBAC9B;oBACF;oBACAkD;gBACF;YACF;QACF;QAEA,OAAO9B;IACT;IAxjBA,YAAY,EACV+B,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTrE,WAAW,EACXsE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGrE;QACnB,IAAI,CAAC,eAAe,GAAGsE;IACzB;AA6iBF"}

package/dist/es/agent/utils.mjs

@@ -123,7 +123,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.7.11-beta-20260512144905.0";
+const getMidsceneVersion = ()=>"1.8.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,