@donggui/core 1.6.3 → 1.6.4

package/dist/es/agent/task-builder.mjs

@@ -363,7 +363,9 @@ class TaskBuilder {
                     if (hasNonEmptyCache(feature)) {
                         feature.cachedCenter = pointForCache;
                         feature.confidenceState = createInitialConfidenceState();
-                        feature.progressiveRecord = createInitialProgressiveRecord(pointForCache);
+                        const existingProgressiveRecord = cacheEntry?.progressiveRecord;
+                        if (existingProgressiveRecord) feature.progressiveRecord = updateProgressiveConvergence(existingProgressiveRecord, pointForCache, 1.0);
+                        else feature.progressiveRecord = createInitialProgressiveRecord(pointForCache);
                         debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
                         const enableSemanticAnchor = 'false' !== process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR];
                         if (enableSemanticAnchor) try {

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import {\n  calculateConfidence,\n  createInitialConfidenceState,\n  createInitialProgressiveRecord,\n  determineVerificationLevel,\n  getVerificationActions,\n  updateConfidenceOnVerify,\n  updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  CacheConfidenceState,\n  CacheValidationOptions,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  ProgressiveLocateRecord,\n  Rect,\n  SemanticAnchor,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n  MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n  MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n  MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord =\n          await this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        let elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        let isCacheHit = !!elementFromCache;\n        const timing = taskContext.task.timing;\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n\n        if (isCacheHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const cachedCenter = cacheFeature?.cachedCenter as\n            | [number, number]\n            | undefined;\n\n          const coordOffsetThreshold =\n            Number.parseInt(\n              process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] || '16',\n              10,\n            ) || 16;\n\n          const confidenceState = (cacheFeature?.confidenceState ||\n            createInitialConfidenceState()) as CacheConfidenceState;\n          const confidence = calculateConfidence(confidenceState);\n          const level = determineVerificationLevel(confidence);\n          const actions = getVerificationActions(level);\n\n          debug('cache confidence assessment', {\n            confidence: confidence.toFixed(3),\n            level,\n            actions,\n            verificationCount: confidenceState.verificationCount,\n          });\n\n          if (actions.skipCache) {\n            debug('cache confidence too low, skipping cache entirely', {\n              confidence,\n              level,\n            });\n            isCacheHit = false;\n          }\n\n          try {\n            if (isCacheHit && actions.coordCheck && cachedCenter) {\n              const offset = Math.sqrt(\n                (elementFromCache!.center[0] - cachedCenter[0]) ** 2 +\n                  (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n              );\n\n              debug('cache coord offset check', {\n                cachedCenter,\n                currentCenter: elementFromCache!.center,\n                offset: Math.round(offset),\n                threshold: coordOffsetThreshold,\n              });\n\n              if (offset > coordOffsetThreshold) {\n                debug(\n                  'cache coord offset exceeded threshold, fallback to AI locate',\n                  { offset, threshold: coordOffsetThreshold },\n                );\n                isCacheHit = false;\n              }\n            }\n\n            if (isCacheHit && actions.visualVerify) {\n              const verification = await this.service.verifyCachedElement(\n                elementFromCache!.center,\n                cachePrompt,\n                modelConfigForDefaultIntent,\n                uiContext,\n              );\n              if (!verification.pass) {\n                debug(\n                  'cache hit but visual verification failed, fallback to AI locate',\n                  {\n                    reason: verification.reason,\n                    description: verification.description,\n                    prompt: cachePrompt,\n                  },\n                );\n                isCacheHit = false;\n              } else {\n                debug('cache hit and visual verification passed', {\n                  description: verification.description,\n                });\n              }\n            }\n          } catch (verifyError) {\n            debug(\n              'cache verification error, fallback to AI locate',\n              verifyError,\n            );\n            isCacheHit = false;\n          }\n\n          if (isCacheHit) {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              true,\n            );\n            cacheFeature.confidenceState = updatedState;\n\n            const progressiveRecord = cacheFeature.progressiveRecord as\n              | ProgressiveLocateRecord\n              | undefined;\n            if (progressiveRecord) {\n              const updated = updateProgressiveConvergence(\n                progressiveRecord,\n                elementFromCache!.center,\n                updatedState.confidenceScore,\n              );\n              cacheFeature.progressiveRecord = updated;\n\n              if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n                debug(\n                  'using converged center instead of single-result center',\n                  {\n                    convergedCenter: updated.convergedCenter.map((v) =>\n                      v.toFixed(1),\n                    ),\n                    singleCenter: elementFromCache!.center,\n                    convergenceRadius: updated.convergenceRadius.toFixed(1),\n                    sampleCount: updated.sampleCount,\n                  },\n                );\n                elementFromCache = {\n                  ...elementFromCache!,\n                  center: [\n                    Math.round(updated.convergedCenter[0]),\n                    Math.round(updated.convergedCenter[1]),\n                  ],\n                };\n              }\n            }\n          } else {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              false,\n            );\n            cacheFeature.confidenceState = updatedState;\n          }\n        }\n\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const semanticAnchor = cacheFeature?.semanticAnchor as\n            | SemanticAnchor\n            | undefined;\n\n          if (\n            semanticAnchor &&\n            process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !== 'false'\n          ) {\n            try {\n              const anchorResult = await this.service.locateBySemanticAnchor(\n                semanticAnchor,\n                modelConfigForDefaultIntent,\n                this.interface,\n                uiContext,\n              );\n              if (anchorResult) {\n                elementFromAiLocate = anchorResult;\n                debug(\n                  'semantic anchor locate succeeded, skipping full AI locate',\n                );\n              }\n            } catch (anchorError) {\n              debug('semantic anchor locate failed:', anchorError);\n            }\n          }\n\n          if (!elementFromAiLocate) {\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              locateResult = await this.service.locate(\n                param,\n                {\n                  context: uiContext,\n                },\n                modelConfigForDefaultIntent,\n                abortSignal,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n            } catch (error) {\n              if (error instanceof ServiceError) {\n                applyDump(error.dump);\n              }\n              throw error;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                feature.cachedCenter = pointForCache;\n                feature.confidenceState = createInitialConfidenceState();\n                feature.progressiveRecord =\n                  createInitialProgressiveRecord(pointForCache);\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n\n                const enableSemanticAnchor =\n                  process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !==\n                  'false';\n                if (enableSemanticAnchor) {\n                  try {\n                    const anchor = await this.service.generateSemanticAnchor(\n                      pointForCache,\n                      modelConfigForDefaultIntent,\n                      uiContext,\n                    );\n                    if (anchor) {\n                      feature.semanticAnchor = anchor;\n                      debug(\n                        'semantic anchor generated for prompt: %s',\n                        cachePrompt,\n                      );\n                    }\n                  } catch (anchorError) {\n                    debug('generateSemanticAnchor failed:', anchorError);\n                  }\n                }\n\n                currentCacheEntry = feature;\n                await this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","cacheFeature","cachedCenter","coordOffsetThreshold","Number","process","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","createInitialProgressiveRecord","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqDA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC3C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,MAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,iBAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,mBAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,wBAAwB5B,SAC5C4D,qBAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,oCACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,MAAMtC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI0C;gBAEJ,IAAID,YAAY;oBACd,MAAME,eAAeP;oBACrB,MAAMQ,eAAeD,cAAc;oBAInC,MAAME,uBACJC,OAAO,QAAQ,CACbC,QAAQ,GAAG,CAACC,sCAAsC,IAAI,MACtD,OACG;oBAEP,MAAMC,kBAAmBN,cAAc,mBACrCO;oBACF,MAAMC,aAAaC,oBAAoBH;oBACvC,MAAMI,QAAQC,2BAA2BH;oBACzC,MAAMI,UAAUC,uBAAuBH;oBAEvC5F,MAAM,+BAA+B;wBACnC,YAAY0F,WAAW,OAAO,CAAC;wBAC/BE;wBACAE;wBACA,mBAAmBN,gBAAgB,iBAAiB;oBACtD;oBAEA,IAAIM,QAAQ,SAAS,EAAE;wBACrB9F,MAAM,qDAAqD;4BACzD0F;4BACAE;wBACF;wBACAZ,aAAa;oBACf;oBAEA,IAAI;wBACF,IAAIA,cAAcc,QAAQ,UAAU,IAAIX,cAAc;4BACpD,MAAMa,SAASC,KAAK,IAAI,CACrBnB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGK,YAAY,CAAC,EAAC,KAAM,IAChDL,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGK,YAAY,CAAC,EAAC,KAAM;4BAGvDnF,MAAM,4BAA4B;gCAChCmF;gCACA,eAAeL,iBAAkB,MAAM;gCACvC,QAAQmB,KAAK,KAAK,CAACD;gCACnB,WAAWZ;4BACb;4BAEA,IAAIY,SAASZ,sBAAsB;gCACjCpF,MACE,gEACA;oCAAEgG;oCAAQ,WAAWZ;gCAAqB;gCAE5CJ,aAAa;4BACf;wBACF;wBAEA,IAAIA,cAAcc,QAAQ,YAAY,EAAE;4BACtC,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDpB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;4BAEF,IAAKyD,aAAa,IAAI,EAWpBlG,MAAM,4CAA4C;gCAChD,aAAakG,aAAa,WAAW;4BACvC;iCAbsB;gCACtBlG,MACE,mEACA;oCACE,QAAQkG,aAAa,MAAM;oCAC3B,aAAaA,aAAa,WAAW;oCACrC,QAAQzB;gCACV;gCAEFO,aAAa;4BACf;wBAKF;oBACF,EAAE,OAAOmB,aAAa;wBACpBnG,MACE,mDACAmG;wBAEFnB,aAAa;oBACf;oBAEA,IAAIA,YAAY;wBACd,MAAMoB,eAAeC,yBACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;wBAE/B,MAAME,oBAAoBpB,aAAa,iBAAiB;wBAGxD,IAAIoB,mBAAmB;4BACrB,MAAMC,UAAUC,6BACdF,mBACAxB,iBAAkB,MAAM,EACxBsB,aAAa,eAAe;4BAE9BlB,aAAa,iBAAiB,GAAGqB;4BAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;gCAC7DvG,MACE,0DACA;oCACE,iBAAiBuG,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;oCAEZ,cAAc3B,iBAAkB,MAAM;oCACtC,mBAAmByB,QAAQ,iBAAiB,CAAC,OAAO,CAAC;oCACrD,aAAaA,QAAQ,WAAW;gCAClC;gCAEFzB,mBAAmB;oCACjB,GAAGA,gBAAgB;oCACnB,QAAQ;wCACNmB,KAAK,KAAK,CAACM,QAAQ,eAAe,CAAC,EAAE;wCACrCN,KAAK,KAAK,CAACM,QAAQ,eAAe,CAAC,EAAE;qCACtC;gCACH;4BACF;wBACF;oBACF,OAAO;wBACL,MAAMH,eAAeC,yBACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;oBACjC;gBACF;gBAEA,IAAI,CAAC5B,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMe,eAAeP;oBACrB,MAAM+B,iBAAiBxB,cAAc;oBAIrC,IACEwB,kBACApB,AAAuD,YAAvDA,QAAQ,GAAG,CAACqB,sCAAsC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACA9F,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAImE,cAAc;4BAChB3B,sBAAsB2B;4BACtB5G,MACE;wBAEJ;oBACF,EAAE,OAAO6G,aAAa;wBACpB7G,MAAM,kCAAkC6G;oBAC1C;oBAGF,IAAI,CAAC5B,qBACH,IAAI;wBACFvC,mBAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BmB,sBAAsBnB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiB2D,cACnB/C,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,mBAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAMuE,UACJ9C,mBACAI,oBACAS,oBACAG;gBAGF,MAAM+B,2BAA2B9G,iBAC/BwE,mBAAmB,cAAc;gBAGnC,IAAIuC;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACd,CAAC/B,cACA,EAACb,aAAa,CAAC6C,wBAAuB,KACvC1G,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAI4G,gBAAkCH,QAAQ,MAAM;oBACpD,IAAI/D,AAA6B,MAA7BA,0BAAgC;wBAClCkE,gBAAgB;4BACdjB,KAAK,KAAK,CAACc,QAAQ,MAAM,CAAC,EAAE,GAAG/D;4BAC/BiD,KAAK,KAAK,CAACc,QAAQ,MAAM,CAAC,EAAE,GAAG/D;yBAChC;wBACDhD,MACE,8DACA+G,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAO5G,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiBiH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG1B;wBAC1B0B,QAAQ,iBAAiB,GACvBC,+BAA+BF;wBACjClH,MACE,uCACAyE,aACA0C;wBAGF,MAAME,uBACJ/B,AACA,YADAA,QAAQ,GAAG,CAACqB,sCAAsC;wBAEpD,IAAIU,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDJ,eACAtG,6BACA6B;4BAEF,IAAI6E,QAAQ;gCACVH,QAAQ,cAAc,GAAGG;gCACzBtH,MACE,4CACAyE;4BAEJ;wBACF,EAAE,OAAOoC,aAAa;4BACpB7G,MAAM,kCAAkC6G;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQ1C;4BACR,OAAO0C;wBACT,GACAzC;oBAEJ,OACE1E,MACE,yDACAyE;gBAGN,EAAE,OAAOtB,OAAO;oBACdnD,MAAM,mCAAmCmD;gBAC3C;qBAEAnD,MAAM;gBAIV,IAAI,CAAC+G,SAAS;oBACZ,IAAIlD,YACF,MAAM,IAAIiD,aACR,CAAC,oBAAoB,EAAExG,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIiH;gBAEJ,IAAIpD,WACFoD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMjH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACT+C,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOjH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACTuC,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP5C;wBACA,aAAasC;oBACf;gBACF;gBAGFzD,WAAWuD;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAKtE,UAAU,aAAa;wBAC9B;oBACF;oBACA8E;gBACF;YACF;QACF;QAEA,OAAO3D;IACT;IA9uBA,YAAY,EACV4D,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTjG,WAAW,EACXkG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGjG;QACnB,IAAI,CAAC,eAAe,GAAGkG;IACzB;AAmuBF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import {\n  calculateConfidence,\n  createInitialConfidenceState,\n  createInitialProgressiveRecord,\n  determineVerificationLevel,\n  getVerificationActions,\n  updateConfidenceOnVerify,\n  updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  CacheConfidenceState,\n  CacheValidationOptions,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  ProgressiveLocateRecord,\n  Rect,\n  SemanticAnchor,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n  MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n  MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n  MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord =\n          await this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        let elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        let isCacheHit = !!elementFromCache;\n        const timing = taskContext.task.timing;\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n\n        if (isCacheHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const cachedCenter = cacheFeature?.cachedCenter as\n            | [number, number]\n            | undefined;\n\n          const coordOffsetThreshold =\n            Number.parseInt(\n              process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] || '16',\n              10,\n            ) || 16;\n\n          const confidenceState = (cacheFeature?.confidenceState ||\n            createInitialConfidenceState()) as CacheConfidenceState;\n          const confidence = calculateConfidence(confidenceState);\n          const level = determineVerificationLevel(confidence);\n          const actions = getVerificationActions(level);\n\n          debug('cache confidence assessment', {\n            confidence: confidence.toFixed(3),\n            level,\n            actions,\n            verificationCount: confidenceState.verificationCount,\n          });\n\n          if (actions.skipCache) {\n            debug('cache confidence too low, skipping cache entirely', {\n              confidence,\n              level,\n            });\n            isCacheHit = false;\n          }\n\n          try {\n            if (isCacheHit && actions.coordCheck && cachedCenter) {\n              const offset = Math.sqrt(\n                (elementFromCache!.center[0] - cachedCenter[0]) ** 2 +\n                  (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n              );\n\n              debug('cache coord offset check', {\n                cachedCenter,\n                currentCenter: elementFromCache!.center,\n                offset: Math.round(offset),\n                threshold: coordOffsetThreshold,\n              });\n\n              if (offset > coordOffsetThreshold) {\n                debug(\n                  'cache coord offset exceeded threshold, fallback to AI locate',\n                  { offset, threshold: coordOffsetThreshold },\n                );\n                isCacheHit = false;\n              }\n            }\n\n            if (isCacheHit && actions.visualVerify) {\n              const verification = await this.service.verifyCachedElement(\n                elementFromCache!.center,\n                cachePrompt,\n                modelConfigForDefaultIntent,\n                uiContext,\n              );\n              if (!verification.pass) {\n                debug(\n                  'cache hit but visual verification failed, fallback to AI locate',\n                  {\n                    reason: verification.reason,\n                    description: verification.description,\n                    prompt: cachePrompt,\n                  },\n                );\n                isCacheHit = false;\n              } else {\n                debug('cache hit and visual verification passed', {\n                  description: verification.description,\n                });\n              }\n            }\n          } catch (verifyError) {\n            debug(\n              'cache verification error, fallback to AI locate',\n              verifyError,\n            );\n            isCacheHit = false;\n          }\n\n          if (isCacheHit) {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              true,\n            );\n            cacheFeature.confidenceState = updatedState;\n\n            const progressiveRecord = cacheFeature.progressiveRecord as\n              | ProgressiveLocateRecord\n              | undefined;\n            if (progressiveRecord) {\n              const updated = updateProgressiveConvergence(\n                progressiveRecord,\n                elementFromCache!.center,\n                updatedState.confidenceScore,\n              );\n              cacheFeature.progressiveRecord = updated;\n\n              if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n                debug(\n                  'using converged center instead of single-result center',\n                  {\n                    convergedCenter: updated.convergedCenter.map((v) =>\n                      v.toFixed(1),\n                    ),\n                    singleCenter: elementFromCache!.center,\n                    convergenceRadius: updated.convergenceRadius.toFixed(1),\n                    sampleCount: updated.sampleCount,\n                  },\n                );\n                elementFromCache = {\n                  ...elementFromCache!,\n                  center: [\n                    Math.round(updated.convergedCenter[0]),\n                    Math.round(updated.convergedCenter[1]),\n                  ],\n                };\n              }\n            }\n          } else {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              false,\n            );\n            cacheFeature.confidenceState = updatedState;\n          }\n        }\n\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const semanticAnchor = cacheFeature?.semanticAnchor as\n            | SemanticAnchor\n            | undefined;\n\n          if (\n            semanticAnchor &&\n            process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !== 'false'\n          ) {\n            try {\n              const anchorResult = await this.service.locateBySemanticAnchor(\n                semanticAnchor,\n                modelConfigForDefaultIntent,\n                this.interface,\n                uiContext,\n              );\n              if (anchorResult) {\n                elementFromAiLocate = anchorResult;\n                debug(\n                  'semantic anchor locate succeeded, skipping full AI locate',\n                );\n              }\n            } catch (anchorError) {\n              debug('semantic anchor locate failed:', anchorError);\n            }\n          }\n\n          if (!elementFromAiLocate) {\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              locateResult = await this.service.locate(\n                param,\n                {\n                  context: uiContext,\n                },\n                modelConfigForDefaultIntent,\n                abortSignal,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n            } catch (error) {\n              if (error instanceof ServiceError) {\n                applyDump(error.dump);\n              }\n              throw error;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                feature.cachedCenter = pointForCache;\n                feature.confidenceState = createInitialConfidenceState();\n\n                const existingProgressiveRecord = (\n                  cacheEntry as ElementCacheFeature\n                )?.progressiveRecord as\n                  | ProgressiveLocateRecord\n                  | undefined;\n                if (existingProgressiveRecord) {\n                  feature.progressiveRecord = updateProgressiveConvergence(\n                    existingProgressiveRecord,\n                    pointForCache,\n                    1.0,\n                  );\n                } else {\n                  feature.progressiveRecord =\n                    createInitialProgressiveRecord(pointForCache);\n                }\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n\n                const enableSemanticAnchor =\n                  process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !==\n                  'false';\n                if (enableSemanticAnchor) {\n                  try {\n                    const anchor = await this.service.generateSemanticAnchor(\n                      pointForCache,\n                      modelConfigForDefaultIntent,\n                      uiContext,\n                    );\n                    if (anchor) {\n                      feature.semanticAnchor = anchor;\n                      debug(\n                        'semantic anchor generated for prompt: %s',\n                        cachePrompt,\n                      );\n                    }\n                  } catch (anchorError) {\n                    debug('generateSemanticAnchor failed:', anchorError);\n                  }\n                }\n\n                currentCacheEntry = feature;\n                await this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","cacheFeature","cachedCenter","coordOffsetThreshold","Number","process","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","existingProgressiveRecord","createInitialProgressiveRecord","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqDA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC3C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,MAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,iBAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,mBAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,wBAAwB5B,SAC5C4D,qBAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,oCACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,MAAMtC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI0C;gBAEJ,IAAID,YAAY;oBACd,MAAME,eAAeP;oBACrB,MAAMQ,eAAeD,cAAc;oBAInC,MAAME,uBACJC,OAAO,QAAQ,CACbC,QAAQ,GAAG,CAACC,sCAAsC,IAAI,MACtD,OACG;oBAEP,MAAMC,kBAAmBN,cAAc,mBACrCO;oBACF,MAAMC,aAAaC,oBAAoBH;oBACvC,MAAMI,QAAQC,2BAA2BH;oBACzC,MAAMI,UAAUC,uBAAuBH;oBAEvC5F,MAAM,+BAA+B;wBACnC,YAAY0F,WAAW,OAAO,CAAC;wBAC/BE;wBACAE;wBACA,mBAAmBN,gBAAgB,iBAAiB;oBACtD;oBAEA,IAAIM,QAAQ,SAAS,EAAE;wBACrB9F,MAAM,qDAAqD;4BACzD0F;4BACAE;wBACF;wBACAZ,aAAa;oBACf;oBAEA,IAAI;wBACF,IAAIA,cAAcc,QAAQ,UAAU,IAAIX,cAAc;4BACpD,MAAMa,SAASC,KAAK,IAAI,CACrBnB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGK,YAAY,CAAC,EAAC,KAAM,IAChDL,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGK,YAAY,CAAC,EAAC,KAAM;4BAGvDnF,MAAM,4BAA4B;gCAChCmF;gCACA,eAAeL,iBAAkB,MAAM;gCACvC,QAAQmB,KAAK,KAAK,CAACD;gCACnB,WAAWZ;4BACb;4BAEA,IAAIY,SAASZ,sBAAsB;gCACjCpF,MACE,gEACA;oCAAEgG;oCAAQ,WAAWZ;gCAAqB;gCAE5CJ,aAAa;4BACf;wBACF;wBAEA,IAAIA,cAAcc,QAAQ,YAAY,EAAE;4BACtC,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDpB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;4BAEF,IAAKyD,aAAa,IAAI,EAWpBlG,MAAM,4CAA4C;gCAChD,aAAakG,aAAa,WAAW;4BACvC;iCAbsB;gCACtBlG,MACE,mEACA;oCACE,QAAQkG,aAAa,MAAM;oCAC3B,aAAaA,aAAa,WAAW;oCACrC,QAAQzB;gCACV;gCAEFO,aAAa;4BACf;wBAKF;oBACF,EAAE,OAAOmB,aAAa;wBACpBnG,MACE,mDACAmG;wBAEFnB,aAAa;oBACf;oBAEA,IAAIA,YAAY;wBACd,MAAMoB,eAAeC,yBACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;wBAE/B,MAAME,oBAAoBpB,aAAa,iBAAiB;wBAGxD,IAAIoB,mBAAmB;4BACrB,MAAMC,UAAUC,6BACdF,mBACAxB,iBAAkB,MAAM,EACxBsB,aAAa,eAAe;4BAE9BlB,aAAa,iBAAiB,GAAGqB;4BAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;gCAC7DvG,MACE,0DACA;oCACE,iBAAiBuG,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;oCAEZ,cAAc3B,iBAAkB,MAAM;oCACtC,mBAAmByB,QAAQ,iBAAiB,CAAC,OAAO,CAAC;oCACrD,aAAaA,QAAQ,WAAW;gCAClC;gCAEFzB,mBAAmB;oCACjB,GAAGA,gBAAgB;oCACnB,QAAQ;wCACNmB,KAAK,KAAK,CAACM,QAAQ,eAAe,CAAC,EAAE;wCACrCN,KAAK,KAAK,CAACM,QAAQ,eAAe,CAAC,EAAE;qCACtC;gCACH;4BACF;wBACF;oBACF,OAAO;wBACL,MAAMH,eAAeC,yBACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;oBACjC;gBACF;gBAEA,IAAI,CAAC5B,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMe,eAAeP;oBACrB,MAAM+B,iBAAiBxB,cAAc;oBAIrC,IACEwB,kBACApB,AAAuD,YAAvDA,QAAQ,GAAG,CAACqB,sCAAsC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACA9F,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAImE,cAAc;4BAChB3B,sBAAsB2B;4BACtB5G,MACE;wBAEJ;oBACF,EAAE,OAAO6G,aAAa;wBACpB7G,MAAM,kCAAkC6G;oBAC1C;oBAGF,IAAI,CAAC5B,qBACH,IAAI;wBACFvC,mBAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BmB,sBAAsBnB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiB2D,cACnB/C,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,mBAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAMuE,UACJ9C,mBACAI,oBACAS,oBACAG;gBAGF,MAAM+B,2BAA2B9G,iBAC/BwE,mBAAmB,cAAc;gBAGnC,IAAIuC;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACd,CAAC/B,cACA,EAACb,aAAa,CAAC6C,wBAAuB,KACvC1G,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAI4G,gBAAkCH,QAAQ,MAAM;oBACpD,IAAI/D,AAA6B,MAA7BA,0BAAgC;wBAClCkE,gBAAgB;4BACdjB,KAAK,KAAK,CAACc,QAAQ,MAAM,CAAC,EAAE,GAAG/D;4BAC/BiD,KAAK,KAAK,CAACc,QAAQ,MAAM,CAAC,EAAE,GAAG/D;yBAChC;wBACDhD,MACE,8DACA+G,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAO5G,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiBiH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG1B;wBAE1B,MAAM2B,4BACJzC,YACC;wBAGH,IAAIyC,2BACFD,QAAQ,iBAAiB,GAAGX,6BAC1BY,2BACAF,eACA;6BAGFC,QAAQ,iBAAiB,GACvBE,+BAA+BH;wBAEnClH,MACE,uCACAyE,aACA0C;wBAGF,MAAMG,uBACJhC,AACA,YADAA,QAAQ,GAAG,CAACqB,sCAAsC;wBAEpD,IAAIW,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDL,eACAtG,6BACA6B;4BAEF,IAAI8E,QAAQ;gCACVJ,QAAQ,cAAc,GAAGI;gCACzBvH,MACE,4CACAyE;4BAEJ;wBACF,EAAE,OAAOoC,aAAa;4BACpB7G,MAAM,kCAAkC6G;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQ1C;4BACR,OAAO0C;wBACT,GACAzC;oBAEJ,OACE1E,MACE,yDACAyE;gBAGN,EAAE,OAAOtB,OAAO;oBACdnD,MAAM,mCAAmCmD;gBAC3C;qBAEAnD,MAAM;gBAIV,IAAI,CAAC+G,SAAS;oBACZ,IAAIlD,YACF,MAAM,IAAIiD,aACR,CAAC,oBAAoB,EAAExG,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIkH;gBAEJ,IAAIrD,WACFqD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMlH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACTgD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOlH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACTwC,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP7C;wBACA,aAAasC;oBACf;gBACF;gBAGFzD,WAAWuD;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAKtE,UAAU,aAAa;wBAC9B;oBACF;oBACA+E;gBACF;YACF;QACF;QAEA,OAAO5D;IACT;IA5vBA,YAAY,EACV6D,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTlG,WAAW,EACXmG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGlG;QACnB,IAAI,CAAC,eAAe,GAAGmG;IACzB;AAivBF"}

package/dist/es/agent/utils.mjs

@@ -152,7 +152,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.6.0";
+const getMidsceneVersion = ()=>"1.6.4";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/prompt/semantic-anchor-search.mjs

@@ -13,19 +13,24 @@ TASK: Find the element matching this description in the image.
 
 RESPONSE FORMAT (JSON):
 {
-  "found": true/false,
-  "center": [x, y] or null,
-  "rect": {"left": 0, "top": 0, "width": 0, "height": 0} or null,
-  "confidence": 0.0-1.0,
+  "found": true,
+  "center": [123, 456],
+  "rect": {"left": 100, "top": 400, "width": 50, "height": 100},
+  "confidence": 0.9,
   "reason": "brief explanation"
 }
 
+IMPORTANT:
+- "center" must be an array with exactly 2 numbers: [x_coordinate, y_coordinate]
+- Example: if the element center is at pixel x=320 and y=150, write "center": [320, 150]
+- Do NOT use variable names like x or y, use actual pixel numbers
+
 RULES:
 1. Look for an element matching the visualFingerprint description
 2. Verify it's in the expected position relative to the contextDescription
 3. Return center coordinates and bounding rect if found
 4. Set confidence based on how well the match is
-5. If not found or confidence < 0.5, set found=false
+5. If not found or confidence < 0.5, set found=false and center=null
 6. Write reason in ${preferredLanguage}
 `;
 };

package/dist/es/ai-model/prompt/semantic-anchor-search.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/semantic-anchor-search.mjs","sources":["../../../../src/ai-model/prompt/semantic-anchor-search.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport const semanticAnchorSearchInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are searching for an element based on its semantic anchor description.\nThe image shows a CROPPED AREA of the page around a known landmark element.\n\nTARGET ELEMENT DESCRIPTION:\n- Visual: {{visualFingerprint}}\n- Context: {{contextDescription}}\n\nTASK: Find the element matching this description in the image.\n\nRESPONSE FORMAT (JSON):\n{\n  \"found\": true/false,\n  \"center\": [x, y] or null,\n  \"rect\": {\"left\": 0, \"top\": 0, \"width\": 0, \"height\": 0} or null,\n  \"confidence\": 0.0-1.0,\n  \"reason\": \"brief explanation\"\n}\n\nRULES:\n1. Look for an element matching the visualFingerprint description\n2. Verify it's in the expected position relative to the contextDescription\n3. Return center coordinates and bounding rect if found\n4. Set confidence based on how well the match is\n5. If not found or confidence < 0.5, set found=false\n6. Write reason in ${preferredLanguage}\n`;\n};\n"],"names":["semanticAnchorSearchInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";AAEO,MAAMA,kCAAkC;IAC7C,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;mBAyBS,EAAED,kBAAkB;AACvC,CAAC;AACD"}
+{"version":3,"file":"ai-model/prompt/semantic-anchor-search.mjs","sources":["../../../../src/ai-model/prompt/semantic-anchor-search.ts"],"sourcesContent":["import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport const semanticAnchorSearchInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are searching for an element based on its semantic anchor description.\nThe image shows a CROPPED AREA of the page around a known landmark element.\n\nTARGET ELEMENT DESCRIPTION:\n- Visual: {{visualFingerprint}}\n- Context: {{contextDescription}}\n\nTASK: Find the element matching this description in the image.\n\nRESPONSE FORMAT (JSON):\n{\n  \"found\": true,\n  \"center\": [123, 456],\n  \"rect\": {\"left\": 100, \"top\": 400, \"width\": 50, \"height\": 100},\n  \"confidence\": 0.9,\n  \"reason\": \"brief explanation\"\n}\n\nIMPORTANT:\n- \"center\" must be an array with exactly 2 numbers: [x_coordinate, y_coordinate]\n- Example: if the element center is at pixel x=320 and y=150, write \"center\": [320, 150]\n- Do NOT use variable names like x or y, use actual pixel numbers\n\nRULES:\n1. Look for an element matching the visualFingerprint description\n2. Verify it's in the expected position relative to the contextDescription\n3. Return center coordinates and bounding rect if found\n4. Set confidence based on how well the match is\n5. If not found or confidence < 0.5, set found=false and center=null\n6. Write reason in ${preferredLanguage}\n`;\n};\n"],"names":["semanticAnchorSearchInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";AAEO,MAAMA,kCAAkC;IAC7C,MAAMC,oBAAoBC;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBA8BS,EAAED,kBAAkB;AACvC,CAAC;AACD"}

package/dist/es/service/index.mjs

@@ -446,6 +446,10 @@ class Service {
                 debug('semantic anchor locate: not found or low confidence', content);
                 return null;
             }
+            if (!Array.isArray(content.center) || 2 !== content.center.length || 'number' != typeof content.center[0] || 'number' != typeof content.center[1] || Number.isNaN(content.center[0]) || Number.isNaN(content.center[1])) {
+                debug('semantic anchor locate: invalid center format', content.center, 'expected [number, number]');
+                return null;
+            }
             const adjustedCenter = [
                 content.center[0] + cropArea.left,
                 content.center[1] + cropArea.top

package/dist/es/service/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"service/index.mjs","sources":["../../../src/service/index.ts"],"sourcesContent":["import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  AIResponseParseError,\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { cacheVerifyInstruction } from '@/ai-model/prompt/cache-verify';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport { semanticAnchorGenerateInstruction } from '@/ai-model/prompt/semantic-anchor';\nimport { semanticAnchorSearchInstruction } from '@/ai-model/prompt/semantic-anchor-search';\nimport { type AIArgs, expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  CacheVerificationResult,\n  DetailedLocateParam,\n  Landmark,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    let searchAreaPrompt;\n    if (query.deepLocate) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { modelFamily } = modelConfig;\n\n    if (searchAreaPrompt && !modelFamily) {\n      console.warn(\n        'The \"deepLocate\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    if (searchAreaPrompt && isAutoGLM(modelFamily)) {\n      console.warn('The \"deepLocate\" feature is not supported with AutoGLM.');\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n        abortSignal,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage, reasoning_content } =\n      await AiLocateElement({\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n        modelConfig,\n        abortSignal,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      deepLocate: !!searchArea || !!searchAreaPrompt,\n      error: errorLog,\n      taskInfo,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new ServiceError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          description: elements[0]!.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelConfig,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          matchedElement: [],\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { modelFamily } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        modelFamily === 'qwen2.5-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n\n  async verifyCachedElement(\n    center: [number, number],\n    targetPrompt: TUserPrompt,\n    modelConfig: IModelConfig,\n    uiContext?: UIContext,\n  ): Promise<CacheVerificationResult> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for verifyCachedElement');\n\n    const defaultRectSize = 60;\n    const targetRect: Rect = {\n      left: Math.floor(center[0] - defaultRectSize / 2),\n      top: Math.floor(center[1] - defaultRectSize / 2),\n      width: defaultRectSize,\n      height: defaultRectSize,\n    };\n\n    const cropArea = expandSearchArea(targetRect, shotSize);\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [{ rect: targetRect }],\n      borderThickness: 3,\n    });\n\n    debug(\n      'verifyCachedElement: cropping to area',\n      cropArea,\n      'from center',\n      center,\n    );\n    const croppedResult = await cropByRect(\n      imagePayload,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    imagePayload = croppedResult.imageBase64;\n\n    const promptText =\n      typeof targetPrompt === 'string'\n        ? targetPrompt\n        : targetPrompt.prompt || '';\n\n    const systemPrompt = cacheVerifyInstruction().replace(\n      /{{targetPrompt}}/g,\n      promptText,\n    );\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'low',\n            },\n          },\n        ],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        match: boolean;\n        description?: string;\n        reason?: string;\n        error?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        match?: boolean;\n        description?: string;\n        reason?: string;\n        error?: string;\n      };\n\n      if (content.error) {\n        debug('verifyCachedElement AI returned error:', content.error);\n        return { pass: false, reason: content.error };\n      }\n\n      const result: CacheVerificationResult = {\n        pass: !!content.match,\n        description: content.description,\n        reason: content.reason,\n      };\n\n      debug('verifyCachedElement result:', {\n        pass: result.pass,\n        description: result.description,\n        reason: result.reason,\n        targetPrompt: promptText,\n      });\n\n      return result;\n    } catch (error) {\n      debug('verifyCachedElement failed:', error);\n      return { pass: false, reason: String(error) };\n    }\n  }\n\n  async generateSemanticAnchor(\n    center: [number, number],\n    modelConfig: IModelConfig,\n    uiContext?: UIContext,\n  ): Promise<import('@/types').SemanticAnchor | undefined> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    if (!screenshotBase64) return undefined;\n\n    const defaultRectSize = 60;\n    const targetRect: Rect = {\n      left: Math.floor(center[0] - defaultRectSize / 2),\n      top: Math.floor(center[1] - defaultRectSize / 2),\n      width: defaultRectSize,\n      height: defaultRectSize,\n    };\n\n    const cropArea = expandSearchArea(targetRect, shotSize);\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [{ rect: targetRect }],\n      borderThickness: 3,\n    });\n\n    const croppedResult = await cropByRect(\n      imagePayload,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    imagePayload = croppedResult.imageBase64;\n\n    const systemPrompt = semanticAnchorGenerateInstruction();\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: { url: imagePayload, detail: 'low' },\n          },\n        ],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        visualFingerprint?: string;\n        contextDescription?: string;\n        nearbyLandmarks?: Array<{ description?: string; xpath?: string }>;\n        error?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        visualFingerprint?: string;\n        contextDescription?: string;\n        nearbyLandmarks?: Array<{ description?: string; xpath?: string }>;\n        error?: string;\n      };\n\n      if (content.error || !content.visualFingerprint) {\n        debug('generateSemanticAnchor AI error:', content.error);\n        return undefined;\n      }\n\n      const landmarks: Landmark[] = (content.nearbyLandmarks || [])\n        .filter((l) => l.description && l.xpath)\n        .map((l) => ({\n          description: l.description!,\n          xpath: l.xpath!,\n        }));\n\n      const anchor: import('@/types').SemanticAnchor = {\n        visualFingerprint: content.visualFingerprint,\n        contextDescription: content.contextDescription || '',\n        nearbyLandmarks: landmarks,\n      };\n\n      debug('generateSemanticAnchor result:', anchor);\n      return anchor;\n    } catch (error) {\n      debug('generateSemanticAnchor failed:', error);\n      return undefined;\n    }\n  }\n\n  async locateBySemanticAnchor(\n    anchor: import('@/types').SemanticAnchor,\n    modelConfig: IModelConfig,\n    interfaceInstance: import('@/device').AbstractInterface,\n    uiContext?: UIContext,\n  ): Promise<LocateResultElement | null> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    if (!screenshotBase64) return null;\n\n    let landmarkRect: Rect | undefined;\n\n    for (const landmark of anchor.nearbyLandmarks) {\n      if (interfaceInstance.rectMatchesCacheFeature) {\n        try {\n          const rect = await interfaceInstance.rectMatchesCacheFeature({\n            xpaths: [landmark.xpath],\n          });\n          if (rect) {\n            landmarkRect = rect;\n            debug('semantic anchor: landmark XPath hit:', landmark.description);\n            break;\n          }\n        } catch {\n          debug('semantic anchor: landmark XPath miss:', landmark.description);\n        }\n      }\n    }\n\n    let cropArea: Rect;\n    if (landmarkRect) {\n      const expandSize = 300;\n      cropArea = {\n        left: Math.max(0, landmarkRect.left - expandSize),\n        top: Math.max(0, landmarkRect.top - expandSize),\n        width: landmarkRect.width + expandSize * 2,\n        height: landmarkRect.height + expandSize * 2,\n      };\n    } else {\n      cropArea = {\n        left: 0,\n        top: 0,\n        width: shotSize.width,\n        height: shotSize.height,\n      };\n    }\n\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    const imagePayload: string = croppedResult.imageBase64;\n\n    const systemPrompt = semanticAnchorSearchInstruction()\n      .replace('{{visualFingerprint}}', anchor.visualFingerprint)\n      .replace('{{contextDescription}}', anchor.contextDescription);\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url' as const,\n            image_url: { url: imagePayload, detail: 'low' },\n          },\n        ] as ChatCompletionUserMessageParam['content'],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        found?: boolean;\n        center?: [number, number] | null;\n        rect?: Rect | null;\n        confidence?: number;\n        reason?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        found?: boolean;\n        center?: [number, number] | null;\n        rect?: Rect | null;\n        confidence?: number;\n        reason?: string;\n      };\n\n      if (\n        !content.found ||\n        !content.center ||\n        (content.confidence ?? 0) < 0.5\n      ) {\n        debug('semantic anchor locate: not found or low confidence', content);\n        return null;\n      }\n\n      const adjustedCenter: [number, number] = [\n        content.center[0] + cropArea.left,\n        content.center[1] + cropArea.top,\n      ];\n\n      let adjustedRect: Rect | undefined;\n      if (content.rect) {\n        adjustedRect = {\n          left: content.rect.left + cropArea.left,\n          top: content.rect.top + cropArea.top,\n          width: content.rect.width,\n          height: content.rect.height,\n        };\n      }\n\n      debug(\n        'semantic anchor locate: found at',\n        adjustedCenter,\n        'confidence:',\n        content.confidence,\n      );\n\n      return {\n        center: adjustedCenter,\n        rect: adjustedRect || {\n          left: adjustedCenter[0] - 30,\n          top: adjustedCenter[1] - 15,\n          width: 60,\n          height: 30,\n        },\n        description: `semantic anchor: ${anchor.contextDescription}`,\n      };\n    } catch (error) {\n      debug('locateBySemanticAnchor failed:', error);\n      return null;\n    }\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelConfig","abortSignal","queryPrompt","assert","searchAreaPrompt","modelFamily","console","undefined","isAutoGLM","context","searchArea","searchAreaRawResponse","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","center","targetPrompt","uiContext","cropArea","promptText","cacheVerifyInstruction","String","semanticAnchorGenerateInstruction","landmarks","l","anchor","interfaceInstance","landmarkRect","landmark","expandSize","semanticAnchorSearchInstruction","adjustedCenter","adjustedRect","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAkDA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACzBC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,IAAIM;QACJ,IAAIN,MAAM,UAAU,EAClBM,mBAAmBN,MAAM,MAAM;QAGjC,MAAM,EAAEO,WAAW,EAAE,GAAGL;QAExB,IAAII,oBAAoB,CAACC,aAAa;YACpCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,IAAIH,oBAAoBI,UAAUH,cAAc;YAC9CC,QAAQ,IAAI,CAAC;YACbF,mBAAmBG;QACrB;QAEA,MAAME,UAAUV,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIW;QACJ,IAAIC;QAEJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,gBAAgB;gBACzCJ;gBACA,oBAAoBL;gBACpBJ;gBACAC;YACF;YACAE,OACES,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJD,wBAAwBC,mBAAmB,WAAW;YACpCA,mBAAmB,KAAK;YAC1CF,aAAaE,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,gBAAgB;YACpBZ;YACA,0BAA0BP;YAC1B,cAAcU;YACdZ;YACAC;QACF;QAEF,MAAMqB,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAT;YACAC;YACAS;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,4BAA4B,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASxB;YACX;YACA,gBAAgB,EAAE;YAClB,aAAae;YACb,MAAM;YACN,YAAY,CAAC,CAACP,cAAc,CAAC,CAACN;YAC9B,OAAOqB;YACPF;QACF;QAEA,MAAMI,WAAWX,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMY,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,aACR,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAV;YACAW;QACF;QAGF,OAAO;YACL,SAAS;YACTX;YACAW;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/B/B,WAAyB,EACzBD,GAA0B,EAC1BiC,eAAwB,EACxBC,gBAAoC,EACpCxB,OAAmB,EACe;QAClCN,OAAOM,SAAS;QAChBN,OACE,AAAsB,YAAtB,OAAO4B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMjB,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMc,SAAS,MAAMC,qBAAwB;gBAC3C1B;gBACA,WAAWsB;gBACXE;gBACA,eAAelC;gBACfC;gBACAgC;YACF;YACAhB,cAAckB,OAAO,WAAW;YAChChB,cAAcgB,OAAO,WAAW;YAChCf,QAAQe,OAAO,KAAK;YACpBd,oBAAoBc,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,sBAAsB;gBAEzC,MAAMf,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAac,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMR,OAAOC,kBAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEE;oBAAW;oBACxB,gBAAgB,EAAE;oBAClB,MAAM;oBACNR;oBACA,OAAOa,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIN,aAAaM,MAAM,OAAO,EAAER;YACxC;YACA,MAAMQ;QACR;QAEA,MAAMd,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,qBAAqB,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNR;YACA,OAAOE;QACT;QAEA,MAAM,EAAEa,IAAI,EAAEC,OAAO,EAAE,GAAGvB,eAAe,CAAC;QAE1C,MAAMY,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXY;QACF;QAEA,IAAIb,YAAY,CAACa,MACf,MAAM,IAAIR,aAAaL,UAAUG;QAGnC,OAAO;YACLU;YACAC;YACApB;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJY,MAA+B,EAC/BxC,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOqC,QAAQ;QACf,MAAM/B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEgC,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClDN,OAAOuC,kBAAkB;QAEzB,MAAM,EAAErC,WAAW,EAAE,GAAGL;QACxB,MAAM2C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAI/C,KAAK,YAAY;YACnB,MAAMW,aAAayC,iBAAiBL,YAAYL;YAMhD9C,MAAM,oCAAoCe;YAC1C,MAAM0C,gBAAgB,MAAMC,WAC1BJ,cACAvC,YACAL,AAAgB,iBAAhBA;YAEF4C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,yBAChBF,MACAtD;QAGF,MAAM,EAAEyD,OAAO,EAAE,GAAGF;QACpBpD,OAAO,CAACsD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DtD,OAAOsD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAEA,MAAM,oBACJC,MAAwB,EACxBC,YAAyB,EACzB3D,WAAyB,EACzB4D,SAAqB,EACa;QAClC,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClDN,OAAOuC,kBAAkB;QAEzB,MAAMG,kBAAkB;QACxB,MAAMC,aAAmB;YACvB,MAAME,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV;QAEA,MAAMgB,WAAWV,iBAAiBL,YAAYL;QAE9C,IAAIQ,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBAAC;oBAAE,MAAMK;gBAAW;aAAE;YAC5C,iBAAiB;QACnB;QAEAnD,MACE,yCACAkE,UACA,eACAH;QAEF,MAAMN,gBAAgB,MAAMC,WAC1BJ,cACAY,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzBiD,eAAeG,cAAc,WAAW;QAExC,MAAMU,aACJ,AAAwB,YAAxB,OAAOH,eACHA,eACAA,aAAa,MAAM,IAAI;QAE7B,MAAMhB,eAAeoB,yBAAyB,OAAO,CACnD,qBACAD;QAGF,MAAMR,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,yBAKfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAO3B,IAAIE,QAAQ,KAAK,EAAE;gBACjB9D,MAAM,0CAA0C8D,QAAQ,KAAK;gBAC7D,OAAO;oBAAE,MAAM;oBAAO,QAAQA,QAAQ,KAAK;gBAAC;YAC9C;YAEA,MAAMvB,SAAkC;gBACtC,MAAM,CAAC,CAACuB,QAAQ,KAAK;gBACrB,aAAaA,QAAQ,WAAW;gBAChC,QAAQA,QAAQ,MAAM;YACxB;YAEA9D,MAAM,+BAA+B;gBACnC,MAAMuC,OAAO,IAAI;gBACjB,aAAaA,OAAO,WAAW;gBAC/B,QAAQA,OAAO,MAAM;gBACrB,cAAc4B;YAChB;YAEA,OAAO5B;QACT,EAAE,OAAOE,OAAO;YACdzC,MAAM,+BAA+ByC;YACrC,OAAO;gBAAE,MAAM;gBAAO,QAAQ4B,OAAO5B;YAAO;QAC9C;IACF;IAEA,MAAM,uBACJsB,MAAwB,EACxB1D,WAAyB,EACzB4D,SAAqB,EACkC;QACvD,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClD,IAAI,CAACiC,kBAAkB;QAEvB,MAAMG,kBAAkB;QACxB,MAAMC,aAAmB;YACvB,MAAME,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV;QAEA,MAAMgB,WAAWV,iBAAiBL,YAAYL;QAE9C,IAAIQ,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBAAC;oBAAE,MAAMK;gBAAW;aAAE;YAC5C,iBAAiB;QACnB;QAEA,MAAMM,gBAAgB,MAAMC,WAC1BJ,cACAY,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzBiD,eAAeG,cAAc,WAAW;QAExC,MAAMT,eAAesB;QAErB,MAAMX,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BAAE,KAAKM;4BAAc,QAAQ;wBAAM;oBAChD;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,yBAKfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAO3B,IAAIE,QAAQ,KAAK,IAAI,CAACA,QAAQ,iBAAiB,EAAE,YAC/C9D,MAAM,oCAAoC8D,QAAQ,KAAK;YAIzD,MAAMS,YAAyBT,AAAAA,CAAAA,QAAQ,eAAe,IAAI,EAAC,EACxD,MAAM,CAAC,CAACU,IAAMA,EAAE,WAAW,IAAIA,EAAE,KAAK,EACtC,GAAG,CAAC,CAACA,IAAO;oBACX,aAAaA,EAAE,WAAW;oBAC1B,OAAOA,EAAE,KAAK;gBAChB;YAEF,MAAMC,SAA2C;gBAC/C,mBAAmBX,QAAQ,iBAAiB;gBAC5C,oBAAoBA,QAAQ,kBAAkB,IAAI;gBAClD,iBAAiBS;YACnB;YAEAvE,MAAM,kCAAkCyE;YACxC,OAAOA;QACT,EAAE,OAAOhC,OAAO;YACdzC,MAAM,kCAAkCyC;YACxC;QACF;IACF;IAEA,MAAM,uBACJgC,MAAwC,EACxCpE,WAAyB,EACzBqE,iBAAuD,EACvDT,SAAqB,EACgB;QACrC,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClD,IAAI,CAACiC,kBAAkB,OAAO;QAE9B,IAAI4B;QAEJ,KAAK,MAAMC,YAAYH,OAAO,eAAe,CAC3C,IAAIC,kBAAkB,uBAAuB,EAC3C,IAAI;YACF,MAAMpD,OAAO,MAAMoD,kBAAkB,uBAAuB,CAAC;gBAC3D,QAAQ;oBAACE,SAAS,KAAK;iBAAC;YAC1B;YACA,IAAItD,MAAM;gBACRqD,eAAerD;gBACftB,MAAM,wCAAwC4E,SAAS,WAAW;gBAClE;YACF;QACF,EAAE,OAAM;YACN5E,MAAM,yCAAyC4E,SAAS,WAAW;QACrE;QAIJ,IAAIV;QACJ,IAAIS,cAAc;YAChB,MAAME,aAAa;YACnBX,WAAW;gBACT,MAAMb,KAAK,GAAG,CAAC,GAAGsB,aAAa,IAAI,GAAGE;gBACtC,KAAKxB,KAAK,GAAG,CAAC,GAAGsB,aAAa,GAAG,GAAGE;gBACpC,OAAOF,aAAa,KAAK,GAAGE,AAAa,IAAbA;gBAC5B,QAAQF,aAAa,MAAM,GAAGE,AAAa,IAAbA;YAChC;QACF,OACEX,WAAW;YACT,MAAM;YACN,KAAK;YACL,OAAOpB,SAAS,KAAK;YACrB,QAAQA,SAAS,MAAM;QACzB;QAGF,MAAMW,gBAAgB,MAAMC,WAC1BX,kBACAmB,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzB,MAAMiD,eAAuBG,cAAc,WAAW;QAEtD,MAAMT,eAAe8B,kCAClB,OAAO,CAAC,yBAAyBL,OAAO,iBAAiB,EACzD,OAAO,CAAC,0BAA0BA,OAAO,kBAAkB;QAE9D,MAAMd,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BAAE,KAAKM;4BAAc,QAAQ;wBAAM;oBAChD;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,yBAMfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAQ3B,IACE,CAACE,QAAQ,KAAK,IACd,CAACA,QAAQ,MAAM,IACdA,AAAAA,CAAAA,QAAQ,UAAU,IAAI,KAAK,KAC5B;gBACA9D,MAAM,uDAAuD8D;gBAC7D,OAAO;YACT;YAEA,MAAMiB,iBAAmC;gBACvCjB,QAAQ,MAAM,CAAC,EAAE,GAAGI,SAAS,IAAI;gBACjCJ,QAAQ,MAAM,CAAC,EAAE,GAAGI,SAAS,GAAG;aACjC;YAED,IAAIc;YACJ,IAAIlB,QAAQ,IAAI,EACdkB,eAAe;gBACb,MAAMlB,QAAQ,IAAI,CAAC,IAAI,GAAGI,SAAS,IAAI;gBACvC,KAAKJ,QAAQ,IAAI,CAAC,GAAG,GAAGI,SAAS,GAAG;gBACpC,OAAOJ,QAAQ,IAAI,CAAC,KAAK;gBACzB,QAAQA,QAAQ,IAAI,CAAC,MAAM;YAC7B;YAGF9D,MACE,oCACA+E,gBACA,eACAjB,QAAQ,UAAU;YAGpB,OAAO;gBACL,QAAQiB;gBACR,MAAMC,gBAAgB;oBACpB,MAAMD,cAAc,CAAC,EAAE,GAAG;oBAC1B,KAAKA,cAAc,CAAC,EAAE,GAAG;oBACzB,OAAO;oBACP,QAAQ;gBACV;gBACA,aAAa,CAAC,iBAAiB,EAAEN,OAAO,kBAAkB,EAAE;YAC9D;QACF,EAAE,OAAOhC,OAAO;YACdzC,MAAM,kCAAkCyC;YACxC,OAAO;QACT;IACF;IAvpBA,YACE3B,OAA2D,EAC3DV,GAAoB,CACpB;QAPF;QAEA;QAMEI,OAAOM,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMmE,QAAQ,OAAO,CAACnE;QAGlD,IAAI,AAAyB,WAAlBV,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA0oBF"}
+{"version":3,"file":"service/index.mjs","sources":["../../../src/service/index.ts"],"sourcesContent":["import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  AIResponseParseError,\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { cacheVerifyInstruction } from '@/ai-model/prompt/cache-verify';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport { semanticAnchorGenerateInstruction } from '@/ai-model/prompt/semantic-anchor';\nimport { semanticAnchorSearchInstruction } from '@/ai-model/prompt/semantic-anchor-search';\nimport { type AIArgs, expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  CacheVerificationResult,\n  DetailedLocateParam,\n  Landmark,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    let searchAreaPrompt;\n    if (query.deepLocate) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { modelFamily } = modelConfig;\n\n    if (searchAreaPrompt && !modelFamily) {\n      console.warn(\n        'The \"deepLocate\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    if (searchAreaPrompt && isAutoGLM(modelFamily)) {\n      console.warn('The \"deepLocate\" feature is not supported with AutoGLM.');\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n        abortSignal,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage, reasoning_content } =\n      await AiLocateElement({\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n        modelConfig,\n        abortSignal,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      deepLocate: !!searchArea || !!searchAreaPrompt,\n      error: errorLog,\n      taskInfo,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new ServiceError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          description: elements[0]!.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelConfig,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          matchedElement: [],\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { modelFamily } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        modelFamily === 'qwen2.5-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n\n  async verifyCachedElement(\n    center: [number, number],\n    targetPrompt: TUserPrompt,\n    modelConfig: IModelConfig,\n    uiContext?: UIContext,\n  ): Promise<CacheVerificationResult> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for verifyCachedElement');\n\n    const defaultRectSize = 60;\n    const targetRect: Rect = {\n      left: Math.floor(center[0] - defaultRectSize / 2),\n      top: Math.floor(center[1] - defaultRectSize / 2),\n      width: defaultRectSize,\n      height: defaultRectSize,\n    };\n\n    const cropArea = expandSearchArea(targetRect, shotSize);\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [{ rect: targetRect }],\n      borderThickness: 3,\n    });\n\n    debug(\n      'verifyCachedElement: cropping to area',\n      cropArea,\n      'from center',\n      center,\n    );\n    const croppedResult = await cropByRect(\n      imagePayload,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    imagePayload = croppedResult.imageBase64;\n\n    const promptText =\n      typeof targetPrompt === 'string'\n        ? targetPrompt\n        : targetPrompt.prompt || '';\n\n    const systemPrompt = cacheVerifyInstruction().replace(\n      /{{targetPrompt}}/g,\n      promptText,\n    );\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'low',\n            },\n          },\n        ],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        match: boolean;\n        description?: string;\n        reason?: string;\n        error?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        match?: boolean;\n        description?: string;\n        reason?: string;\n        error?: string;\n      };\n\n      if (content.error) {\n        debug('verifyCachedElement AI returned error:', content.error);\n        return { pass: false, reason: content.error };\n      }\n\n      const result: CacheVerificationResult = {\n        pass: !!content.match,\n        description: content.description,\n        reason: content.reason,\n      };\n\n      debug('verifyCachedElement result:', {\n        pass: result.pass,\n        description: result.description,\n        reason: result.reason,\n        targetPrompt: promptText,\n      });\n\n      return result;\n    } catch (error) {\n      debug('verifyCachedElement failed:', error);\n      return { pass: false, reason: String(error) };\n    }\n  }\n\n  async generateSemanticAnchor(\n    center: [number, number],\n    modelConfig: IModelConfig,\n    uiContext?: UIContext,\n  ): Promise<import('@/types').SemanticAnchor | undefined> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    if (!screenshotBase64) return undefined;\n\n    const defaultRectSize = 60;\n    const targetRect: Rect = {\n      left: Math.floor(center[0] - defaultRectSize / 2),\n      top: Math.floor(center[1] - defaultRectSize / 2),\n      width: defaultRectSize,\n      height: defaultRectSize,\n    };\n\n    const cropArea = expandSearchArea(targetRect, shotSize);\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [{ rect: targetRect }],\n      borderThickness: 3,\n    });\n\n    const croppedResult = await cropByRect(\n      imagePayload,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    imagePayload = croppedResult.imageBase64;\n\n    const systemPrompt = semanticAnchorGenerateInstruction();\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: { url: imagePayload, detail: 'low' },\n          },\n        ],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        visualFingerprint?: string;\n        contextDescription?: string;\n        nearbyLandmarks?: Array<{ description?: string; xpath?: string }>;\n        error?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        visualFingerprint?: string;\n        contextDescription?: string;\n        nearbyLandmarks?: Array<{ description?: string; xpath?: string }>;\n        error?: string;\n      };\n\n      if (content.error || !content.visualFingerprint) {\n        debug('generateSemanticAnchor AI error:', content.error);\n        return undefined;\n      }\n\n      const landmarks: Landmark[] = (content.nearbyLandmarks || [])\n        .filter((l) => l.description && l.xpath)\n        .map((l) => ({\n          description: l.description!,\n          xpath: l.xpath!,\n        }));\n\n      const anchor: import('@/types').SemanticAnchor = {\n        visualFingerprint: content.visualFingerprint,\n        contextDescription: content.contextDescription || '',\n        nearbyLandmarks: landmarks,\n      };\n\n      debug('generateSemanticAnchor result:', anchor);\n      return anchor;\n    } catch (error) {\n      debug('generateSemanticAnchor failed:', error);\n      return undefined;\n    }\n  }\n\n  async locateBySemanticAnchor(\n    anchor: import('@/types').SemanticAnchor,\n    modelConfig: IModelConfig,\n    interfaceInstance: import('@/device').AbstractInterface,\n    uiContext?: UIContext,\n  ): Promise<LocateResultElement | null> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    if (!screenshotBase64) return null;\n\n    let landmarkRect: Rect | undefined;\n\n    for (const landmark of anchor.nearbyLandmarks) {\n      if (interfaceInstance.rectMatchesCacheFeature) {\n        try {\n          const rect = await interfaceInstance.rectMatchesCacheFeature({\n            xpaths: [landmark.xpath],\n          });\n          if (rect) {\n            landmarkRect = rect;\n            debug('semantic anchor: landmark XPath hit:', landmark.description);\n            break;\n          }\n        } catch {\n          debug('semantic anchor: landmark XPath miss:', landmark.description);\n        }\n      }\n    }\n\n    let cropArea: Rect;\n    if (landmarkRect) {\n      const expandSize = 300;\n      cropArea = {\n        left: Math.max(0, landmarkRect.left - expandSize),\n        top: Math.max(0, landmarkRect.top - expandSize),\n        width: landmarkRect.width + expandSize * 2,\n        height: landmarkRect.height + expandSize * 2,\n      };\n    } else {\n      cropArea = {\n        left: 0,\n        top: 0,\n        width: shotSize.width,\n        height: shotSize.height,\n      };\n    }\n\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    const imagePayload: string = croppedResult.imageBase64;\n\n    const systemPrompt = semanticAnchorSearchInstruction()\n      .replace('{{visualFingerprint}}', anchor.visualFingerprint)\n      .replace('{{contextDescription}}', anchor.contextDescription);\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url' as const,\n            image_url: { url: imagePayload, detail: 'low' },\n          },\n        ] as ChatCompletionUserMessageParam['content'],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        found?: boolean;\n        center?: [number, number] | null;\n        rect?: Rect | null;\n        confidence?: number;\n        reason?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        found?: boolean;\n        center?: [number, number] | null;\n        rect?: Rect | null;\n        confidence?: number;\n        reason?: string;\n      };\n\n      if (\n        !content.found ||\n        !content.center ||\n        (content.confidence ?? 0) < 0.5\n      ) {\n        debug('semantic anchor locate: not found or low confidence', content);\n        return null;\n      }\n\n      // Validate center format: must be [number, number]\n      if (\n        !Array.isArray(content.center) ||\n        content.center.length !== 2 ||\n        typeof content.center[0] !== 'number' ||\n        typeof content.center[1] !== 'number' ||\n        Number.isNaN(content.center[0]) ||\n        Number.isNaN(content.center[1])\n      ) {\n        debug(\n          'semantic anchor locate: invalid center format',\n          content.center,\n          'expected [number, number]',\n        );\n        return null;\n      }\n\n      const adjustedCenter: [number, number] = [\n        content.center[0] + cropArea.left,\n        content.center[1] + cropArea.top,\n      ];\n\n      let adjustedRect: Rect | undefined;\n      if (content.rect) {\n        adjustedRect = {\n          left: content.rect.left + cropArea.left,\n          top: content.rect.top + cropArea.top,\n          width: content.rect.width,\n          height: content.rect.height,\n        };\n      }\n\n      debug(\n        'semantic anchor locate: found at',\n        adjustedCenter,\n        'confidence:',\n        content.confidence,\n      );\n\n      return {\n        center: adjustedCenter,\n        rect: adjustedRect || {\n          left: adjustedCenter[0] - 30,\n          top: adjustedCenter[1] - 15,\n          width: 60,\n          height: 30,\n        },\n        description: `semantic anchor: ${anchor.contextDescription}`,\n      };\n    } catch (error) {\n      debug('locateBySemanticAnchor failed:', error);\n      return null;\n    }\n  }\n}\n"],"names":["debug","getDebug","Service","query","opt","modelConfig","abortSignal","queryPrompt","assert","searchAreaPrompt","modelFamily","console","undefined","isAutoGLM","context","searchArea","searchAreaRawResponse","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","center","targetPrompt","uiContext","cropArea","promptText","cacheVerifyInstruction","String","semanticAnchorGenerateInstruction","landmarks","l","anchor","interfaceInstance","landmarkRect","landmark","expandSize","semanticAnchorSearchInstruction","Number","adjustedCenter","adjustedRect","Promise"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AAkDA,MAAMA,QAAQC,SAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACzBC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,OAAOD,aAAa;QAEpBC,OAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,IAAIM;QACJ,IAAIN,MAAM,UAAU,EAClBM,mBAAmBN,MAAM,MAAM;QAGjC,MAAM,EAAEO,WAAW,EAAE,GAAGL;QAExB,IAAII,oBAAoB,CAACC,aAAa;YACpCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,IAAIH,oBAAoBI,UAAUH,cAAc;YAC9CC,QAAQ,IAAI,CAAC;YACbF,mBAAmBG;QACrB;QAEA,MAAME,UAAUV,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIW;QACJ,IAAIC;QAEJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,gBAAgB;gBACzCJ;gBACA,oBAAoBL;gBACpBJ;gBACAC;YACF;YACAE,OACES,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJD,wBAAwBC,mBAAmB,WAAW;YACpCA,mBAAmB,KAAK;YAC1CF,aAAaE,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,gBAAgB;YACpBZ;YACA,0BAA0BP;YAC1B,cAAcU;YACdZ;YACAC;QACF;QAEF,MAAMqB,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAT;YACAC;YACAS;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,4BAA4B,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASxB;YACX;YACA,gBAAgB,EAAE;YAClB,aAAae;YACb,MAAM;YACN,YAAY,CAAC,CAACP,cAAc,CAAC,CAACN;YAC9B,OAAOqB;YACPF;QACF;QAEA,MAAMI,WAAWX,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMY,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,aAAaL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,aACR,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAV;YACAW;QACF;QAGF,OAAO;YACL,SAAS;YACTX;YACAW;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/B/B,WAAyB,EACzBD,GAA0B,EAC1BiC,eAAwB,EACxBC,gBAAoC,EACpCxB,OAAmB,EACe;QAClCN,OAAOM,SAAS;QAChBN,OACE,AAAsB,YAAtB,OAAO4B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMjB,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMc,SAAS,MAAMC,qBAAwB;gBAC3C1B;gBACA,WAAWsB;gBACXE;gBACA,eAAelC;gBACfC;gBACAgC;YACF;YACAhB,cAAckB,OAAO,WAAW;YAChChB,cAAcgB,OAAO,WAAW;YAChCf,QAAQe,OAAO,KAAK;YACpBd,oBAAoBc,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,sBAAsB;gBAEzC,MAAMf,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAac,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMR,OAAOC,kBAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEE;oBAAW;oBACxB,gBAAgB,EAAE;oBAClB,MAAM;oBACNR;oBACA,OAAOa,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIN,aAAaM,MAAM,OAAO,EAAER;YACxC;YACA,MAAMQ;QACR;QAEA,MAAMd,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,qBAAqB,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNR;YACA,OAAOE;QACT;QAEA,MAAM,EAAEa,IAAI,EAAEC,OAAO,EAAE,GAAGvB,eAAe,CAAC;QAE1C,MAAMY,OAAOC,kBAAkB;YAC7B,GAAGH,QAAQ;YACXY;QACF;QAEA,IAAIb,YAAY,CAACa,MACf,MAAM,IAAIR,aAAaL,UAAUG;QAGnC,OAAO;YACLU;YACAC;YACApB;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJY,MAA+B,EAC/BxC,WAAyB,EACzBD,GAEC,EACwD;QACzDI,OAAOqC,QAAQ;QACf,MAAM/B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEgC,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClDN,OAAOuC,kBAAkB;QAEzB,MAAM,EAAErC,WAAW,EAAE,GAAGL;QACxB,MAAM2C,eAAeC;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAI/C,KAAK,YAAY;YACnB,MAAMW,aAAayC,iBAAiBL,YAAYL;YAMhD9C,MAAM,oCAAoCe;YAC1C,MAAM0C,gBAAgB,MAAMC,WAC1BJ,cACAvC,YACAL,AAAgB,iBAAhBA;YAEF4C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,yBAChBF,MACAtD;QAGF,MAAM,EAAEyD,OAAO,EAAE,GAAGF;QACpBpD,OAAO,CAACsD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DtD,OAAOsD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAEA,MAAM,oBACJC,MAAwB,EACxBC,YAAyB,EACzB3D,WAAyB,EACzB4D,SAAqB,EACa;QAClC,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClDN,OAAOuC,kBAAkB;QAEzB,MAAMG,kBAAkB;QACxB,MAAMC,aAAmB;YACvB,MAAME,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV;QAEA,MAAMgB,WAAWV,iBAAiBL,YAAYL;QAE9C,IAAIQ,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBAAC;oBAAE,MAAMK;gBAAW;aAAE;YAC5C,iBAAiB;QACnB;QAEAnD,MACE,yCACAkE,UACA,eACAH;QAEF,MAAMN,gBAAgB,MAAMC,WAC1BJ,cACAY,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzBiD,eAAeG,cAAc,WAAW;QAExC,MAAMU,aACJ,AAAwB,YAAxB,OAAOH,eACHA,eACAA,aAAa,MAAM,IAAI;QAE7B,MAAMhB,eAAeoB,yBAAyB,OAAO,CACnD,qBACAD;QAGF,MAAMR,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,yBAKfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAO3B,IAAIE,QAAQ,KAAK,EAAE;gBACjB9D,MAAM,0CAA0C8D,QAAQ,KAAK;gBAC7D,OAAO;oBAAE,MAAM;oBAAO,QAAQA,QAAQ,KAAK;gBAAC;YAC9C;YAEA,MAAMvB,SAAkC;gBACtC,MAAM,CAAC,CAACuB,QAAQ,KAAK;gBACrB,aAAaA,QAAQ,WAAW;gBAChC,QAAQA,QAAQ,MAAM;YACxB;YAEA9D,MAAM,+BAA+B;gBACnC,MAAMuC,OAAO,IAAI;gBACjB,aAAaA,OAAO,WAAW;gBAC/B,QAAQA,OAAO,MAAM;gBACrB,cAAc4B;YAChB;YAEA,OAAO5B;QACT,EAAE,OAAOE,OAAO;YACdzC,MAAM,+BAA+ByC;YACrC,OAAO;gBAAE,MAAM;gBAAO,QAAQ4B,OAAO5B;YAAO;QAC9C;IACF;IAEA,MAAM,uBACJsB,MAAwB,EACxB1D,WAAyB,EACzB4D,SAAqB,EACkC;QACvD,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClD,IAAI,CAACiC,kBAAkB;QAEvB,MAAMG,kBAAkB;QACxB,MAAMC,aAAmB;YACvB,MAAME,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV;QAEA,MAAMgB,WAAWV,iBAAiBL,YAAYL;QAE9C,IAAIQ,eAAe,MAAMC,wBAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBAAC;oBAAE,MAAMK;gBAAW;aAAE;YAC5C,iBAAiB;QACnB;QAEA,MAAMM,gBAAgB,MAAMC,WAC1BJ,cACAY,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzBiD,eAAeG,cAAc,WAAW;QAExC,MAAMT,eAAesB;QAErB,MAAMX,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BAAE,KAAKM;4BAAc,QAAQ;wBAAM;oBAChD;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,yBAKfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAO3B,IAAIE,QAAQ,KAAK,IAAI,CAACA,QAAQ,iBAAiB,EAAE,YAC/C9D,MAAM,oCAAoC8D,QAAQ,KAAK;YAIzD,MAAMS,YAAyBT,AAAAA,CAAAA,QAAQ,eAAe,IAAI,EAAC,EACxD,MAAM,CAAC,CAACU,IAAMA,EAAE,WAAW,IAAIA,EAAE,KAAK,EACtC,GAAG,CAAC,CAACA,IAAO;oBACX,aAAaA,EAAE,WAAW;oBAC1B,OAAOA,EAAE,KAAK;gBAChB;YAEF,MAAMC,SAA2C;gBAC/C,mBAAmBX,QAAQ,iBAAiB;gBAC5C,oBAAoBA,QAAQ,kBAAkB,IAAI;gBAClD,iBAAiBS;YACnB;YAEAvE,MAAM,kCAAkCyE;YACxC,OAAOA;QACT,EAAE,OAAOhC,OAAO;YACdzC,MAAM,kCAAkCyC;YACxC;QACF;IACF;IAEA,MAAM,uBACJgC,MAAwC,EACxCpE,WAAyB,EACzBqE,iBAAuD,EACvDT,SAAqB,EACgB;QACrC,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClD,IAAI,CAACiC,kBAAkB,OAAO;QAE9B,IAAI4B;QAEJ,KAAK,MAAMC,YAAYH,OAAO,eAAe,CAC3C,IAAIC,kBAAkB,uBAAuB,EAC3C,IAAI;YACF,MAAMpD,OAAO,MAAMoD,kBAAkB,uBAAuB,CAAC;gBAC3D,QAAQ;oBAACE,SAAS,KAAK;iBAAC;YAC1B;YACA,IAAItD,MAAM;gBACRqD,eAAerD;gBACftB,MAAM,wCAAwC4E,SAAS,WAAW;gBAClE;YACF;QACF,EAAE,OAAM;YACN5E,MAAM,yCAAyC4E,SAAS,WAAW;QACrE;QAIJ,IAAIV;QACJ,IAAIS,cAAc;YAChB,MAAME,aAAa;YACnBX,WAAW;gBACT,MAAMb,KAAK,GAAG,CAAC,GAAGsB,aAAa,IAAI,GAAGE;gBACtC,KAAKxB,KAAK,GAAG,CAAC,GAAGsB,aAAa,GAAG,GAAGE;gBACpC,OAAOF,aAAa,KAAK,GAAGE,AAAa,IAAbA;gBAC5B,QAAQF,aAAa,MAAM,GAAGE,AAAa,IAAbA;YAChC;QACF,OACEX,WAAW;YACT,MAAM;YACN,KAAK;YACL,OAAOpB,SAAS,KAAK;YACrB,QAAQA,SAAS,MAAM;QACzB;QAGF,MAAMW,gBAAgB,MAAMC,WAC1BX,kBACAmB,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzB,MAAMiD,eAAuBG,cAAc,WAAW;QAEtD,MAAMT,eAAe8B,kCAClB,OAAO,CAAC,yBAAyBL,OAAO,iBAAiB,EACzD,OAAO,CAAC,0BAA0BA,OAAO,kBAAkB;QAE9D,MAAMd,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BAAE,KAAKM;4BAAc,QAAQ;wBAAM;oBAChD;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,yBAMfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAQ3B,IACE,CAACE,QAAQ,KAAK,IACd,CAACA,QAAQ,MAAM,IACdA,AAAAA,CAAAA,QAAQ,UAAU,IAAI,KAAK,KAC5B;gBACA9D,MAAM,uDAAuD8D;gBAC7D,OAAO;YACT;YAGA,IACE,CAACV,MAAM,OAAO,CAACU,QAAQ,MAAM,KAC7BA,AAA0B,MAA1BA,QAAQ,MAAM,CAAC,MAAM,IACrB,AAA6B,YAA7B,OAAOA,QAAQ,MAAM,CAAC,EAAE,IACxB,AAA6B,YAA7B,OAAOA,QAAQ,MAAM,CAAC,EAAE,IACxBiB,OAAO,KAAK,CAACjB,QAAQ,MAAM,CAAC,EAAE,KAC9BiB,OAAO,KAAK,CAACjB,QAAQ,MAAM,CAAC,EAAE,GAC9B;gBACA9D,MACE,iDACA8D,QAAQ,MAAM,EACd;gBAEF,OAAO;YACT;YAEA,MAAMkB,iBAAmC;gBACvClB,QAAQ,MAAM,CAAC,EAAE,GAAGI,SAAS,IAAI;gBACjCJ,QAAQ,MAAM,CAAC,EAAE,GAAGI,SAAS,GAAG;aACjC;YAED,IAAIe;YACJ,IAAInB,QAAQ,IAAI,EACdmB,eAAe;gBACb,MAAMnB,QAAQ,IAAI,CAAC,IAAI,GAAGI,SAAS,IAAI;gBACvC,KAAKJ,QAAQ,IAAI,CAAC,GAAG,GAAGI,SAAS,GAAG;gBACpC,OAAOJ,QAAQ,IAAI,CAAC,KAAK;gBACzB,QAAQA,QAAQ,IAAI,CAAC,MAAM;YAC7B;YAGF9D,MACE,oCACAgF,gBACA,eACAlB,QAAQ,UAAU;YAGpB,OAAO;gBACL,QAAQkB;gBACR,MAAMC,gBAAgB;oBACpB,MAAMD,cAAc,CAAC,EAAE,GAAG;oBAC1B,KAAKA,cAAc,CAAC,EAAE,GAAG;oBACzB,OAAO;oBACP,QAAQ;gBACV;gBACA,aAAa,CAAC,iBAAiB,EAAEP,OAAO,kBAAkB,EAAE;YAC9D;QACF,EAAE,OAAOhC,OAAO;YACdzC,MAAM,kCAAkCyC;YACxC,OAAO;QACT;IACF;IAxqBA,YACE3B,OAA2D,EAC3DV,GAAoB,CACpB;QAPF;QAEA;QAMEI,OAAOM,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMoE,QAAQ,OAAO,CAACpE;QAGlD,IAAI,AAAyB,WAAlBV,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA2pBF"}

package/dist/es/utils.mjs

@@ -159,7 +159,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "1.6.0";
+    return "1.6.4";
 }
 function debugLog(...message) {
     const debugMode = process.env[MIDSCENE_DEBUG_MODE];

package/dist/lib/agent/task-builder.js

@@ -392,7 +392,9 @@ class TaskBuilder {
                     if (hasNonEmptyCache(feature)) {
                         feature.cachedCenter = pointForCache;
                         feature.confidenceState = (0, external_cache_confidence_js_namespaceObject.createInitialConfidenceState)();
-                        feature.progressiveRecord = (0, external_cache_confidence_js_namespaceObject.createInitialProgressiveRecord)(pointForCache);
+                        const existingProgressiveRecord = cacheEntry?.progressiveRecord;
+                        if (existingProgressiveRecord) feature.progressiveRecord = (0, external_cache_confidence_js_namespaceObject.updateProgressiveConvergence)(existingProgressiveRecord, pointForCache, 1.0);
+                        else feature.progressiveRecord = (0, external_cache_confidence_js_namespaceObject.createInitialProgressiveRecord)(pointForCache);
                         debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
                         const enableSemanticAnchor = 'false' !== process.env[constants_namespaceObject.MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR];
                         if (enableSemanticAnchor) try {

package/dist/lib/agent/task-builder.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/task-builder.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  calculateConfidence,\n  createInitialConfidenceState,\n  createInitialProgressiveRecord,\n  determineVerificationLevel,\n  getVerificationActions,\n  updateConfidenceOnVerify,\n  updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  CacheConfidenceState,\n  CacheValidationOptions,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  ProgressiveLocateRecord,\n  Rect,\n  SemanticAnchor,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n  MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n  MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n  MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord =\n          await this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        let elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        let isCacheHit = !!elementFromCache;\n        const timing = taskContext.task.timing;\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n\n        if (isCacheHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const cachedCenter = cacheFeature?.cachedCenter as\n            | [number, number]\n            | undefined;\n\n          const coordOffsetThreshold =\n            Number.parseInt(\n              process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] || '16',\n              10,\n            ) || 16;\n\n          const confidenceState = (cacheFeature?.confidenceState ||\n            createInitialConfidenceState()) as CacheConfidenceState;\n          const confidence = calculateConfidence(confidenceState);\n          const level = determineVerificationLevel(confidence);\n          const actions = getVerificationActions(level);\n\n          debug('cache confidence assessment', {\n            confidence: confidence.toFixed(3),\n            level,\n            actions,\n            verificationCount: confidenceState.verificationCount,\n          });\n\n          if (actions.skipCache) {\n            debug('cache confidence too low, skipping cache entirely', {\n              confidence,\n              level,\n            });\n            isCacheHit = false;\n          }\n\n          try {\n            if (isCacheHit && actions.coordCheck && cachedCenter) {\n              const offset = Math.sqrt(\n                (elementFromCache!.center[0] - cachedCenter[0]) ** 2 +\n                  (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n              );\n\n              debug('cache coord offset check', {\n                cachedCenter,\n                currentCenter: elementFromCache!.center,\n                offset: Math.round(offset),\n                threshold: coordOffsetThreshold,\n              });\n\n              if (offset > coordOffsetThreshold) {\n                debug(\n                  'cache coord offset exceeded threshold, fallback to AI locate',\n                  { offset, threshold: coordOffsetThreshold },\n                );\n                isCacheHit = false;\n              }\n            }\n\n            if (isCacheHit && actions.visualVerify) {\n              const verification = await this.service.verifyCachedElement(\n                elementFromCache!.center,\n                cachePrompt,\n                modelConfigForDefaultIntent,\n                uiContext,\n              );\n              if (!verification.pass) {\n                debug(\n                  'cache hit but visual verification failed, fallback to AI locate',\n                  {\n                    reason: verification.reason,\n                    description: verification.description,\n                    prompt: cachePrompt,\n                  },\n                );\n                isCacheHit = false;\n              } else {\n                debug('cache hit and visual verification passed', {\n                  description: verification.description,\n                });\n              }\n            }\n          } catch (verifyError) {\n            debug(\n              'cache verification error, fallback to AI locate',\n              verifyError,\n            );\n            isCacheHit = false;\n          }\n\n          if (isCacheHit) {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              true,\n            );\n            cacheFeature.confidenceState = updatedState;\n\n            const progressiveRecord = cacheFeature.progressiveRecord as\n              | ProgressiveLocateRecord\n              | undefined;\n            if (progressiveRecord) {\n              const updated = updateProgressiveConvergence(\n                progressiveRecord,\n                elementFromCache!.center,\n                updatedState.confidenceScore,\n              );\n              cacheFeature.progressiveRecord = updated;\n\n              if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n                debug(\n                  'using converged center instead of single-result center',\n                  {\n                    convergedCenter: updated.convergedCenter.map((v) =>\n                      v.toFixed(1),\n                    ),\n                    singleCenter: elementFromCache!.center,\n                    convergenceRadius: updated.convergenceRadius.toFixed(1),\n                    sampleCount: updated.sampleCount,\n                  },\n                );\n                elementFromCache = {\n                  ...elementFromCache!,\n                  center: [\n                    Math.round(updated.convergedCenter[0]),\n                    Math.round(updated.convergedCenter[1]),\n                  ],\n                };\n              }\n            }\n          } else {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              false,\n            );\n            cacheFeature.confidenceState = updatedState;\n          }\n        }\n\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const semanticAnchor = cacheFeature?.semanticAnchor as\n            | SemanticAnchor\n            | undefined;\n\n          if (\n            semanticAnchor &&\n            process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !== 'false'\n          ) {\n            try {\n              const anchorResult = await this.service.locateBySemanticAnchor(\n                semanticAnchor,\n                modelConfigForDefaultIntent,\n                this.interface,\n                uiContext,\n              );\n              if (anchorResult) {\n                elementFromAiLocate = anchorResult;\n                debug(\n                  'semantic anchor locate succeeded, skipping full AI locate',\n                );\n              }\n            } catch (anchorError) {\n              debug('semantic anchor locate failed:', anchorError);\n            }\n          }\n\n          if (!elementFromAiLocate) {\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              locateResult = await this.service.locate(\n                param,\n                {\n                  context: uiContext,\n                },\n                modelConfigForDefaultIntent,\n                abortSignal,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n            } catch (error) {\n              if (error instanceof ServiceError) {\n                applyDump(error.dump);\n              }\n              throw error;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                feature.cachedCenter = pointForCache;\n                feature.confidenceState = createInitialConfidenceState();\n                feature.progressiveRecord =\n                  createInitialProgressiveRecord(pointForCache);\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n\n                const enableSemanticAnchor =\n                  process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !==\n                  'false';\n                if (enableSemanticAnchor) {\n                  try {\n                    const anchor = await this.service.generateSemanticAnchor(\n                      pointForCache,\n                      modelConfigForDefaultIntent,\n                      uiContext,\n                    );\n                    if (anchor) {\n                      feature.semanticAnchor = anchor;\n                      debug(\n                        'semantic anchor generated for prompt: %s',\n                        cachePrompt,\n                      );\n                    }\n                  } catch (anchorError) {\n                    debug('generateSemanticAnchor failed:', anchorError);\n                  }\n                }\n\n                currentCacheEntry = feature;\n                await this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","hasNonEmptyCache","cache","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","cacheFeature","cachedCenter","coordOffsetThreshold","Number","process","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","createInitialProgressiveRecord","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC+CA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPP,OAAO,IAAI,CAACO,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnD/B,MACE,uCACA,CAAC,YAAY,EAAEuB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,EAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3EvB,MAAM,CAAC,OAAO,EAAE+B,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCtC,MACE,oBACAuB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,IAAAA,sBAAAA,MAAAA,AAAAA,EACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC1C,MACE,CAAC,8DAA8D,EAAEyB,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDL,MACE,CAAC,2DAA2D,EAAEyB,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,IAAAA,kCAAAA,KAAAA,AAAAA,EAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,AAAAA,IAAAA,yBAAAA,gBAAAA,AAAAA,EAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3BvC,MAAM,kBAAkByB,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC3BvC,MAAM,iBAAiByB,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCrD,MACE,CAAC,6DAA6D,EAAEyB,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDL,MACE,CAAC,0DAA0D,EAAEyB,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,IAAAA,sBAAAA,MAAAA,AAAAA,EACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,EAAwB5B,SAC5C4D,AAAAA,IAAAA,oCAAAA,oBAAAA,AAAAA,EAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EAEEC,AAAAA,IAAAA,oCAAAA,oCAAAA,AAAAA,EACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,AAAAA,IAAAA,oCAAAA,qBAAAA,AAAAA,EACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,AAAAA,IAAAA,oCAAAA,mCAAAA,AAAAA,EACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,MAAMtC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI0C;gBAEJ,IAAID,YAAY;oBACd,MAAME,eAAeP;oBACrB,MAAMQ,eAAeD,cAAc;oBAInC,MAAME,uBACJC,OAAO,QAAQ,CACbC,QAAQ,GAAG,CAACC,0BAAAA,qCAAqCA,CAAC,IAAI,MACtD,OACG;oBAEP,MAAMC,kBAAmBN,cAAc,mBACrCO,AAAAA,IAAAA,6CAAAA,4BAAAA,AAAAA;oBACF,MAAMC,aAAaC,AAAAA,IAAAA,6CAAAA,mBAAAA,AAAAA,EAAoBH;oBACvC,MAAMI,QAAQC,AAAAA,IAAAA,6CAAAA,0BAAAA,AAAAA,EAA2BH;oBACzC,MAAMI,UAAUC,AAAAA,IAAAA,6CAAAA,sBAAAA,AAAAA,EAAuBH;oBAEvC3F,MAAM,+BAA+B;wBACnC,YAAYyF,WAAW,OAAO,CAAC;wBAC/BE;wBACAE;wBACA,mBAAmBN,gBAAgB,iBAAiB;oBACtD;oBAEA,IAAIM,QAAQ,SAAS,EAAE;wBACrB7F,MAAM,qDAAqD;4BACzDyF;4BACAE;wBACF;wBACAZ,aAAa;oBACf;oBAEA,IAAI;wBACF,IAAIA,cAAcc,QAAQ,UAAU,IAAIX,cAAc;4BACpD,MAAMa,SAASC,KAAK,IAAI,CACrBnB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGK,YAAY,CAAC,EAAC,KAAM,IAChDL,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGK,YAAY,CAAC,EAAC,KAAM;4BAGvDlF,MAAM,4BAA4B;gCAChCkF;gCACA,eAAeL,iBAAkB,MAAM;gCACvC,QAAQmB,KAAK,KAAK,CAACD;gCACnB,WAAWZ;4BACb;4BAEA,IAAIY,SAASZ,sBAAsB;gCACjCnF,MACE,gEACA;oCAAE+F;oCAAQ,WAAWZ;gCAAqB;gCAE5CJ,aAAa;4BACf;wBACF;wBAEA,IAAIA,cAAcc,QAAQ,YAAY,EAAE;4BACtC,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDpB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;4BAEF,IAAKyD,aAAa,IAAI,EAWpBjG,MAAM,4CAA4C;gCAChD,aAAaiG,aAAa,WAAW;4BACvC;iCAbsB;gCACtBjG,MACE,mEACA;oCACE,QAAQiG,aAAa,MAAM;oCAC3B,aAAaA,aAAa,WAAW;oCACrC,QAAQzB;gCACV;gCAEFO,aAAa;4BACf;wBAKF;oBACF,EAAE,OAAOmB,aAAa;wBACpBlG,MACE,mDACAkG;wBAEFnB,aAAa;oBACf;oBAEA,IAAIA,YAAY;wBACd,MAAMoB,eAAeC,AAAAA,IAAAA,6CAAAA,wBAAAA,AAAAA,EACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;wBAE/B,MAAME,oBAAoBpB,aAAa,iBAAiB;wBAGxD,IAAIoB,mBAAmB;4BACrB,MAAMC,UAAUC,AAAAA,IAAAA,6CAAAA,4BAAAA,AAAAA,EACdF,mBACAxB,iBAAkB,MAAM,EACxBsB,aAAa,eAAe;4BAE9BlB,aAAa,iBAAiB,GAAGqB;4BAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;gCAC7DtG,MACE,0DACA;oCACE,iBAAiBsG,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;oCAEZ,cAAc3B,iBAAkB,MAAM;oCACtC,mBAAmByB,QAAQ,iBAAiB,CAAC,OAAO,CAAC;oCACrD,aAAaA,QAAQ,WAAW;gCAClC;gCAEFzB,mBAAmB;oCACjB,GAAGA,gBAAgB;oCACnB,QAAQ;wCACNmB,KAAK,KAAK,CAACM,QAAQ,eAAe,CAAC,EAAE;wCACrCN,KAAK,KAAK,CAACM,QAAQ,eAAe,CAAC,EAAE;qCACtC;gCACH;4BACF;wBACF;oBACF,OAAO;wBACL,MAAMH,eAAeC,AAAAA,IAAAA,6CAAAA,wBAAAA,AAAAA,EACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;oBACjC;gBACF;gBAEA,IAAI,CAAC5B,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMe,eAAeP;oBACrB,MAAM+B,iBAAiBxB,cAAc;oBAIrC,IACEwB,kBACApB,AAAuD,YAAvDA,QAAQ,GAAG,CAACqB,0BAAAA,qCAAqCA,CAAC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACA9F,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAImE,cAAc;4BAChB3B,sBAAsB2B;4BACtB3G,MACE;wBAEJ;oBACF,EAAE,OAAO4G,aAAa;wBACpB5G,MAAM,kCAAkC4G;oBAC1C;oBAGF,IAAI,CAAC5B,qBACH,IAAI;wBACFvC,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BmB,sBAAsBnB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiB2D,kCAAAA,YAAYA,EAC/B/C,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAMuE,UACJ9C,mBACAI,oBACAS,oBACAG;gBAGF,MAAM+B,2BAA2B7G,iBAC/BuE,mBAAmB,cAAc;gBAGnC,IAAIuC;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACd,CAAC/B,cACA,EAACb,aAAa,CAAC6C,wBAAuB,KACvC1G,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAI4G,gBAAkCH,QAAQ,MAAM;oBACpD,IAAI/D,AAA6B,MAA7BA,0BAAgC;wBAClCkE,gBAAgB;4BACdjB,KAAK,KAAK,CAACc,QAAQ,MAAM,CAAC,EAAE,GAAG/D;4BAC/BiD,KAAK,KAAK,CAACc,QAAQ,MAAM,CAAC,EAAE,GAAG/D;yBAChC;wBACD/C,MACE,8DACA8G,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAO5G,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIT,iBAAiBgH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG1B,AAAAA,IAAAA,6CAAAA,4BAAAA,AAAAA;wBAC1B0B,QAAQ,iBAAiB,GACvBC,AAAAA,IAAAA,6CAAAA,8BAAAA,AAAAA,EAA+BF;wBACjCjH,MACE,uCACAwE,aACA0C;wBAGF,MAAME,uBACJ/B,AACA,YADAA,QAAQ,GAAG,CAACqB,0BAAAA,qCAAqCA,CAAC;wBAEpD,IAAIU,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDJ,eACAtG,6BACA6B;4BAEF,IAAI6E,QAAQ;gCACVH,QAAQ,cAAc,GAAGG;gCACzBrH,MACE,4CACAwE;4BAEJ;wBACF,EAAE,OAAOoC,aAAa;4BACpB5G,MAAM,kCAAkC4G;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQ1C;4BACR,OAAO0C;wBACT,GACAzC;oBAEJ,OACEzE,MACE,yDACAwE;gBAGN,EAAE,OAAOtB,OAAO;oBACdlD,MAAM,mCAAmCkD;gBAC3C;qBAEAlD,MAAM;gBAIV,IAAI,CAAC8G,SAAS;oBACZ,IAAIlD,YACF,MAAM,IAAIiD,kCAAAA,YAAYA,CACpB,CAAC,oBAAoB,EAAExG,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIiH;gBAEJ,IAAIpD,WACFoD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMjH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACT+C,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOjH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACTuC,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP5C;wBACA,aAAasC;oBACf;gBACF;gBAGFzD,WAAWuD;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAKtE,UAAU,aAAa;wBAC9B;oBACF;oBACA8E;gBACF;YACF;QACF;QAEA,OAAO3D;IACT;IA9uBA,YAAY,EACV4D,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTjG,WAAW,EACXkG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGjG;QACnB,IAAI,CAAC,eAAe,GAAGkG;IACzB;AAmuBF"}
+{"version":3,"file":"agent/task-builder.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/task-builder.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  calculateConfidence,\n  createInitialConfidenceState,\n  createInitialProgressiveRecord,\n  determineVerificationLevel,\n  getVerificationActions,\n  updateConfidenceOnVerify,\n  updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  CacheConfidenceState,\n  CacheValidationOptions,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  ProgressiveLocateRecord,\n  Rect,\n  SemanticAnchor,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n  MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n  MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n  MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord =\n          await this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        let elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        let isCacheHit = !!elementFromCache;\n        const timing = taskContext.task.timing;\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n\n        if (isCacheHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const cachedCenter = cacheFeature?.cachedCenter as\n            | [number, number]\n            | undefined;\n\n          const coordOffsetThreshold =\n            Number.parseInt(\n              process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] || '16',\n              10,\n            ) || 16;\n\n          const confidenceState = (cacheFeature?.confidenceState ||\n            createInitialConfidenceState()) as CacheConfidenceState;\n          const confidence = calculateConfidence(confidenceState);\n          const level = determineVerificationLevel(confidence);\n          const actions = getVerificationActions(level);\n\n          debug('cache confidence assessment', {\n            confidence: confidence.toFixed(3),\n            level,\n            actions,\n            verificationCount: confidenceState.verificationCount,\n          });\n\n          if (actions.skipCache) {\n            debug('cache confidence too low, skipping cache entirely', {\n              confidence,\n              level,\n            });\n            isCacheHit = false;\n          }\n\n          try {\n            if (isCacheHit && actions.coordCheck && cachedCenter) {\n              const offset = Math.sqrt(\n                (elementFromCache!.center[0] - cachedCenter[0]) ** 2 +\n                  (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n              );\n\n              debug('cache coord offset check', {\n                cachedCenter,\n                currentCenter: elementFromCache!.center,\n                offset: Math.round(offset),\n                threshold: coordOffsetThreshold,\n              });\n\n              if (offset > coordOffsetThreshold) {\n                debug(\n                  'cache coord offset exceeded threshold, fallback to AI locate',\n                  { offset, threshold: coordOffsetThreshold },\n                );\n                isCacheHit = false;\n              }\n            }\n\n            if (isCacheHit && actions.visualVerify) {\n              const verification = await this.service.verifyCachedElement(\n                elementFromCache!.center,\n                cachePrompt,\n                modelConfigForDefaultIntent,\n                uiContext,\n              );\n              if (!verification.pass) {\n                debug(\n                  'cache hit but visual verification failed, fallback to AI locate',\n                  {\n                    reason: verification.reason,\n                    description: verification.description,\n                    prompt: cachePrompt,\n                  },\n                );\n                isCacheHit = false;\n              } else {\n                debug('cache hit and visual verification passed', {\n                  description: verification.description,\n                });\n              }\n            }\n          } catch (verifyError) {\n            debug(\n              'cache verification error, fallback to AI locate',\n              verifyError,\n            );\n            isCacheHit = false;\n          }\n\n          if (isCacheHit) {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              true,\n            );\n            cacheFeature.confidenceState = updatedState;\n\n            const progressiveRecord = cacheFeature.progressiveRecord as\n              | ProgressiveLocateRecord\n              | undefined;\n            if (progressiveRecord) {\n              const updated = updateProgressiveConvergence(\n                progressiveRecord,\n                elementFromCache!.center,\n                updatedState.confidenceScore,\n              );\n              cacheFeature.progressiveRecord = updated;\n\n              if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n                debug(\n                  'using converged center instead of single-result center',\n                  {\n                    convergedCenter: updated.convergedCenter.map((v) =>\n                      v.toFixed(1),\n                    ),\n                    singleCenter: elementFromCache!.center,\n                    convergenceRadius: updated.convergenceRadius.toFixed(1),\n                    sampleCount: updated.sampleCount,\n                  },\n                );\n                elementFromCache = {\n                  ...elementFromCache!,\n                  center: [\n                    Math.round(updated.convergedCenter[0]),\n                    Math.round(updated.convergedCenter[1]),\n                  ],\n                };\n              }\n            }\n          } else {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              false,\n            );\n            cacheFeature.confidenceState = updatedState;\n          }\n        }\n\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const semanticAnchor = cacheFeature?.semanticAnchor as\n            | SemanticAnchor\n            | undefined;\n\n          if (\n            semanticAnchor &&\n            process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !== 'false'\n          ) {\n            try {\n              const anchorResult = await this.service.locateBySemanticAnchor(\n                semanticAnchor,\n                modelConfigForDefaultIntent,\n                this.interface,\n                uiContext,\n              );\n              if (anchorResult) {\n                elementFromAiLocate = anchorResult;\n                debug(\n                  'semantic anchor locate succeeded, skipping full AI locate',\n                );\n              }\n            } catch (anchorError) {\n              debug('semantic anchor locate failed:', anchorError);\n            }\n          }\n\n          if (!elementFromAiLocate) {\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              locateResult = await this.service.locate(\n                param,\n                {\n                  context: uiContext,\n                },\n                modelConfigForDefaultIntent,\n                abortSignal,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n            } catch (error) {\n              if (error instanceof ServiceError) {\n                applyDump(error.dump);\n              }\n              throw error;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                feature.cachedCenter = pointForCache;\n                feature.confidenceState = createInitialConfidenceState();\n\n                const existingProgressiveRecord = (\n                  cacheEntry as ElementCacheFeature\n                )?.progressiveRecord as\n                  | ProgressiveLocateRecord\n                  | undefined;\n                if (existingProgressiveRecord) {\n                  feature.progressiveRecord = updateProgressiveConvergence(\n                    existingProgressiveRecord,\n                    pointForCache,\n                    1.0,\n                  );\n                } else {\n                  feature.progressiveRecord =\n                    createInitialProgressiveRecord(pointForCache);\n                }\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n\n                const enableSemanticAnchor =\n                  process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !==\n                  'false';\n                if (enableSemanticAnchor) {\n                  try {\n                    const anchor = await this.service.generateSemanticAnchor(\n                      pointForCache,\n                      modelConfigForDefaultIntent,\n                      uiContext,\n                    );\n                    if (anchor) {\n                      feature.semanticAnchor = anchor;\n                      debug(\n                        'semantic anchor generated for prompt: %s',\n                        cachePrompt,\n                      );\n                    }\n                  } catch (anchorError) {\n                    debug('generateSemanticAnchor failed:', anchorError);\n                  }\n                }\n\n                currentCacheEntry = feature;\n                await this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","hasNonEmptyCache","cache","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","cacheFeature","cachedCenter","coordOffsetThreshold","Number","process","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","existingProgressiveRecord","createInitialProgressiveRecord","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC+CA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPP,OAAO,IAAI,CAACO,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,AAAAA,IAAAA,yBAAAA,2BAAAA,AAAAA,EAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnD/B,MACE,uCACA,CAAC,YAAY,EAAEuB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,EAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3EvB,MAAM,CAAC,OAAO,EAAE+B,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCtC,MACE,oBACAuB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,IAAAA,sBAAAA,MAAAA,AAAAA,EACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC1C,MACE,CAAC,8DAA8D,EAAEyB,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDL,MACE,CAAC,2DAA2D,EAAEyB,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,IAAAA,kCAAAA,KAAAA,AAAAA,EAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,AAAAA,IAAAA,yBAAAA,gBAAAA,AAAAA,EAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3BvC,MAAM,kBAAkByB,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAC3BvC,MAAM,iBAAiByB,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,AAAAA,IAAAA,kCAAAA,KAAAA,AAAAA,EAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCrD,MACE,CAAC,6DAA6D,EAAEyB,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDL,MACE,CAAC,0DAA0D,EAAEyB,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,IAAAA,sBAAAA,MAAAA,AAAAA,EACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,EAAwB5B,SAC5C4D,AAAAA,IAAAA,oCAAAA,oBAAAA,AAAAA,EAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EAEEC,AAAAA,IAAAA,oCAAAA,oCAAAA,AAAAA,EACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,AAAAA,IAAAA,oCAAAA,qBAAAA,AAAAA,EACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,AAAAA,IAAAA,oCAAAA,mCAAAA,AAAAA,EACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,MAAMtC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI0C;gBAEJ,IAAID,YAAY;oBACd,MAAME,eAAeP;oBACrB,MAAMQ,eAAeD,cAAc;oBAInC,MAAME,uBACJC,OAAO,QAAQ,CACbC,QAAQ,GAAG,CAACC,0BAAAA,qCAAqCA,CAAC,IAAI,MACtD,OACG;oBAEP,MAAMC,kBAAmBN,cAAc,mBACrCO,AAAAA,IAAAA,6CAAAA,4BAAAA,AAAAA;oBACF,MAAMC,aAAaC,AAAAA,IAAAA,6CAAAA,mBAAAA,AAAAA,EAAoBH;oBACvC,MAAMI,QAAQC,AAAAA,IAAAA,6CAAAA,0BAAAA,AAAAA,EAA2BH;oBACzC,MAAMI,UAAUC,AAAAA,IAAAA,6CAAAA,sBAAAA,AAAAA,EAAuBH;oBAEvC3F,MAAM,+BAA+B;wBACnC,YAAYyF,WAAW,OAAO,CAAC;wBAC/BE;wBACAE;wBACA,mBAAmBN,gBAAgB,iBAAiB;oBACtD;oBAEA,IAAIM,QAAQ,SAAS,EAAE;wBACrB7F,MAAM,qDAAqD;4BACzDyF;4BACAE;wBACF;wBACAZ,aAAa;oBACf;oBAEA,IAAI;wBACF,IAAIA,cAAcc,QAAQ,UAAU,IAAIX,cAAc;4BACpD,MAAMa,SAASC,KAAK,IAAI,CACrBnB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGK,YAAY,CAAC,EAAC,KAAM,IAChDL,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGK,YAAY,CAAC,EAAC,KAAM;4BAGvDlF,MAAM,4BAA4B;gCAChCkF;gCACA,eAAeL,iBAAkB,MAAM;gCACvC,QAAQmB,KAAK,KAAK,CAACD;gCACnB,WAAWZ;4BACb;4BAEA,IAAIY,SAASZ,sBAAsB;gCACjCnF,MACE,gEACA;oCAAE+F;oCAAQ,WAAWZ;gCAAqB;gCAE5CJ,aAAa;4BACf;wBACF;wBAEA,IAAIA,cAAcc,QAAQ,YAAY,EAAE;4BACtC,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDpB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;4BAEF,IAAKyD,aAAa,IAAI,EAWpBjG,MAAM,4CAA4C;gCAChD,aAAaiG,aAAa,WAAW;4BACvC;iCAbsB;gCACtBjG,MACE,mEACA;oCACE,QAAQiG,aAAa,MAAM;oCAC3B,aAAaA,aAAa,WAAW;oCACrC,QAAQzB;gCACV;gCAEFO,aAAa;4BACf;wBAKF;oBACF,EAAE,OAAOmB,aAAa;wBACpBlG,MACE,mDACAkG;wBAEFnB,aAAa;oBACf;oBAEA,IAAIA,YAAY;wBACd,MAAMoB,eAAeC,AAAAA,IAAAA,6CAAAA,wBAAAA,AAAAA,EACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;wBAE/B,MAAME,oBAAoBpB,aAAa,iBAAiB;wBAGxD,IAAIoB,mBAAmB;4BACrB,MAAMC,UAAUC,AAAAA,IAAAA,6CAAAA,4BAAAA,AAAAA,EACdF,mBACAxB,iBAAkB,MAAM,EACxBsB,aAAa,eAAe;4BAE9BlB,aAAa,iBAAiB,GAAGqB;4BAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;gCAC7DtG,MACE,0DACA;oCACE,iBAAiBsG,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;oCAEZ,cAAc3B,iBAAkB,MAAM;oCACtC,mBAAmByB,QAAQ,iBAAiB,CAAC,OAAO,CAAC;oCACrD,aAAaA,QAAQ,WAAW;gCAClC;gCAEFzB,mBAAmB;oCACjB,GAAGA,gBAAgB;oCACnB,QAAQ;wCACNmB,KAAK,KAAK,CAACM,QAAQ,eAAe,CAAC,EAAE;wCACrCN,KAAK,KAAK,CAACM,QAAQ,eAAe,CAAC,EAAE;qCACtC;gCACH;4BACF;wBACF;oBACF,OAAO;wBACL,MAAMH,eAAeC,AAAAA,IAAAA,6CAAAA,wBAAAA,AAAAA,EACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;oBACjC;gBACF;gBAEA,IAAI,CAAC5B,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMe,eAAeP;oBACrB,MAAM+B,iBAAiBxB,cAAc;oBAIrC,IACEwB,kBACApB,AAAuD,YAAvDA,QAAQ,GAAG,CAACqB,0BAAAA,qCAAqCA,CAAC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACA9F,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAImE,cAAc;4BAChB3B,sBAAsB2B;4BACtB3G,MACE;wBAEJ;oBACF,EAAE,OAAO4G,aAAa;wBACpB5G,MAAM,kCAAkC4G;oBAC1C;oBAGF,IAAI,CAAC5B,qBACH,IAAI;wBACFvC,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BmB,sBAAsBnB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiB2D,kCAAAA,YAAYA,EAC/B/C,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,IAAAA,wCAAAA,kBAAAA,AAAAA,EAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAMuE,UACJ9C,mBACAI,oBACAS,oBACAG;gBAGF,MAAM+B,2BAA2B7G,iBAC/BuE,mBAAmB,cAAc;gBAGnC,IAAIuC;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACd,CAAC/B,cACA,EAACb,aAAa,CAAC6C,wBAAuB,KACvC1G,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAI4G,gBAAkCH,QAAQ,MAAM;oBACpD,IAAI/D,AAA6B,MAA7BA,0BAAgC;wBAClCkE,gBAAgB;4BACdjB,KAAK,KAAK,CAACc,QAAQ,MAAM,CAAC,EAAE,GAAG/D;4BAC/BiD,KAAK,KAAK,CAACc,QAAQ,MAAM,CAAC,EAAE,GAAG/D;yBAChC;wBACD/C,MACE,8DACA8G,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAO5G,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIT,iBAAiBgH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG1B,AAAAA,IAAAA,6CAAAA,4BAAAA,AAAAA;wBAE1B,MAAM2B,4BACJzC,YACC;wBAGH,IAAIyC,2BACFD,QAAQ,iBAAiB,GAAGX,AAAAA,IAAAA,6CAAAA,4BAAAA,AAAAA,EAC1BY,2BACAF,eACA;6BAGFC,QAAQ,iBAAiB,GACvBE,AAAAA,IAAAA,6CAAAA,8BAAAA,AAAAA,EAA+BH;wBAEnCjH,MACE,uCACAwE,aACA0C;wBAGF,MAAMG,uBACJhC,AACA,YADAA,QAAQ,GAAG,CAACqB,0BAAAA,qCAAqCA,CAAC;wBAEpD,IAAIW,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDL,eACAtG,6BACA6B;4BAEF,IAAI8E,QAAQ;gCACVJ,QAAQ,cAAc,GAAGI;gCACzBtH,MACE,4CACAwE;4BAEJ;wBACF,EAAE,OAAOoC,aAAa;4BACpB5G,MAAM,kCAAkC4G;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQ1C;4BACR,OAAO0C;wBACT,GACAzC;oBAEJ,OACEzE,MACE,yDACAwE;gBAGN,EAAE,OAAOtB,OAAO;oBACdlD,MAAM,mCAAmCkD;gBAC3C;qBAEAlD,MAAM;gBAIV,IAAI,CAAC8G,SAAS;oBACZ,IAAIlD,YACF,MAAM,IAAIiD,kCAAAA,YAAYA,CACpB,CAAC,oBAAoB,EAAExG,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIkH;gBAEJ,IAAIrD,WACFqD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMlH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACTgD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOlH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACTwC,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP7C;wBACA,aAAasC;oBACf;gBACF;gBAGFzD,WAAWuD;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAKtE,UAAU,aAAa;wBAC9B;oBACF;oBACA+E;gBACF;YACF;QACF;QAEA,OAAO5D;IACT;IA5vBA,YAAY,EACV6D,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTlG,WAAW,EACXmG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGlG;QACnB,IAAI,CAAC,eAAe,GAAGmG;IACzB;AAivBF"}

package/dist/lib/agent/utils.js

@@ -201,7 +201,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.6.0";
+const getMidsceneVersion = ()=>"1.6.4";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/ai-model/prompt/semantic-anchor-search.js

@@ -41,19 +41,24 @@ TASK: Find the element matching this description in the image.
 
 RESPONSE FORMAT (JSON):
 {
-  "found": true/false,
-  "center": [x, y] or null,
-  "rect": {"left": 0, "top": 0, "width": 0, "height": 0} or null,
-  "confidence": 0.0-1.0,
+  "found": true,
+  "center": [123, 456],
+  "rect": {"left": 100, "top": 400, "width": 50, "height": 100},
+  "confidence": 0.9,
   "reason": "brief explanation"
 }
 
+IMPORTANT:
+- "center" must be an array with exactly 2 numbers: [x_coordinate, y_coordinate]
+- Example: if the element center is at pixel x=320 and y=150, write "center": [320, 150]
+- Do NOT use variable names like x or y, use actual pixel numbers
+
 RULES:
 1. Look for an element matching the visualFingerprint description
 2. Verify it's in the expected position relative to the contextDescription
 3. Return center coordinates and bounding rect if found
 4. Set confidence based on how well the match is
-5. If not found or confidence < 0.5, set found=false
+5. If not found or confidence < 0.5, set found=false and center=null
 6. Write reason in ${preferredLanguage}
 `;
 };

package/dist/lib/ai-model/prompt/semantic-anchor-search.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/semantic-anchor-search.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/semantic-anchor-search.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport const semanticAnchorSearchInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are searching for an element based on its semantic anchor description.\nThe image shows a CROPPED AREA of the page around a known landmark element.\n\nTARGET ELEMENT DESCRIPTION:\n- Visual: {{visualFingerprint}}\n- Context: {{contextDescription}}\n\nTASK: Find the element matching this description in the image.\n\nRESPONSE FORMAT (JSON):\n{\n  \"found\": true/false,\n  \"center\": [x, y] or null,\n  \"rect\": {\"left\": 0, \"top\": 0, \"width\": 0, \"height\": 0} or null,\n  \"confidence\": 0.0-1.0,\n  \"reason\": \"brief explanation\"\n}\n\nRULES:\n1. Look for an element matching the visualFingerprint description\n2. Verify it's in the expected position relative to the contextDescription\n3. Return center coordinates and bounding rect if found\n4. Set confidence based on how well the match is\n5. If not found or confidence < 0.5, set found=false\n6. Write reason in ${preferredLanguage}\n`;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","semanticAnchorSearchInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACJO,MAAMI,kCAAkC;IAC7C,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;mBAyBS,EAAED,kBAAkB;AACvC,CAAC;AACD"}
+{"version":3,"file":"ai-model/prompt/semantic-anchor-search.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/semantic-anchor-search.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport const semanticAnchorSearchInstruction = () => {\n  const preferredLanguage = getPreferredLanguage();\n\n  return `\nYou are searching for an element based on its semantic anchor description.\nThe image shows a CROPPED AREA of the page around a known landmark element.\n\nTARGET ELEMENT DESCRIPTION:\n- Visual: {{visualFingerprint}}\n- Context: {{contextDescription}}\n\nTASK: Find the element matching this description in the image.\n\nRESPONSE FORMAT (JSON):\n{\n  \"found\": true,\n  \"center\": [123, 456],\n  \"rect\": {\"left\": 100, \"top\": 400, \"width\": 50, \"height\": 100},\n  \"confidence\": 0.9,\n  \"reason\": \"brief explanation\"\n}\n\nIMPORTANT:\n- \"center\" must be an array with exactly 2 numbers: [x_coordinate, y_coordinate]\n- Example: if the element center is at pixel x=320 and y=150, write \"center\": [320, 150]\n- Do NOT use variable names like x or y, use actual pixel numbers\n\nRULES:\n1. Look for an element matching the visualFingerprint description\n2. Verify it's in the expected position relative to the contextDescription\n3. Return center coordinates and bounding rect if found\n4. Set confidence based on how well the match is\n5. If not found or confidence < 0.5, set found=false and center=null\n6. Write reason in ${preferredLanguage}\n`;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","semanticAnchorSearchInstruction","preferredLanguage","getPreferredLanguage"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;ACJO,MAAMI,kCAAkC;IAC7C,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAE1B,OAAO,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;mBA8BS,EAAED,kBAAkB;AACvC,CAAC;AACD"}

package/dist/lib/service/index.js

@@ -474,6 +474,10 @@ class Service {
                 debug('semantic anchor locate: not found or low confidence', content);
                 return null;
             }
+            if (!Array.isArray(content.center) || 2 !== content.center.length || 'number' != typeof content.center[0] || 'number' != typeof content.center[1] || Number.isNaN(content.center[0]) || Number.isNaN(content.center[1])) {
+                debug('semantic anchor locate: invalid center format', content.center, 'expected [number, number]');
+                return null;
+            }
             const adjustedCenter = [
                 content.center[0] + cropArea.left,
                 content.center[1] + cropArea.top

package/dist/lib/service/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  AIResponseParseError,\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { cacheVerifyInstruction } from '@/ai-model/prompt/cache-verify';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport { semanticAnchorGenerateInstruction } from '@/ai-model/prompt/semantic-anchor';\nimport { semanticAnchorSearchInstruction } from '@/ai-model/prompt/semantic-anchor-search';\nimport { type AIArgs, expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  CacheVerificationResult,\n  DetailedLocateParam,\n  Landmark,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    let searchAreaPrompt;\n    if (query.deepLocate) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { modelFamily } = modelConfig;\n\n    if (searchAreaPrompt && !modelFamily) {\n      console.warn(\n        'The \"deepLocate\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    if (searchAreaPrompt && isAutoGLM(modelFamily)) {\n      console.warn('The \"deepLocate\" feature is not supported with AutoGLM.');\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n        abortSignal,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage, reasoning_content } =\n      await AiLocateElement({\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n        modelConfig,\n        abortSignal,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      deepLocate: !!searchArea || !!searchAreaPrompt,\n      error: errorLog,\n      taskInfo,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new ServiceError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          description: elements[0]!.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelConfig,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          matchedElement: [],\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { modelFamily } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        modelFamily === 'qwen2.5-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n\n  async verifyCachedElement(\n    center: [number, number],\n    targetPrompt: TUserPrompt,\n    modelConfig: IModelConfig,\n    uiContext?: UIContext,\n  ): Promise<CacheVerificationResult> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for verifyCachedElement');\n\n    const defaultRectSize = 60;\n    const targetRect: Rect = {\n      left: Math.floor(center[0] - defaultRectSize / 2),\n      top: Math.floor(center[1] - defaultRectSize / 2),\n      width: defaultRectSize,\n      height: defaultRectSize,\n    };\n\n    const cropArea = expandSearchArea(targetRect, shotSize);\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [{ rect: targetRect }],\n      borderThickness: 3,\n    });\n\n    debug(\n      'verifyCachedElement: cropping to area',\n      cropArea,\n      'from center',\n      center,\n    );\n    const croppedResult = await cropByRect(\n      imagePayload,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    imagePayload = croppedResult.imageBase64;\n\n    const promptText =\n      typeof targetPrompt === 'string'\n        ? targetPrompt\n        : targetPrompt.prompt || '';\n\n    const systemPrompt = cacheVerifyInstruction().replace(\n      /{{targetPrompt}}/g,\n      promptText,\n    );\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'low',\n            },\n          },\n        ],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        match: boolean;\n        description?: string;\n        reason?: string;\n        error?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        match?: boolean;\n        description?: string;\n        reason?: string;\n        error?: string;\n      };\n\n      if (content.error) {\n        debug('verifyCachedElement AI returned error:', content.error);\n        return { pass: false, reason: content.error };\n      }\n\n      const result: CacheVerificationResult = {\n        pass: !!content.match,\n        description: content.description,\n        reason: content.reason,\n      };\n\n      debug('verifyCachedElement result:', {\n        pass: result.pass,\n        description: result.description,\n        reason: result.reason,\n        targetPrompt: promptText,\n      });\n\n      return result;\n    } catch (error) {\n      debug('verifyCachedElement failed:', error);\n      return { pass: false, reason: String(error) };\n    }\n  }\n\n  async generateSemanticAnchor(\n    center: [number, number],\n    modelConfig: IModelConfig,\n    uiContext?: UIContext,\n  ): Promise<import('@/types').SemanticAnchor | undefined> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    if (!screenshotBase64) return undefined;\n\n    const defaultRectSize = 60;\n    const targetRect: Rect = {\n      left: Math.floor(center[0] - defaultRectSize / 2),\n      top: Math.floor(center[1] - defaultRectSize / 2),\n      width: defaultRectSize,\n      height: defaultRectSize,\n    };\n\n    const cropArea = expandSearchArea(targetRect, shotSize);\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [{ rect: targetRect }],\n      borderThickness: 3,\n    });\n\n    const croppedResult = await cropByRect(\n      imagePayload,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    imagePayload = croppedResult.imageBase64;\n\n    const systemPrompt = semanticAnchorGenerateInstruction();\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: { url: imagePayload, detail: 'low' },\n          },\n        ],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        visualFingerprint?: string;\n        contextDescription?: string;\n        nearbyLandmarks?: Array<{ description?: string; xpath?: string }>;\n        error?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        visualFingerprint?: string;\n        contextDescription?: string;\n        nearbyLandmarks?: Array<{ description?: string; xpath?: string }>;\n        error?: string;\n      };\n\n      if (content.error || !content.visualFingerprint) {\n        debug('generateSemanticAnchor AI error:', content.error);\n        return undefined;\n      }\n\n      const landmarks: Landmark[] = (content.nearbyLandmarks || [])\n        .filter((l) => l.description && l.xpath)\n        .map((l) => ({\n          description: l.description!,\n          xpath: l.xpath!,\n        }));\n\n      const anchor: import('@/types').SemanticAnchor = {\n        visualFingerprint: content.visualFingerprint,\n        contextDescription: content.contextDescription || '',\n        nearbyLandmarks: landmarks,\n      };\n\n      debug('generateSemanticAnchor result:', anchor);\n      return anchor;\n    } catch (error) {\n      debug('generateSemanticAnchor failed:', error);\n      return undefined;\n    }\n  }\n\n  async locateBySemanticAnchor(\n    anchor: import('@/types').SemanticAnchor,\n    modelConfig: IModelConfig,\n    interfaceInstance: import('@/device').AbstractInterface,\n    uiContext?: UIContext,\n  ): Promise<LocateResultElement | null> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    if (!screenshotBase64) return null;\n\n    let landmarkRect: Rect | undefined;\n\n    for (const landmark of anchor.nearbyLandmarks) {\n      if (interfaceInstance.rectMatchesCacheFeature) {\n        try {\n          const rect = await interfaceInstance.rectMatchesCacheFeature({\n            xpaths: [landmark.xpath],\n          });\n          if (rect) {\n            landmarkRect = rect;\n            debug('semantic anchor: landmark XPath hit:', landmark.description);\n            break;\n          }\n        } catch {\n          debug('semantic anchor: landmark XPath miss:', landmark.description);\n        }\n      }\n    }\n\n    let cropArea: Rect;\n    if (landmarkRect) {\n      const expandSize = 300;\n      cropArea = {\n        left: Math.max(0, landmarkRect.left - expandSize),\n        top: Math.max(0, landmarkRect.top - expandSize),\n        width: landmarkRect.width + expandSize * 2,\n        height: landmarkRect.height + expandSize * 2,\n      };\n    } else {\n      cropArea = {\n        left: 0,\n        top: 0,\n        width: shotSize.width,\n        height: shotSize.height,\n      };\n    }\n\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    const imagePayload: string = croppedResult.imageBase64;\n\n    const systemPrompt = semanticAnchorSearchInstruction()\n      .replace('{{visualFingerprint}}', anchor.visualFingerprint)\n      .replace('{{contextDescription}}', anchor.contextDescription);\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url' as const,\n            image_url: { url: imagePayload, detail: 'low' },\n          },\n        ] as ChatCompletionUserMessageParam['content'],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        found?: boolean;\n        center?: [number, number] | null;\n        rect?: Rect | null;\n        confidence?: number;\n        reason?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        found?: boolean;\n        center?: [number, number] | null;\n        rect?: Rect | null;\n        confidence?: number;\n        reason?: string;\n      };\n\n      if (\n        !content.found ||\n        !content.center ||\n        (content.confidence ?? 0) < 0.5\n      ) {\n        debug('semantic anchor locate: not found or low confidence', content);\n        return null;\n      }\n\n      const adjustedCenter: [number, number] = [\n        content.center[0] + cropArea.left,\n        content.center[1] + cropArea.top,\n      ];\n\n      let adjustedRect: Rect | undefined;\n      if (content.rect) {\n        adjustedRect = {\n          left: content.rect.left + cropArea.left,\n          top: content.rect.top + cropArea.top,\n          width: content.rect.width,\n          height: content.rect.height,\n        };\n      }\n\n      debug(\n        'semantic anchor locate: found at',\n        adjustedCenter,\n        'confidence:',\n        content.confidence,\n      );\n\n      return {\n        center: adjustedCenter,\n        rect: adjustedRect || {\n          left: adjustedCenter[0] - 30,\n          top: adjustedCenter[1] - 15,\n          width: 60,\n          height: 30,\n        },\n        description: `semantic anchor: ${anchor.contextDescription}`,\n      };\n    } catch (error) {\n      debug('locateBySemanticAnchor failed:', error);\n      return null;\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelConfig","abortSignal","queryPrompt","assert","searchAreaPrompt","modelFamily","console","undefined","isAutoGLM","context","searchArea","searchAreaRawResponse","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","center","targetPrompt","uiContext","cropArea","promptText","cacheVerifyInstruction","String","semanticAnchorGenerateInstruction","landmarks","l","anchor","interfaceInstance","landmarkRect","landmark","expandSize","semanticAnchorSearchInstruction","adjustedCenter","adjustedRect","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC4CA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACzBC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,IAAIM;QACJ,IAAIN,MAAM,UAAU,EAClBM,mBAAmBN,MAAM,MAAM;QAGjC,MAAM,EAAEO,WAAW,EAAE,GAAGL;QAExB,IAAII,oBAAoB,CAACC,aAAa;YACpCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,IAAIH,oBAAoBI,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUH,cAAc;YAC9CC,QAAQ,IAAI,CAAC;YACbF,mBAAmBG;QACrB;QAEA,MAAME,UAAUV,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIW;QACJ,IAAIC;QAEJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBACzCJ;gBACA,oBAAoBL;gBACpBJ;gBACAC;YACF;YACAE,IAAAA,sBAAAA,MAAAA,AAAAA,EACES,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJD,wBAAwBC,mBAAmB,WAAW;YACpCA,mBAAmB,KAAK;YAC1CF,aAAaE,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgB;YACpBZ;YACA,0BAA0BP;YAC1B,cAAcU;YACdZ;YACAC;QACF;QAEF,MAAMqB,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAT;YACAC;YACAS;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,4BAA4B,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASxB;YACX;YACA,gBAAgB,EAAE;YAClB,aAAae;YACb,MAAM;YACN,YAAY,CAAC,CAACP,cAAc,CAAC,CAACN;YAC9B,OAAOqB;YACPF;QACF;QAEA,MAAMI,WAAWX,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMY,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,kCAAAA,YAAYA,CACpB,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAV;YACAW;QACF;QAGF,OAAO;YACL,SAAS;YACTX;YACAW;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/B/B,WAAyB,EACzBD,GAA0B,EAC1BiC,eAAwB,EACxBC,gBAAoC,EACpCxB,OAAmB,EACe;QAClCN,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,SAAS;QAChBN,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAO4B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMjB,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMc,SAAS,MAAMC,AAAAA,IAAAA,yBAAAA,oBAAAA,AAAAA,EAAwB;gBAC3C1B;gBACA,WAAWsB;gBACXE;gBACA,eAAelC;gBACfC;gBACAgC;YACF;YACAhB,cAAckB,OAAO,WAAW;YAChChB,cAAcgB,OAAO,WAAW;YAChCf,QAAQe,OAAO,KAAK;YACpBd,oBAAoBc,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,yBAAAA,oBAAoBA,EAAE;gBAEzC,MAAMf,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAac,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMR,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEE;oBAAW;oBACxB,gBAAgB,EAAE;oBAClB,MAAM;oBACNR;oBACA,OAAOa,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIN,kCAAAA,YAAYA,CAACM,MAAM,OAAO,EAAER;YACxC;YACA,MAAMQ;QACR;QAEA,MAAMd,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,qBAAqB,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNR;YACA,OAAOE;QACT;QAEA,MAAM,EAAEa,IAAI,EAAEC,OAAO,EAAE,GAAGvB,eAAe,CAAC;QAE1C,MAAMY,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXY;QACF;QAEA,IAAIb,YAAY,CAACa,MACf,MAAM,IAAIR,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLU;YACAC;YACApB;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJY,MAA+B,EAC/BxC,WAAyB,EACzBD,GAEC,EACwD;QACzDI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqC,QAAQ;QACf,MAAM/B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEgC,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClDN,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuC,kBAAkB;QAEzB,MAAM,EAAErC,WAAW,EAAE,GAAGL;QACxB,MAAM2C,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAI/C,KAAK,YAAY;YACnB,MAAMW,aAAayC,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;YAMhD9C,MAAM,oCAAoCe;YAC1C,MAAM0C,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BJ,cACAvC,YACAL,AAAgB,iBAAhBA;YAEF4C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChBF,MACAtD;QAGF,MAAM,EAAEyD,OAAO,EAAE,GAAGF;QACpBpD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACsD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DtD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAEA,MAAM,oBACJC,MAAwB,EACxBC,YAAyB,EACzB3D,WAAyB,EACzB4D,SAAqB,EACa;QAClC,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClDN,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuC,kBAAkB;QAEzB,MAAMG,kBAAkB;QACxB,MAAMC,aAAmB;YACvB,MAAME,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV;QAEA,MAAMgB,WAAWV,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;QAE9C,IAAIQ,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBAAC;oBAAE,MAAMK;gBAAW;aAAE;YAC5C,iBAAiB;QACnB;QAEAnD,MACE,yCACAkE,UACA,eACAH;QAEF,MAAMN,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BJ,cACAY,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzBiD,eAAeG,cAAc,WAAW;QAExC,MAAMU,aACJ,AAAwB,YAAxB,OAAOH,eACHA,eACAA,aAAa,MAAM,IAAI;QAE7B,MAAMhB,eAAeoB,AAAAA,IAAAA,gCAAAA,sBAAAA,AAAAA,IAAyB,OAAO,CACnD,qBACAD;QAGF,MAAMR,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAKfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAO3B,IAAIE,QAAQ,KAAK,EAAE;gBACjB9D,MAAM,0CAA0C8D,QAAQ,KAAK;gBAC7D,OAAO;oBAAE,MAAM;oBAAO,QAAQA,QAAQ,KAAK;gBAAC;YAC9C;YAEA,MAAMvB,SAAkC;gBACtC,MAAM,CAAC,CAACuB,QAAQ,KAAK;gBACrB,aAAaA,QAAQ,WAAW;gBAChC,QAAQA,QAAQ,MAAM;YACxB;YAEA9D,MAAM,+BAA+B;gBACnC,MAAMuC,OAAO,IAAI;gBACjB,aAAaA,OAAO,WAAW;gBAC/B,QAAQA,OAAO,MAAM;gBACrB,cAAc4B;YAChB;YAEA,OAAO5B;QACT,EAAE,OAAOE,OAAO;YACdzC,MAAM,+BAA+ByC;YACrC,OAAO;gBAAE,MAAM;gBAAO,QAAQ4B,OAAO5B;YAAO;QAC9C;IACF;IAEA,MAAM,uBACJsB,MAAwB,EACxB1D,WAAyB,EACzB4D,SAAqB,EACkC;QACvD,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClD,IAAI,CAACiC,kBAAkB;QAEvB,MAAMG,kBAAkB;QACxB,MAAMC,aAAmB;YACvB,MAAME,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV;QAEA,MAAMgB,WAAWV,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;QAE9C,IAAIQ,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBAAC;oBAAE,MAAMK;gBAAW;aAAE;YAC5C,iBAAiB;QACnB;QAEA,MAAMM,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BJ,cACAY,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzBiD,eAAeG,cAAc,WAAW;QAExC,MAAMT,eAAesB,AAAAA,IAAAA,mCAAAA,iCAAAA,AAAAA;QAErB,MAAMX,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BAAE,KAAKM;4BAAc,QAAQ;wBAAM;oBAChD;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAKfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAO3B,IAAIE,QAAQ,KAAK,IAAI,CAACA,QAAQ,iBAAiB,EAAE,YAC/C9D,MAAM,oCAAoC8D,QAAQ,KAAK;YAIzD,MAAMS,YAAyBT,AAAAA,CAAAA,QAAQ,eAAe,IAAI,EAAC,EACxD,MAAM,CAAC,CAACU,IAAMA,EAAE,WAAW,IAAIA,EAAE,KAAK,EACtC,GAAG,CAAC,CAACA,IAAO;oBACX,aAAaA,EAAE,WAAW;oBAC1B,OAAOA,EAAE,KAAK;gBAChB;YAEF,MAAMC,SAA2C;gBAC/C,mBAAmBX,QAAQ,iBAAiB;gBAC5C,oBAAoBA,QAAQ,kBAAkB,IAAI;gBAClD,iBAAiBS;YACnB;YAEAvE,MAAM,kCAAkCyE;YACxC,OAAOA;QACT,EAAE,OAAOhC,OAAO;YACdzC,MAAM,kCAAkCyC;YACxC;QACF;IACF;IAEA,MAAM,uBACJgC,MAAwC,EACxCpE,WAAyB,EACzBqE,iBAAuD,EACvDT,SAAqB,EACgB;QACrC,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClD,IAAI,CAACiC,kBAAkB,OAAO;QAE9B,IAAI4B;QAEJ,KAAK,MAAMC,YAAYH,OAAO,eAAe,CAC3C,IAAIC,kBAAkB,uBAAuB,EAC3C,IAAI;YACF,MAAMpD,OAAO,MAAMoD,kBAAkB,uBAAuB,CAAC;gBAC3D,QAAQ;oBAACE,SAAS,KAAK;iBAAC;YAC1B;YACA,IAAItD,MAAM;gBACRqD,eAAerD;gBACftB,MAAM,wCAAwC4E,SAAS,WAAW;gBAClE;YACF;QACF,EAAE,OAAM;YACN5E,MAAM,yCAAyC4E,SAAS,WAAW;QACrE;QAIJ,IAAIV;QACJ,IAAIS,cAAc;YAChB,MAAME,aAAa;YACnBX,WAAW;gBACT,MAAMb,KAAK,GAAG,CAAC,GAAGsB,aAAa,IAAI,GAAGE;gBACtC,KAAKxB,KAAK,GAAG,CAAC,GAAGsB,aAAa,GAAG,GAAGE;gBACpC,OAAOF,aAAa,KAAK,GAAGE,AAAa,IAAbA;gBAC5B,QAAQF,aAAa,MAAM,GAAGE,AAAa,IAAbA;YAChC;QACF,OACEX,WAAW;YACT,MAAM;YACN,KAAK;YACL,OAAOpB,SAAS,KAAK;YACrB,QAAQA,SAAS,MAAM;QACzB;QAGF,MAAMW,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BX,kBACAmB,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzB,MAAMiD,eAAuBG,cAAc,WAAW;QAEtD,MAAMT,eAAe8B,AAAAA,IAAAA,0CAAAA,+BAAAA,AAAAA,IAClB,OAAO,CAAC,yBAAyBL,OAAO,iBAAiB,EACzD,OAAO,CAAC,0BAA0BA,OAAO,kBAAkB;QAE9D,MAAMd,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BAAE,KAAKM;4BAAc,QAAQ;wBAAM;oBAChD;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAMfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAQ3B,IACE,CAACE,QAAQ,KAAK,IACd,CAACA,QAAQ,MAAM,IACdA,AAAAA,CAAAA,QAAQ,UAAU,IAAI,KAAK,KAC5B;gBACA9D,MAAM,uDAAuD8D;gBAC7D,OAAO;YACT;YAEA,MAAMiB,iBAAmC;gBACvCjB,QAAQ,MAAM,CAAC,EAAE,GAAGI,SAAS,IAAI;gBACjCJ,QAAQ,MAAM,CAAC,EAAE,GAAGI,SAAS,GAAG;aACjC;YAED,IAAIc;YACJ,IAAIlB,QAAQ,IAAI,EACdkB,eAAe;gBACb,MAAMlB,QAAQ,IAAI,CAAC,IAAI,GAAGI,SAAS,IAAI;gBACvC,KAAKJ,QAAQ,IAAI,CAAC,GAAG,GAAGI,SAAS,GAAG;gBACpC,OAAOJ,QAAQ,IAAI,CAAC,KAAK;gBACzB,QAAQA,QAAQ,IAAI,CAAC,MAAM;YAC7B;YAGF9D,MACE,oCACA+E,gBACA,eACAjB,QAAQ,UAAU;YAGpB,OAAO;gBACL,QAAQiB;gBACR,MAAMC,gBAAgB;oBACpB,MAAMD,cAAc,CAAC,EAAE,GAAG;oBAC1B,KAAKA,cAAc,CAAC,EAAE,GAAG;oBACzB,OAAO;oBACP,QAAQ;gBACV;gBACA,aAAa,CAAC,iBAAiB,EAAEN,OAAO,kBAAkB,EAAE;YAC9D;QACF,EAAE,OAAOhC,OAAO;YACdzC,MAAM,kCAAkCyC;YACxC,OAAO;QACT;IACF;IAvpBA,YACE3B,OAA2D,EAC3DV,GAAoB,CACpB;QAPF;QAEA;QAMEI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMmE,QAAQ,OAAO,CAACnE;QAGlD,IAAI,AAAyB,WAAlBV,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA0oBF"}
+{"version":3,"file":"service/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/service/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  AIResponseParseError,\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { cacheVerifyInstruction } from '@/ai-model/prompt/cache-verify';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport { semanticAnchorGenerateInstruction } from '@/ai-model/prompt/semantic-anchor';\nimport { semanticAnchorSearchInstruction } from '@/ai-model/prompt/semantic-anchor-search';\nimport { type AIArgs, expandSearchArea } from '@/common';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  CacheVerificationResult,\n  DetailedLocateParam,\n  Landmark,\n  LocateResultElement,\n  LocateResultWithDump,\n  PartialServiceDumpFromSDK,\n  Rect,\n  ServiceExtractOption,\n  ServiceExtractParam,\n  ServiceExtractResult,\n  ServiceTaskInfo,\n  UIContext,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport { createServiceDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface ServiceOptions {\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n}\n\nconst debug = getDebug('ai:service');\nexport default class Service {\n  contextRetrieverFn: () => Promise<UIContext> | UIContext;\n\n  taskInfo?: Omit<ServiceTaskInfo, 'durationMs'>;\n\n  constructor(\n    context: UIContext | (() => Promise<UIContext> | UIContext),\n    opt?: ServiceOptions,\n  ) {\n    assert(context, 'context is required for Service');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n    abortSignal?: AbortSignal,\n  ): Promise<LocateResultWithDump> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    let searchAreaPrompt;\n    if (query.deepLocate) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { modelFamily } = modelConfig;\n\n    if (searchAreaPrompt && !modelFamily) {\n      console.warn(\n        'The \"deepLocate\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    if (searchAreaPrompt && isAutoGLM(modelFamily)) {\n      console.warn('The \"deepLocate\" feature is not supported with AutoGLM.');\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn());\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n        abortSignal,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, rawResponse, usage, reasoning_content } =\n      await AiLocateElement({\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n        modelConfig,\n        abortSignal,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `failed to locate element: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      deepLocate: !!searchArea || !!searchAreaPrompt,\n      error: errorLog,\n      taskInfo,\n    };\n\n    const elements = parseResult.elements || [];\n\n    const dump = createServiceDump({\n      ...dumpData,\n      matchedElement: elements,\n    });\n\n    if (errorLog) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    if (elements.length > 1) {\n      throw new ServiceError(\n        `locate: multiple elements found, length = ${elements.length}`,\n        dump,\n      );\n    }\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          description: elements[0]!.description,\n        },\n        rect,\n        dump,\n      };\n    }\n\n    return {\n      element: null,\n      rect,\n      dump,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    pageDescription?: string,\n    multimodalPrompt?: TMultimodalPrompt,\n    context?: UIContext,\n  ): Promise<ServiceExtractResult<T>> {\n    assert(context, 'context is required for extract');\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n\n    const startTime = Date.now();\n\n    let parseResult: Awaited<\n      ReturnType<typeof AiExtractElementInfo<T>>\n    >['parseResult'];\n    let rawResponse: string;\n    let usage: Awaited<ReturnType<typeof AiExtractElementInfo<T>>>['usage'];\n    let reasoning_content: string | undefined;\n\n    try {\n      const result = await AiExtractElementInfo<T>({\n        context,\n        dataQuery: dataDemand,\n        multimodalPrompt,\n        extractOption: opt,\n        modelConfig,\n        pageDescription,\n      });\n      parseResult = result.parseResult;\n      rawResponse = result.rawResponse;\n      usage = result.usage;\n      reasoning_content = result.reasoning_content;\n    } catch (error) {\n      if (error instanceof AIResponseParseError) {\n        // Create dump with usage and rawResponse from the error\n        const timeCost = Date.now() - startTime;\n        const taskInfo: ServiceTaskInfo = {\n          ...(this.taskInfo ? this.taskInfo : {}),\n          durationMs: timeCost,\n          rawResponse: error.rawResponse,\n          usage: error.usage,\n        };\n        const dump = createServiceDump({\n          type: 'extract',\n          userQuery: { dataDemand },\n          matchedElement: [],\n          data: null,\n          taskInfo,\n          error: error.message,\n        });\n        throw new ServiceError(error.message, dump);\n      }\n      throw error;\n    }\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: ServiceTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse,\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      reasoning_content,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialServiceDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    const dump = createServiceDump({\n      ...dumpData,\n      data,\n    });\n\n    if (errorLog && !data) {\n      throw new ServiceError(errorLog, dump);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n      reasoning_content,\n      dump,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepLocate?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for service.describe');\n    const context = await this.contextRetrieverFn();\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for service.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { modelFamily } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepLocate) {\n      const searchArea = expandSearchArea(targetRect, shotSize);\n      // Always crop in describe mode. Unlike locate's deepLocate (where\n      // cropping too small loses context for finding elements), describe's\n      // deepLocate intentionally zooms in so the model produces a more\n      // precise description from a focused view. expandSearchArea already\n      // guarantees a minimum 400x400 area with surrounding context.\n      debug('describe: cropping to searchArea', searchArea);\n      const croppedResult = await cropByRect(\n        imagePayload,\n        searchArea,\n        modelFamily === 'qwen2.5-vl',\n      );\n      imagePayload = croppedResult.imageBase64;\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const res = await callAIWithObjectResponse<AIDescribeElementResponse>(\n      msgs,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n\n  async verifyCachedElement(\n    center: [number, number],\n    targetPrompt: TUserPrompt,\n    modelConfig: IModelConfig,\n    uiContext?: UIContext,\n  ): Promise<CacheVerificationResult> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    assert(screenshotBase64, 'screenshot is required for verifyCachedElement');\n\n    const defaultRectSize = 60;\n    const targetRect: Rect = {\n      left: Math.floor(center[0] - defaultRectSize / 2),\n      top: Math.floor(center[1] - defaultRectSize / 2),\n      width: defaultRectSize,\n      height: defaultRectSize,\n    };\n\n    const cropArea = expandSearchArea(targetRect, shotSize);\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [{ rect: targetRect }],\n      borderThickness: 3,\n    });\n\n    debug(\n      'verifyCachedElement: cropping to area',\n      cropArea,\n      'from center',\n      center,\n    );\n    const croppedResult = await cropByRect(\n      imagePayload,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    imagePayload = croppedResult.imageBase64;\n\n    const promptText =\n      typeof targetPrompt === 'string'\n        ? targetPrompt\n        : targetPrompt.prompt || '';\n\n    const systemPrompt = cacheVerifyInstruction().replace(\n      /{{targetPrompt}}/g,\n      promptText,\n    );\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'low',\n            },\n          },\n        ],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        match: boolean;\n        description?: string;\n        reason?: string;\n        error?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        match?: boolean;\n        description?: string;\n        reason?: string;\n        error?: string;\n      };\n\n      if (content.error) {\n        debug('verifyCachedElement AI returned error:', content.error);\n        return { pass: false, reason: content.error };\n      }\n\n      const result: CacheVerificationResult = {\n        pass: !!content.match,\n        description: content.description,\n        reason: content.reason,\n      };\n\n      debug('verifyCachedElement result:', {\n        pass: result.pass,\n        description: result.description,\n        reason: result.reason,\n        targetPrompt: promptText,\n      });\n\n      return result;\n    } catch (error) {\n      debug('verifyCachedElement failed:', error);\n      return { pass: false, reason: String(error) };\n    }\n  }\n\n  async generateSemanticAnchor(\n    center: [number, number],\n    modelConfig: IModelConfig,\n    uiContext?: UIContext,\n  ): Promise<import('@/types').SemanticAnchor | undefined> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    if (!screenshotBase64) return undefined;\n\n    const defaultRectSize = 60;\n    const targetRect: Rect = {\n      left: Math.floor(center[0] - defaultRectSize / 2),\n      top: Math.floor(center[1] - defaultRectSize / 2),\n      width: defaultRectSize,\n      height: defaultRectSize,\n    };\n\n    const cropArea = expandSearchArea(targetRect, shotSize);\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size: shotSize,\n      elementsPositionInfo: [{ rect: targetRect }],\n      borderThickness: 3,\n    });\n\n    const croppedResult = await cropByRect(\n      imagePayload,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    imagePayload = croppedResult.imageBase64;\n\n    const systemPrompt = semanticAnchorGenerateInstruction();\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: { url: imagePayload, detail: 'low' },\n          },\n        ],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        visualFingerprint?: string;\n        contextDescription?: string;\n        nearbyLandmarks?: Array<{ description?: string; xpath?: string }>;\n        error?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        visualFingerprint?: string;\n        contextDescription?: string;\n        nearbyLandmarks?: Array<{ description?: string; xpath?: string }>;\n        error?: string;\n      };\n\n      if (content.error || !content.visualFingerprint) {\n        debug('generateSemanticAnchor AI error:', content.error);\n        return undefined;\n      }\n\n      const landmarks: Landmark[] = (content.nearbyLandmarks || [])\n        .filter((l) => l.description && l.xpath)\n        .map((l) => ({\n          description: l.description!,\n          xpath: l.xpath!,\n        }));\n\n      const anchor: import('@/types').SemanticAnchor = {\n        visualFingerprint: content.visualFingerprint,\n        contextDescription: content.contextDescription || '',\n        nearbyLandmarks: landmarks,\n      };\n\n      debug('generateSemanticAnchor result:', anchor);\n      return anchor;\n    } catch (error) {\n      debug('generateSemanticAnchor failed:', error);\n      return undefined;\n    }\n  }\n\n  async locateBySemanticAnchor(\n    anchor: import('@/types').SemanticAnchor,\n    modelConfig: IModelConfig,\n    interfaceInstance: import('@/device').AbstractInterface,\n    uiContext?: UIContext,\n  ): Promise<LocateResultElement | null> {\n    const context = uiContext || (await this.contextRetrieverFn());\n    const { shotSize } = context;\n    const screenshotBase64 = context.screenshot.base64;\n    if (!screenshotBase64) return null;\n\n    let landmarkRect: Rect | undefined;\n\n    for (const landmark of anchor.nearbyLandmarks) {\n      if (interfaceInstance.rectMatchesCacheFeature) {\n        try {\n          const rect = await interfaceInstance.rectMatchesCacheFeature({\n            xpaths: [landmark.xpath],\n          });\n          if (rect) {\n            landmarkRect = rect;\n            debug('semantic anchor: landmark XPath hit:', landmark.description);\n            break;\n          }\n        } catch {\n          debug('semantic anchor: landmark XPath miss:', landmark.description);\n        }\n      }\n    }\n\n    let cropArea: Rect;\n    if (landmarkRect) {\n      const expandSize = 300;\n      cropArea = {\n        left: Math.max(0, landmarkRect.left - expandSize),\n        top: Math.max(0, landmarkRect.top - expandSize),\n        width: landmarkRect.width + expandSize * 2,\n        height: landmarkRect.height + expandSize * 2,\n      };\n    } else {\n      cropArea = {\n        left: 0,\n        top: 0,\n        width: shotSize.width,\n        height: shotSize.height,\n      };\n    }\n\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      cropArea,\n      modelConfig.modelFamily === 'qwen2.5-vl',\n    );\n    const imagePayload: string = croppedResult.imageBase64;\n\n    const systemPrompt = semanticAnchorSearchInstruction()\n      .replace('{{visualFingerprint}}', anchor.visualFingerprint)\n      .replace('{{contextDescription}}', anchor.contextDescription);\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url' as const,\n            image_url: { url: imagePayload, detail: 'low' },\n          },\n        ] as ChatCompletionUserMessageParam['content'],\n      },\n    ];\n\n    try {\n      const res = await callAIWithObjectResponse<{\n        found?: boolean;\n        center?: [number, number] | null;\n        rect?: Rect | null;\n        confidence?: number;\n        reason?: string;\n      }>(msgs, modelConfig);\n      const content = res.content as {\n        found?: boolean;\n        center?: [number, number] | null;\n        rect?: Rect | null;\n        confidence?: number;\n        reason?: string;\n      };\n\n      if (\n        !content.found ||\n        !content.center ||\n        (content.confidence ?? 0) < 0.5\n      ) {\n        debug('semantic anchor locate: not found or low confidence', content);\n        return null;\n      }\n\n      // Validate center format: must be [number, number]\n      if (\n        !Array.isArray(content.center) ||\n        content.center.length !== 2 ||\n        typeof content.center[0] !== 'number' ||\n        typeof content.center[1] !== 'number' ||\n        Number.isNaN(content.center[0]) ||\n        Number.isNaN(content.center[1])\n      ) {\n        debug(\n          'semantic anchor locate: invalid center format',\n          content.center,\n          'expected [number, number]',\n        );\n        return null;\n      }\n\n      const adjustedCenter: [number, number] = [\n        content.center[0] + cropArea.left,\n        content.center[1] + cropArea.top,\n      ];\n\n      let adjustedRect: Rect | undefined;\n      if (content.rect) {\n        adjustedRect = {\n          left: content.rect.left + cropArea.left,\n          top: content.rect.top + cropArea.top,\n          width: content.rect.width,\n          height: content.rect.height,\n        };\n      }\n\n      debug(\n        'semantic anchor locate: found at',\n        adjustedCenter,\n        'confidence:',\n        content.confidence,\n      );\n\n      return {\n        center: adjustedCenter,\n        rect: adjustedRect || {\n          left: adjustedCenter[0] - 30,\n          top: adjustedCenter[1] - 15,\n          width: 60,\n          height: 30,\n        },\n        description: `semantic anchor: ${anchor.contextDescription}`,\n      };\n    } catch (error) {\n      debug('locateBySemanticAnchor failed:', error);\n      return null;\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Service","query","opt","modelConfig","abortSignal","queryPrompt","assert","searchAreaPrompt","modelFamily","console","undefined","isAutoGLM","context","searchArea","searchAreaRawResponse","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","rawResponse","usage","reasoning_content","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","dump","createServiceDump","ServiceError","dataDemand","pageDescription","multimodalPrompt","result","AiExtractElementInfo","error","AIResponseParseError","data","thought","target","shotSize","screenshotBase64","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","croppedResult","cropByRect","msgs","res","callAIWithObjectResponse","content","center","targetPrompt","uiContext","cropArea","promptText","cacheVerifyInstruction","String","semanticAnchorGenerateInstruction","landmarks","l","anchor","interfaceInstance","landmarkRect","landmark","expandSize","semanticAnchorSearchInstruction","Number","adjustedCenter","adjustedRect","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AC4CA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAqBnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACzBC,WAAyB,EACM;QAC/B,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QAEpBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,IAAIM;QACJ,IAAIN,MAAM,UAAU,EAClBM,mBAAmBN,MAAM,MAAM;QAGjC,MAAM,EAAEO,WAAW,EAAE,GAAGL;QAExB,IAAII,oBAAoB,CAACC,aAAa;YACpCC,QAAQ,IAAI,CACV;YAEFF,mBAAmBG;QACrB;QAEA,IAAIH,oBAAoBI,AAAAA,IAAAA,wBAAAA,SAAAA,AAAAA,EAAUH,cAAc;YAC9CC,QAAQ,IAAI,CAAC;YACbF,mBAAmBG;QACrB;QAEA,MAAME,UAAUV,KAAK,WAAY,MAAM,IAAI,CAAC,kBAAkB;QAE9D,IAAIW;QACJ,IAAIC;QAEJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBACzCJ;gBACA,oBAAoBL;gBACpBJ;gBACAC;YACF;YACAE,IAAAA,sBAAAA,MAAAA,AAAAA,EACES,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJD,wBAAwBC,mBAAmB,WAAW;YACpCA,mBAAmB,KAAK;YAC1CF,aAAaE,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEC,IAAI,EAAEC,WAAW,EAAEC,KAAK,EAAEC,iBAAiB,EAAE,GAChE,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgB;YACpBZ;YACA,0BAA0BP;YAC1B,cAAcU;YACdZ;YACAC;QACF;QAEF,MAAMqB,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAT;YACAC;YACAS;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,4BAA4B,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG3E,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAASxB;YACX;YACA,gBAAgB,EAAE;YAClB,aAAae;YACb,MAAM;YACN,YAAY,CAAC,CAACP,cAAc,CAAC,CAACN;YAC9B,OAAOqB;YACPF;QACF;QAEA,MAAMI,WAAWX,YAAY,QAAQ,IAAI,EAAE;QAE3C,MAAMY,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACX,gBAAgBC;QAClB;QAEA,IAAIF,UACF,MAAM,IAAIK,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,IAAID,SAAS,MAAM,GAAG,GACpB,MAAM,IAAIG,kCAAAA,YAAYA,CACpB,CAAC,0CAA0C,EAAEH,SAAS,MAAM,EAAE,EAC9DC;QAIJ,IAAID,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,aAAaA,QAAQ,CAAC,EAAE,CAAE,WAAW;YACvC;YACAV;YACAW;QACF;QAGF,OAAO;YACL,SAAS;YACTX;YACAW;QACF;IACF;IAEA,MAAM,QACJG,UAA+B,EAC/B/B,WAAyB,EACzBD,GAA0B,EAC1BiC,eAAwB,EACxBC,gBAAoC,EACpCxB,OAAmB,EACe;QAClCN,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,SAAS;QAChBN,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAO4B,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAGvE,MAAMjB,YAAYC,KAAK,GAAG;QAE1B,IAAIC;QAGJ,IAAIE;QACJ,IAAIC;QACJ,IAAIC;QAEJ,IAAI;YACF,MAAMc,SAAS,MAAMC,AAAAA,IAAAA,yBAAAA,oBAAAA,AAAAA,EAAwB;gBAC3C1B;gBACA,WAAWsB;gBACXE;gBACA,eAAelC;gBACfC;gBACAgC;YACF;YACAhB,cAAckB,OAAO,WAAW;YAChChB,cAAcgB,OAAO,WAAW;YAChCf,QAAQe,OAAO,KAAK;YACpBd,oBAAoBc,OAAO,iBAAiB;QAC9C,EAAE,OAAOE,OAAO;YACd,IAAIA,iBAAiBC,yBAAAA,oBAAoBA,EAAE;gBAEzC,MAAMf,WAAWP,KAAK,GAAG,KAAKD;gBAC9B,MAAMS,WAA4B;oBAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;oBACtC,YAAYD;oBACZ,aAAac,MAAM,WAAW;oBAC9B,OAAOA,MAAM,KAAK;gBACpB;gBACA,MAAMR,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;oBAC7B,MAAM;oBACN,WAAW;wBAAEE;oBAAW;oBACxB,gBAAgB,EAAE;oBAClB,MAAM;oBACNR;oBACA,OAAOa,MAAM,OAAO;gBACtB;gBACA,MAAM,IAAIN,kCAAAA,YAAYA,CAACM,MAAM,OAAO,EAAER;YACxC;YACA,MAAMQ;QACR;QAEA,MAAMd,WAAWP,KAAK,GAAG,KAAKD;QAC9B,MAAMS,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZJ;YACA,gBAAgBM,KAAK,SAAS,CAACR;YAC/BG;YACAC;QACF;QAEA,IAAIK;QACJ,IAAIT,YAAY,MAAM,EAAE,QACtBS,WAAW,CAAC,qBAAqB,EAAET,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMU,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTK;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNR;YACA,OAAOE;QACT;QAEA,MAAM,EAAEa,IAAI,EAAEC,OAAO,EAAE,GAAGvB,eAAe,CAAC;QAE1C,MAAMY,OAAOC,AAAAA,IAAAA,kCAAAA,iBAAAA,AAAAA,EAAkB;YAC7B,GAAGH,QAAQ;YACXY;QACF;QAEA,IAAIb,YAAY,CAACa,MACf,MAAM,IAAIR,kCAAAA,YAAYA,CAACL,UAAUG;QAGnC,OAAO;YACLU;YACAC;YACApB;YACAC;YACAQ;QACF;IACF;IAEA,MAAM,SACJY,MAA+B,EAC/BxC,WAAyB,EACzBD,GAEC,EACwD;QACzDI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOqC,QAAQ;QACf,MAAM/B,UAAU,MAAM,IAAI,CAAC,kBAAkB;QAC7C,MAAM,EAAEgC,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClDN,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuC,kBAAkB;QAEzB,MAAM,EAAErC,WAAW,EAAE,GAAGL;QACxB,MAAM2C,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBACpB;oBACE,MAAMK;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAI/C,KAAK,YAAY;YACnB,MAAMW,aAAayC,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;YAMhD9C,MAAM,oCAAoCe;YAC1C,MAAM0C,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BJ,cACAvC,YACAL,AAAgB,iBAAhBA;YAEF4C,eAAeG,cAAc,WAAW;QAC1C;QAEA,MAAME,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChBF,MACAtD;QAGF,MAAM,EAAEyD,OAAO,EAAE,GAAGF;QACpBpD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACsD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DtD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOsD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAEA,MAAM,oBACJC,MAAwB,EACxBC,YAAyB,EACzB3D,WAAyB,EACzB4D,SAAqB,EACa;QAClC,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClDN,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuC,kBAAkB;QAEzB,MAAMG,kBAAkB;QACxB,MAAMC,aAAmB;YACvB,MAAME,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV;QAEA,MAAMgB,WAAWV,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;QAE9C,IAAIQ,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBAAC;oBAAE,MAAMK;gBAAW;aAAE;YAC5C,iBAAiB;QACnB;QAEAnD,MACE,yCACAkE,UACA,eACAH;QAEF,MAAMN,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BJ,cACAY,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzBiD,eAAeG,cAAc,WAAW;QAExC,MAAMU,aACJ,AAAwB,YAAxB,OAAOH,eACHA,eACAA,aAAa,MAAM,IAAI;QAE7B,MAAMhB,eAAeoB,AAAAA,IAAAA,gCAAAA,sBAAAA,AAAAA,IAAyB,OAAO,CACnD,qBACAD;QAGF,MAAMR,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAKfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAO3B,IAAIE,QAAQ,KAAK,EAAE;gBACjB9D,MAAM,0CAA0C8D,QAAQ,KAAK;gBAC7D,OAAO;oBAAE,MAAM;oBAAO,QAAQA,QAAQ,KAAK;gBAAC;YAC9C;YAEA,MAAMvB,SAAkC;gBACtC,MAAM,CAAC,CAACuB,QAAQ,KAAK;gBACrB,aAAaA,QAAQ,WAAW;gBAChC,QAAQA,QAAQ,MAAM;YACxB;YAEA9D,MAAM,+BAA+B;gBACnC,MAAMuC,OAAO,IAAI;gBACjB,aAAaA,OAAO,WAAW;gBAC/B,QAAQA,OAAO,MAAM;gBACrB,cAAc4B;YAChB;YAEA,OAAO5B;QACT,EAAE,OAAOE,OAAO;YACdzC,MAAM,+BAA+ByC;YACrC,OAAO;gBAAE,MAAM;gBAAO,QAAQ4B,OAAO5B;YAAO;QAC9C;IACF;IAEA,MAAM,uBACJsB,MAAwB,EACxB1D,WAAyB,EACzB4D,SAAqB,EACkC;QACvD,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClD,IAAI,CAACiC,kBAAkB;QAEvB,MAAMG,kBAAkB;QACxB,MAAMC,aAAmB;YACvB,MAAME,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACU,MAAM,CAAC,EAAE,GAAGb,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV;QAEA,MAAMgB,WAAWV,AAAAA,IAAAA,mCAAAA,gBAAAA,AAAAA,EAAiBL,YAAYL;QAE9C,IAAIQ,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBR;YAChB,MAAMD;YACN,sBAAsB;gBAAC;oBAAE,MAAMK;gBAAW;aAAE;YAC5C,iBAAiB;QACnB;QAEA,MAAMM,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BJ,cACAY,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzBiD,eAAeG,cAAc,WAAW;QAExC,MAAMT,eAAesB,AAAAA,IAAAA,mCAAAA,iCAAAA,AAAAA;QAErB,MAAMX,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BAAE,KAAKM;4BAAc,QAAQ;wBAAM;oBAChD;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAKfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAO3B,IAAIE,QAAQ,KAAK,IAAI,CAACA,QAAQ,iBAAiB,EAAE,YAC/C9D,MAAM,oCAAoC8D,QAAQ,KAAK;YAIzD,MAAMS,YAAyBT,AAAAA,CAAAA,QAAQ,eAAe,IAAI,EAAC,EACxD,MAAM,CAAC,CAACU,IAAMA,EAAE,WAAW,IAAIA,EAAE,KAAK,EACtC,GAAG,CAAC,CAACA,IAAO;oBACX,aAAaA,EAAE,WAAW;oBAC1B,OAAOA,EAAE,KAAK;gBAChB;YAEF,MAAMC,SAA2C;gBAC/C,mBAAmBX,QAAQ,iBAAiB;gBAC5C,oBAAoBA,QAAQ,kBAAkB,IAAI;gBAClD,iBAAiBS;YACnB;YAEAvE,MAAM,kCAAkCyE;YACxC,OAAOA;QACT,EAAE,OAAOhC,OAAO;YACdzC,MAAM,kCAAkCyC;YACxC;QACF;IACF;IAEA,MAAM,uBACJgC,MAAwC,EACxCpE,WAAyB,EACzBqE,iBAAuD,EACvDT,SAAqB,EACgB;QACrC,MAAMnD,UAAUmD,aAAc,MAAM,IAAI,CAAC,kBAAkB;QAC3D,MAAM,EAAEnB,QAAQ,EAAE,GAAGhC;QACrB,MAAMiC,mBAAmBjC,QAAQ,UAAU,CAAC,MAAM;QAClD,IAAI,CAACiC,kBAAkB,OAAO;QAE9B,IAAI4B;QAEJ,KAAK,MAAMC,YAAYH,OAAO,eAAe,CAC3C,IAAIC,kBAAkB,uBAAuB,EAC3C,IAAI;YACF,MAAMpD,OAAO,MAAMoD,kBAAkB,uBAAuB,CAAC;gBAC3D,QAAQ;oBAACE,SAAS,KAAK;iBAAC;YAC1B;YACA,IAAItD,MAAM;gBACRqD,eAAerD;gBACftB,MAAM,wCAAwC4E,SAAS,WAAW;gBAClE;YACF;QACF,EAAE,OAAM;YACN5E,MAAM,yCAAyC4E,SAAS,WAAW;QACrE;QAIJ,IAAIV;QACJ,IAAIS,cAAc;YAChB,MAAME,aAAa;YACnBX,WAAW;gBACT,MAAMb,KAAK,GAAG,CAAC,GAAGsB,aAAa,IAAI,GAAGE;gBACtC,KAAKxB,KAAK,GAAG,CAAC,GAAGsB,aAAa,GAAG,GAAGE;gBACpC,OAAOF,aAAa,KAAK,GAAGE,AAAa,IAAbA;gBAC5B,QAAQF,aAAa,MAAM,GAAGE,AAAa,IAAbA;YAChC;QACF,OACEX,WAAW;YACT,MAAM;YACN,KAAK;YACL,OAAOpB,SAAS,KAAK;YACrB,QAAQA,SAAS,MAAM;QACzB;QAGF,MAAMW,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EAC1BX,kBACAmB,UACA7D,AAA4B,iBAA5BA,YAAY,WAAW;QAEzB,MAAMiD,eAAuBG,cAAc,WAAW;QAEtD,MAAMT,eAAe8B,AAAAA,IAAAA,0CAAAA,+BAAAA,AAAAA,IAClB,OAAO,CAAC,yBAAyBL,OAAO,iBAAiB,EACzD,OAAO,CAAC,0BAA0BA,OAAO,kBAAkB;QAE9D,MAAMd,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BAAE,KAAKM;4BAAc,QAAQ;wBAAM;oBAChD;iBACD;YACH;SACD;QAED,IAAI;YACF,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAMfF,MAAMtD;YACT,MAAMyD,UAAUF,IAAI,OAAO;YAQ3B,IACE,CAACE,QAAQ,KAAK,IACd,CAACA,QAAQ,MAAM,IACdA,AAAAA,CAAAA,QAAQ,UAAU,IAAI,KAAK,KAC5B;gBACA9D,MAAM,uDAAuD8D;gBAC7D,OAAO;YACT;YAGA,IACE,CAACV,MAAM,OAAO,CAACU,QAAQ,MAAM,KAC7BA,AAA0B,MAA1BA,QAAQ,MAAM,CAAC,MAAM,IACrB,AAA6B,YAA7B,OAAOA,QAAQ,MAAM,CAAC,EAAE,IACxB,AAA6B,YAA7B,OAAOA,QAAQ,MAAM,CAAC,EAAE,IACxBiB,OAAO,KAAK,CAACjB,QAAQ,MAAM,CAAC,EAAE,KAC9BiB,OAAO,KAAK,CAACjB,QAAQ,MAAM,CAAC,EAAE,GAC9B;gBACA9D,MACE,iDACA8D,QAAQ,MAAM,EACd;gBAEF,OAAO;YACT;YAEA,MAAMkB,iBAAmC;gBACvClB,QAAQ,MAAM,CAAC,EAAE,GAAGI,SAAS,IAAI;gBACjCJ,QAAQ,MAAM,CAAC,EAAE,GAAGI,SAAS,GAAG;aACjC;YAED,IAAIe;YACJ,IAAInB,QAAQ,IAAI,EACdmB,eAAe;gBACb,MAAMnB,QAAQ,IAAI,CAAC,IAAI,GAAGI,SAAS,IAAI;gBACvC,KAAKJ,QAAQ,IAAI,CAAC,GAAG,GAAGI,SAAS,GAAG;gBACpC,OAAOJ,QAAQ,IAAI,CAAC,KAAK;gBACzB,QAAQA,QAAQ,IAAI,CAAC,MAAM;YAC7B;YAGF9D,MACE,oCACAgF,gBACA,eACAlB,QAAQ,UAAU;YAGpB,OAAO;gBACL,QAAQkB;gBACR,MAAMC,gBAAgB;oBACpB,MAAMD,cAAc,CAAC,EAAE,GAAG;oBAC1B,KAAKA,cAAc,CAAC,EAAE,GAAG;oBACzB,OAAO;oBACP,QAAQ;gBACV;gBACA,aAAa,CAAC,iBAAiB,EAAEP,OAAO,kBAAkB,EAAE;YAC9D;QACF,EAAE,OAAOhC,OAAO;YACdzC,MAAM,kCAAkCyC;YACxC,OAAO;QACT;IACF;IAxqBA,YACE3B,OAA2D,EAC3DV,GAAoB,CACpB;QAPF;QAEA;QAMEI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOM,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMoE,QAAQ,OAAO,CAACpE;QAGlD,IAAI,AAAyB,WAAlBV,KAAK,UACd,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AA2pBF"}

package/dist/lib/types.js

@@ -73,10 +73,10 @@ var __webpack_exports__ = {};
     var __rspack_reexport = {};
     for(const __rspack_import_key in _yaml__rspack_import_4)if ([
         "UIContext",
-        "ServiceError",
-        "default",
         "GroupedActionDump",
-        "ExecutionDump"
+        "default",
+        "ExecutionDump",
+        "ServiceError"
     ].indexOf(__rspack_import_key) < 0) __rspack_reexport[__rspack_import_key] = ()=>_yaml__rspack_import_4[__rspack_import_key];
     __webpack_require__.d(__webpack_exports__, __rspack_reexport);
     function _define_property(obj, key, value) {

package/dist/lib/utils.js

@@ -202,7 +202,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "1.6.0";
+    return "1.6.4";
 }
 function debugLog(...message) {
     const debugMode = process.env[env_namespaceObject.MIDSCENE_DEBUG_MODE];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@donggui/core",
   "description": "DongGUI Core - AI-powered automation SDK with enhanced assertion capabilities",
-  "version": "1.6.3",
+  "version": "1.6.4",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",