@donggui/core 1.6.9 → 1.6.11

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import {\n  calculateConfidence,\n  createInitialConfidenceState,\n  createInitialProgressiveRecord,\n  determineVerificationLevel,\n  getVerificationActions,\n  updateConfidenceOnVerify,\n  updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  CacheConfidenceState,\n  CacheValidationOptions,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  ProgressiveLocateRecord,\n  Rect,\n  SemanticAnchor,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n  MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n  MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n  MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord =\n          await this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        let elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        let isCacheHit = !!elementFromCache;\n        let isLegacyCache = false;\n        const timing = taskContext.task.timing;\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n\n        if (isCacheHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const cachedCenter = cacheFeature?.cachedCenter as\n            | [number, number]\n            | undefined;\n\n          isLegacyCache = !cachedCenter;\n\n          const enableCoordCheck =\n            process.env[MIDSCENE_CACHE_ENABLE_COORD_CHECK] === 'true';\n          const enableVisualVerify =\n            process.env[MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY] === 'true';\n\n          const coordOffsetThreshold =\n            Number.parseInt(\n              process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] || '16',\n              10,\n            ) || 16;\n\n          if (enableCoordCheck || enableVisualVerify) {\n            const confidenceState = (cacheFeature?.confidenceState ||\n              createInitialConfidenceState()) as CacheConfidenceState;\n            const confidence = isLegacyCache\n              ? 0.35\n              : calculateConfidence(confidenceState);\n            const level = determineVerificationLevel(confidence);\n            const actions = getVerificationActions(level);\n\n            debug('cache confidence assessment', {\n              confidence: confidence.toFixed(3),\n              level,\n              actions,\n              verificationCount: confidenceState.verificationCount,\n              isLegacyCache,\n            });\n\n            if (actions.skipCache) {\n              debug('cache confidence too low, skipping cache entirely', {\n                confidence,\n                level,\n              });\n              isCacheHit = false;\n            }\n\n            try {\n              if (\n                isCacheHit &&\n                enableCoordCheck &&\n                actions.coordCheck &&\n                cachedCenter\n              ) {\n                const offset = Math.sqrt(\n                  (elementFromCache!.center[0] - cachedCenter[0]) ** 2 +\n                    (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n                );\n\n                debug('cache coord offset check', {\n                  cachedCenter,\n                  currentCenter: elementFromCache!.center,\n                  offset: Math.round(offset),\n                  threshold: coordOffsetThreshold,\n                });\n\n                if (offset > coordOffsetThreshold) {\n                  debug(\n                    'cache coord offset exceeded threshold, fallback to AI locate',\n                    { offset, threshold: coordOffsetThreshold },\n                  );\n                  isCacheHit = false;\n                }\n              }\n\n              if (isCacheHit && enableVisualVerify && actions.visualVerify) {\n                const verification = await this.service.verifyCachedElement(\n                  elementFromCache!.center,\n                  cachePrompt,\n                  modelConfigForDefaultIntent,\n                  uiContext,\n                );\n                if (!verification.pass) {\n                  debug(\n                    'cache hit but visual verification failed, fallback to AI locate',\n                    {\n                      reason: verification.reason,\n                      description: verification.description,\n                      prompt: cachePrompt,\n                    },\n                  );\n                  isCacheHit = false;\n                } else {\n                  debug('cache hit and visual verification passed', {\n                    description: verification.description,\n                  });\n                }\n              }\n            } catch (verifyError) {\n              debug(\n                'cache verification error, fallback to AI locate',\n                verifyError,\n              );\n              isCacheHit = false;\n            }\n\n            if (isCacheHit) {\n              const updatedState = updateConfidenceOnVerify(\n                confidenceState,\n                true,\n              );\n              cacheFeature.confidenceState = updatedState;\n\n              if (isLegacyCache) {\n                cacheFeature.cachedCenter = elementFromCache!.center;\n                cacheFeature.progressiveRecord = createInitialProgressiveRecord(\n                  elementFromCache!.center,\n                );\n                debug('legacy cache upgraded with new fields', {\n                  cachedCenter: cacheFeature.cachedCenter,\n                });\n              }\n\n              const progressiveRecord = cacheFeature.progressiveRecord as\n                | ProgressiveLocateRecord\n                | undefined;\n              if (progressiveRecord) {\n                const updated = updateProgressiveConvergence(\n                  progressiveRecord,\n                  elementFromCache!.center,\n                  updatedState.confidenceScore,\n                );\n                cacheFeature.progressiveRecord = updated;\n\n                if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n                  debug(\n                    'using converged center instead of single-result center',\n                    {\n                      convergedCenter: updated.convergedCenter.map((v) =>\n                        v.toFixed(1),\n                      ),\n                      singleCenter: elementFromCache!.center,\n                      convergenceRadius: updated.convergenceRadius.toFixed(1),\n                      sampleCount: updated.sampleCount,\n                    },\n                  );\n                  elementFromCache = {\n                    ...elementFromCache!,\n                    center: [\n                      Math.round(updated.convergedCenter[0]),\n                      Math.round(updated.convergedCenter[1]),\n                    ],\n                  };\n                }\n              }\n            } else {\n              const updatedState = updateConfidenceOnVerify(\n                confidenceState,\n                false,\n              );\n              cacheFeature.confidenceState = updatedState;\n            }\n          }\n        }\n\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const semanticAnchor = cacheFeature?.semanticAnchor as\n            | SemanticAnchor\n            | undefined;\n\n          if (\n            semanticAnchor &&\n            process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true'\n          ) {\n            try {\n              const anchorResult = await this.service.locateBySemanticAnchor(\n                semanticAnchor,\n                modelConfigForDefaultIntent,\n                this.interface,\n                uiContext,\n              );\n              if (anchorResult) {\n                elementFromAiLocate = anchorResult;\n                debug(\n                  'semantic anchor locate succeeded, skipping full AI locate',\n                );\n              }\n            } catch (anchorError) {\n              debug('semantic anchor locate failed:', anchorError);\n            }\n          }\n\n          if (!elementFromAiLocate) {\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              locateResult = await this.service.locate(\n                param,\n                {\n                  context: uiContext,\n                },\n                modelConfigForDefaultIntent,\n                abortSignal,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n            } catch (error) {\n              if (error instanceof ServiceError) {\n                applyDump(error.dump);\n              }\n              throw error;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read), OR cache hit but legacy cache needs upgrade\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          (!isCacheHit || isLegacyCache) &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                feature.cachedCenter = pointForCache;\n                feature.confidenceState = createInitialConfidenceState();\n                feature.progressiveRecord =\n                  createInitialProgressiveRecord(pointForCache);\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n\n                const enableSemanticAnchor =\n                  process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true';\n                if (enableSemanticAnchor) {\n                  try {\n                    const anchor = await this.service.generateSemanticAnchor(\n                      pointForCache,\n                      modelConfigForDefaultIntent,\n                      uiContext,\n                    );\n                    if (anchor) {\n                      feature.semanticAnchor = anchor;\n                      debug(\n                        'semantic anchor generated for prompt: %s',\n                        cachePrompt,\n                      );\n                    }\n                  } catch (anchorError) {\n                    debug('generateSemanticAnchor failed:', anchorError);\n                  }\n                }\n\n                currentCacheEntry = feature;\n                await this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","isLegacyCache","elementFromAiLocate","cacheFeature","cachedCenter","enableCoordCheck","process","MIDSCENE_CACHE_ENABLE_COORD_CHECK","enableVisualVerify","MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY","coordOffsetThreshold","Number","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","createInitialProgressiveRecord","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqDA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC3C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,MAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,iBAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,mBAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,wBAAwB5B,SAC5C4D,qBAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,oCACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,IAAIG,gBAAgB;gBACpB,MAAMzC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI2C;gBAEJ,IAAIF,YAAY;oBACd,MAAMG,eAAeR;oBACrB,MAAMS,eAAeD,cAAc;oBAInCF,gBAAgB,CAACG;oBAEjB,MAAMC,mBACJC,AAAmD,WAAnDA,QAAQ,GAAG,CAACC,kCAAkC;oBAChD,MAAMC,qBACJF,AAAqD,WAArDA,QAAQ,GAAG,CAACG,oCAAoC;oBAElD,MAAMC,uBACJC,OAAO,QAAQ,CACbL,QAAQ,GAAG,CAACM,sCAAsC,IAAI,MACtD,OACG;oBAEP,IAAIP,oBAAoBG,oBAAoB;wBAC1C,MAAMK,kBAAmBV,cAAc,mBACrCW;wBACF,MAAMC,aAAad,gBACf,OACAe,oBAAoBH;wBACxB,MAAMI,QAAQC,2BAA2BH;wBACzC,MAAMI,UAAUC,uBAAuBH;wBAEvCjG,MAAM,+BAA+B;4BACnC,YAAY+F,WAAW,OAAO,CAAC;4BAC/BE;4BACAE;4BACA,mBAAmBN,gBAAgB,iBAAiB;4BACpDZ;wBACF;wBAEA,IAAIkB,QAAQ,SAAS,EAAE;4BACrBnG,MAAM,qDAAqD;gCACzD+F;gCACAE;4BACF;4BACAjB,aAAa;wBACf;wBAEA,IAAI;4BACF,IACEA,cACAK,oBACAc,QAAQ,UAAU,IAClBf,cACA;gCACA,MAAMiB,SAASC,KAAK,IAAI,CACrBxB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM,IAChDN,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM;gCAGvDpF,MAAM,4BAA4B;oCAChCoF;oCACA,eAAeN,iBAAkB,MAAM;oCACvC,QAAQwB,KAAK,KAAK,CAACD;oCACnB,WAAWX;gCACb;gCAEA,IAAIW,SAASX,sBAAsB;oCACjC1F,MACE,gEACA;wCAAEqG;wCAAQ,WAAWX;oCAAqB;oCAE5CV,aAAa;gCACf;4BACF;4BAEA,IAAIA,cAAcQ,sBAAsBW,QAAQ,YAAY,EAAE;gCAC5D,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDzB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;gCAEF,IAAK8D,aAAa,IAAI,EAWpBvG,MAAM,4CAA4C;oCAChD,aAAauG,aAAa,WAAW;gCACvC;qCAbsB;oCACtBvG,MACE,mEACA;wCACE,QAAQuG,aAAa,MAAM;wCAC3B,aAAaA,aAAa,WAAW;wCACrC,QAAQ9B;oCACV;oCAEFO,aAAa;gCACf;4BAKF;wBACF,EAAE,OAAOwB,aAAa;4BACpBxG,MACE,mDACAwG;4BAEFxB,aAAa;wBACf;wBAEA,IAAIA,YAAY;4BACd,MAAMyB,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;4BAE/B,IAAIxB,eAAe;gCACjBE,aAAa,YAAY,GAAGL,iBAAkB,MAAM;gCACpDK,aAAa,iBAAiB,GAAGwB,+BAC/B7B,iBAAkB,MAAM;gCAE1B9E,MAAM,yCAAyC;oCAC7C,cAAcmF,aAAa,YAAY;gCACzC;4BACF;4BAEA,MAAMyB,oBAAoBzB,aAAa,iBAAiB;4BAGxD,IAAIyB,mBAAmB;gCACrB,MAAMC,UAAUC,6BACdF,mBACA9B,iBAAkB,MAAM,EACxB2B,aAAa,eAAe;gCAE9BtB,aAAa,iBAAiB,GAAG0B;gCAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;oCAC7D7G,MACE,0DACA;wCACE,iBAAiB6G,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;wCAEZ,cAAcjC,iBAAkB,MAAM;wCACtC,mBAAmB+B,QAAQ,iBAAiB,CAAC,OAAO,CAAC;wCACrD,aAAaA,QAAQ,WAAW;oCAClC;oCAEF/B,mBAAmB;wCACjB,GAAGA,gBAAgB;wCACnB,QAAQ;4CACNwB,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;4CACrCP,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;yCACtC;oCACH;gCACF;4BACF;wBACF,OAAO;4BACL,MAAMJ,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;wBACjC;oBACF;gBACF;gBAEA,IAAI,CAACjC,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMgB,eAAeR;oBACrB,MAAMqC,iBAAiB7B,cAAc;oBAIrC,IACE6B,kBACA1B,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACApG,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAIyE,cAAc;4BAChBhC,sBAAsBgC;4BACtBlH,MACE;wBAEJ;oBACF,EAAE,OAAOmH,aAAa;wBACpBnH,MAAM,kCAAkCmH;oBAC1C;oBAGF,IAAI,CAACjC,qBACH,IAAI;wBACFxC,mBAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BoB,sBAAsBpB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiBiE,cACnBrD,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,mBAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAM6E,UACJpD,mBACAI,oBACAS,oBACAI;gBAGF,MAAMoC,2BAA2BpH,iBAC/BwE,mBAAmB,cAAc;gBAGnC,IAAI6C;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACb,EAACrC,cAAcC,aAAY,KAC3B,EAACd,aAAa,CAACmD,wBAAuB,KACvChH,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIkH,gBAAkCH,QAAQ,MAAM;oBACpD,IAAIrE,AAA6B,MAA7BA,0BAAgC;wBAClCwE,gBAAgB;4BACdlB,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;4BAC/BsD,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;yBAChC;wBACDhD,MACE,8DACAqH,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOlH,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiBuH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG3B;wBAC1B2B,QAAQ,iBAAiB,GACvBd,+BAA+Ba;wBACjCxH,MACE,uCACAyE,aACAgD;wBAGF,MAAMC,uBACJpC,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC;wBACpD,IAAIS,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDH,eACA5G,6BACA6B;4BAEF,IAAIkF,QAAQ;gCACVF,QAAQ,cAAc,GAAGE;gCACzB3H,MACE,4CACAyE;4BAEJ;wBACF,EAAE,OAAO0C,aAAa;4BACpBnH,MAAM,kCAAkCmH;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQhD;4BACR,OAAOgD;wBACT,GACA/C;oBAEJ,OACE1E,MACE,yDACAyE;gBAGN,EAAE,OAAOtB,OAAO;oBACdnD,MAAM,mCAAmCmD;gBAC3C;qBAEAnD,MAAM;gBAIV,IAAI,CAACqH,SAAS;oBACZ,IAAIxD,YACF,MAAM,IAAIuD,aACR,CAAC,oBAAoB,EAAE9G,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIsH;gBAEJ,IAAIzD,WACFyD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMtH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACToD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOtH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACT4C,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPjD;wBACA,aAAa4C;oBACf;gBACF;gBAGF/D,WAAW6D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK5E,UAAU,aAAa;wBAC9B;oBACF;oBACAmF;gBACF;YACF;QACF;QAEA,OAAOhE;IACT;IAzwBA,YAAY,EACViE,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTtG,WAAW,EACXuG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGtG;QACnB,IAAI,CAAC,eAAe,GAAGuG;IACzB;AA8vBF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import {\n  calculateConfidence,\n  createInitialConfidenceState,\n  createInitialProgressiveRecord,\n  determineVerificationLevel,\n  getVerificationActions,\n  updateConfidenceOnVerify,\n  updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  CacheConfidenceState,\n  CacheValidationOptions,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  ProgressiveLocateRecord,\n  Rect,\n  SemanticAnchor,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n  MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n  MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n  MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord =\n          await this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        let elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        let isCacheHit = !!elementFromCache;\n        let isLegacyCache = false;\n        const timing = taskContext.task.timing;\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n\n        if (isCacheHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const cachedCenter = cacheFeature?.cachedCenter as\n            | [number, number]\n            | undefined;\n\n          isLegacyCache = !cachedCenter;\n\n          const enableCoordCheck =\n            process.env[MIDSCENE_CACHE_ENABLE_COORD_CHECK] === 'true';\n          const enableVisualVerify =\n            process.env[MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY] === 'true';\n\n          const coordOffsetThreshold =\n            Number.parseInt(\n              process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] || '16',\n              10,\n            ) || 16;\n\n          if (enableCoordCheck || enableVisualVerify) {\n            const confidenceState = (cacheFeature?.confidenceState ||\n              createInitialConfidenceState()) as CacheConfidenceState;\n            const confidence = isLegacyCache\n              ? 0.35\n              : calculateConfidence(confidenceState);\n            const level = determineVerificationLevel(confidence);\n            const actions = getVerificationActions(level);\n\n            debug('cache confidence assessment', {\n              confidence: confidence.toFixed(3),\n              level,\n              actions,\n              verificationCount: confidenceState.verificationCount,\n              isLegacyCache,\n            });\n\n            if (actions.skipCache) {\n              debug('cache confidence too low, skipping cache entirely', {\n                confidence,\n                level,\n              });\n              isCacheHit = false;\n              elementFromCache = undefined;\n            }\n\n            try {\n              if (\n                isCacheHit &&\n                enableCoordCheck &&\n                actions.coordCheck &&\n                cachedCenter\n              ) {\n                const offset = Math.sqrt(\n                  (elementFromCache!.center[0] - cachedCenter[0]) ** 2 +\n                    (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n                );\n\n                debug('cache coord offset check', {\n                  cachedCenter,\n                  currentCenter: elementFromCache!.center,\n                  offset: Math.round(offset),\n                  threshold: coordOffsetThreshold,\n                });\n\n                if (offset > coordOffsetThreshold) {\n                  debug(\n                    'cache coord offset exceeded threshold, fallback to AI locate',\n                    { offset, threshold: coordOffsetThreshold },\n                  );\n                  isCacheHit = false;\n                  elementFromCache = undefined;\n                }\n              }\n\n              if (isCacheHit && enableVisualVerify && actions.visualVerify) {\n                const verification = await this.service.verifyCachedElement(\n                  elementFromCache!.center,\n                  cachePrompt,\n                  modelConfigForDefaultIntent,\n                  uiContext,\n                );\n                if (!verification.pass) {\n                  debug(\n                    'cache hit but visual verification failed, fallback to AI locate',\n                    {\n                      reason: verification.reason,\n                      description: verification.description,\n                      prompt: cachePrompt,\n                    },\n                  );\n                  isCacheHit = false;\n                  elementFromCache = undefined;\n                } else {\n                  debug('cache hit and visual verification passed', {\n                    description: verification.description,\n                  });\n                }\n              }\n            } catch (verifyError) {\n              debug(\n                'cache verification error, fallback to AI locate',\n                verifyError,\n              );\n              isCacheHit = false;\n              elementFromCache = undefined;\n            }\n\n            if (isCacheHit) {\n              const updatedState = updateConfidenceOnVerify(\n                confidenceState,\n                true,\n              );\n              cacheFeature.confidenceState = updatedState;\n\n              if (isLegacyCache) {\n                cacheFeature.cachedCenter = elementFromCache!.center;\n                cacheFeature.progressiveRecord = createInitialProgressiveRecord(\n                  elementFromCache!.center,\n                );\n                debug('legacy cache upgraded with new fields', {\n                  cachedCenter: cacheFeature.cachedCenter,\n                });\n              }\n\n              const progressiveRecord = cacheFeature.progressiveRecord as\n                | ProgressiveLocateRecord\n                | undefined;\n              if (progressiveRecord) {\n                const updated = updateProgressiveConvergence(\n                  progressiveRecord,\n                  elementFromCache!.center,\n                  updatedState.confidenceScore,\n                );\n                cacheFeature.progressiveRecord = updated;\n\n                if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n                  debug(\n                    'using converged center instead of single-result center',\n                    {\n                      convergedCenter: updated.convergedCenter.map((v) =>\n                        v.toFixed(1),\n                      ),\n                      singleCenter: elementFromCache!.center,\n                      convergenceRadius: updated.convergenceRadius.toFixed(1),\n                      sampleCount: updated.sampleCount,\n                    },\n                  );\n                  elementFromCache = {\n                    ...elementFromCache!,\n                    center: [\n                      Math.round(updated.convergedCenter[0]),\n                      Math.round(updated.convergedCenter[1]),\n                    ],\n                  };\n                }\n              }\n            } else {\n              const updatedState = updateConfidenceOnVerify(\n                confidenceState,\n                false,\n              );\n              cacheFeature.confidenceState = updatedState;\n              elementFromCache = undefined;\n              debug(\n                'cache verification failed, clearing cached element for AI fallback',\n                {\n                  punishedScore: updatedState.confidenceScore.toFixed(3),\n                  verificationCount: updatedState.verificationCount,\n                },\n              );\n            }\n          }\n        }\n\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const semanticAnchor = cacheFeature?.semanticAnchor as\n            | SemanticAnchor\n            | undefined;\n\n          if (\n            semanticAnchor &&\n            process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true'\n          ) {\n            try {\n              const anchorResult = await this.service.locateBySemanticAnchor(\n                semanticAnchor,\n                modelConfigForDefaultIntent,\n                this.interface,\n                uiContext,\n              );\n              if (anchorResult) {\n                elementFromAiLocate = anchorResult;\n                debug(\n                  'semantic anchor locate succeeded, skipping full AI locate',\n                );\n              }\n            } catch (anchorError) {\n              debug('semantic anchor locate failed:', anchorError);\n            }\n          }\n\n          if (!elementFromAiLocate) {\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              locateResult = await this.service.locate(\n                param,\n                {\n                  context: uiContext,\n                },\n                modelConfigForDefaultIntent,\n                abortSignal,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n            } catch (error) {\n              if (error instanceof ServiceError) {\n                applyDump(error.dump);\n              }\n              throw error;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read), OR cache hit but legacy cache needs upgrade\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          (!isCacheHit || isLegacyCache) &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                feature.cachedCenter = pointForCache;\n                feature.confidenceState = createInitialConfidenceState();\n                feature.progressiveRecord =\n                  createInitialProgressiveRecord(pointForCache);\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n\n                const enableSemanticAnchor =\n                  process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true';\n                if (enableSemanticAnchor) {\n                  try {\n                    const anchor = await this.service.generateSemanticAnchor(\n                      pointForCache,\n                      modelConfigForDefaultIntent,\n                      uiContext,\n                    );\n                    if (anchor) {\n                      feature.semanticAnchor = anchor;\n                      debug(\n                        'semantic anchor generated for prompt: %s',\n                        cachePrompt,\n                      );\n                    }\n                  } catch (anchorError) {\n                    debug('generateSemanticAnchor failed:', anchorError);\n                  }\n                }\n\n                currentCacheEntry = feature;\n                await this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","isLegacyCache","elementFromAiLocate","cacheFeature","cachedCenter","enableCoordCheck","process","MIDSCENE_CACHE_ENABLE_COORD_CHECK","enableVisualVerify","MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY","coordOffsetThreshold","Number","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","createInitialProgressiveRecord","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqDA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC3C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,MAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,iBAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,mBAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,wBAAwB5B,SAC5C4D,qBAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,oCACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,IAAIG,gBAAgB;gBACpB,MAAMzC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI2C;gBAEJ,IAAIF,YAAY;oBACd,MAAMG,eAAeR;oBACrB,MAAMS,eAAeD,cAAc;oBAInCF,gBAAgB,CAACG;oBAEjB,MAAMC,mBACJC,AAAmD,WAAnDA,QAAQ,GAAG,CAACC,kCAAkC;oBAChD,MAAMC,qBACJF,AAAqD,WAArDA,QAAQ,GAAG,CAACG,oCAAoC;oBAElD,MAAMC,uBACJC,OAAO,QAAQ,CACbL,QAAQ,GAAG,CAACM,sCAAsC,IAAI,MACtD,OACG;oBAEP,IAAIP,oBAAoBG,oBAAoB;wBAC1C,MAAMK,kBAAmBV,cAAc,mBACrCW;wBACF,MAAMC,aAAad,gBACf,OACAe,oBAAoBH;wBACxB,MAAMI,QAAQC,2BAA2BH;wBACzC,MAAMI,UAAUC,uBAAuBH;wBAEvCjG,MAAM,+BAA+B;4BACnC,YAAY+F,WAAW,OAAO,CAAC;4BAC/BE;4BACAE;4BACA,mBAAmBN,gBAAgB,iBAAiB;4BACpDZ;wBACF;wBAEA,IAAIkB,QAAQ,SAAS,EAAE;4BACrBnG,MAAM,qDAAqD;gCACzD+F;gCACAE;4BACF;4BACAjB,aAAa;4BACbF,mBAAmB7B;wBACrB;wBAEA,IAAI;4BACF,IACE+B,cACAK,oBACAc,QAAQ,UAAU,IAClBf,cACA;gCACA,MAAMiB,SAASC,KAAK,IAAI,CACrBxB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM,IAChDN,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM;gCAGvDpF,MAAM,4BAA4B;oCAChCoF;oCACA,eAAeN,iBAAkB,MAAM;oCACvC,QAAQwB,KAAK,KAAK,CAACD;oCACnB,WAAWX;gCACb;gCAEA,IAAIW,SAASX,sBAAsB;oCACjC1F,MACE,gEACA;wCAAEqG;wCAAQ,WAAWX;oCAAqB;oCAE5CV,aAAa;oCACbF,mBAAmB7B;gCACrB;4BACF;4BAEA,IAAI+B,cAAcQ,sBAAsBW,QAAQ,YAAY,EAAE;gCAC5D,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDzB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;gCAEF,IAAK8D,aAAa,IAAI,EAYpBvG,MAAM,4CAA4C;oCAChD,aAAauG,aAAa,WAAW;gCACvC;qCAdsB;oCACtBvG,MACE,mEACA;wCACE,QAAQuG,aAAa,MAAM;wCAC3B,aAAaA,aAAa,WAAW;wCACrC,QAAQ9B;oCACV;oCAEFO,aAAa;oCACbF,mBAAmB7B;gCACrB;4BAKF;wBACF,EAAE,OAAOuD,aAAa;4BACpBxG,MACE,mDACAwG;4BAEFxB,aAAa;4BACbF,mBAAmB7B;wBACrB;wBAEA,IAAI+B,YAAY;4BACd,MAAMyB,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;4BAE/B,IAAIxB,eAAe;gCACjBE,aAAa,YAAY,GAAGL,iBAAkB,MAAM;gCACpDK,aAAa,iBAAiB,GAAGwB,+BAC/B7B,iBAAkB,MAAM;gCAE1B9E,MAAM,yCAAyC;oCAC7C,cAAcmF,aAAa,YAAY;gCACzC;4BACF;4BAEA,MAAMyB,oBAAoBzB,aAAa,iBAAiB;4BAGxD,IAAIyB,mBAAmB;gCACrB,MAAMC,UAAUC,6BACdF,mBACA9B,iBAAkB,MAAM,EACxB2B,aAAa,eAAe;gCAE9BtB,aAAa,iBAAiB,GAAG0B;gCAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;oCAC7D7G,MACE,0DACA;wCACE,iBAAiB6G,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;wCAEZ,cAAcjC,iBAAkB,MAAM;wCACtC,mBAAmB+B,QAAQ,iBAAiB,CAAC,OAAO,CAAC;wCACrD,aAAaA,QAAQ,WAAW;oCAClC;oCAEF/B,mBAAmB;wCACjB,GAAGA,gBAAgB;wCACnB,QAAQ;4CACNwB,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;4CACrCP,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;yCACtC;oCACH;gCACF;4BACF;wBACF,OAAO;4BACL,MAAMJ,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;4BAC/B3B,mBAAmB7B;4BACnBjD,MACE,sEACA;gCACE,eAAeyG,aAAa,eAAe,CAAC,OAAO,CAAC;gCACpD,mBAAmBA,aAAa,iBAAiB;4BACnD;wBAEJ;oBACF;gBACF;gBAEA,IAAI,CAACjC,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMgB,eAAeR;oBACrB,MAAMqC,iBAAiB7B,cAAc;oBAIrC,IACE6B,kBACA1B,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACApG,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAIyE,cAAc;4BAChBhC,sBAAsBgC;4BACtBlH,MACE;wBAEJ;oBACF,EAAE,OAAOmH,aAAa;wBACpBnH,MAAM,kCAAkCmH;oBAC1C;oBAGF,IAAI,CAACjC,qBACH,IAAI;wBACFxC,mBAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BoB,sBAAsBpB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiBiE,cACnBrD,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,mBAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAM6E,UACJpD,mBACAI,oBACAS,oBACAI;gBAGF,MAAMoC,2BAA2BpH,iBAC/BwE,mBAAmB,cAAc;gBAGnC,IAAI6C;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACb,EAACrC,cAAcC,aAAY,KAC3B,EAACd,aAAa,CAACmD,wBAAuB,KACvChH,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIkH,gBAAkCH,QAAQ,MAAM;oBACpD,IAAIrE,AAA6B,MAA7BA,0BAAgC;wBAClCwE,gBAAgB;4BACdlB,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;4BAC/BsD,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;yBAChC;wBACDhD,MACE,8DACAqH,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOlH,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiBuH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG3B;wBAC1B2B,QAAQ,iBAAiB,GACvBd,+BAA+Ba;wBACjCxH,MACE,uCACAyE,aACAgD;wBAGF,MAAMC,uBACJpC,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC;wBACpD,IAAIS,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDH,eACA5G,6BACA6B;4BAEF,IAAIkF,QAAQ;gCACVF,QAAQ,cAAc,GAAGE;gCACzB3H,MACE,4CACAyE;4BAEJ;wBACF,EAAE,OAAO0C,aAAa;4BACpBnH,MAAM,kCAAkCmH;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQhD;4BACR,OAAOgD;wBACT,GACA/C;oBAEJ,OACE1E,MACE,yDACAyE;gBAGN,EAAE,OAAOtB,OAAO;oBACdnD,MAAM,mCAAmCmD;gBAC3C;qBAEAnD,MAAM;gBAIV,IAAI,CAACqH,SAAS;oBACZ,IAAIxD,YACF,MAAM,IAAIuD,aACR,CAAC,oBAAoB,EAAE9G,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIsH;gBAEJ,IAAIzD,WACFyD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMtH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACToD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOtH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACT4C,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPjD;wBACA,aAAa4C;oBACf;gBACF;gBAGF/D,WAAW6D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK5E,UAAU,aAAa;wBAC9B;oBACF;oBACAmF;gBACF;YACF;QACF;QAEA,OAAOhE;IACT;IArxBA,YAAY,EACViE,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTtG,WAAW,EACXuG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGtG;QACnB,IAAI,CAAC,eAAe,GAAGuG;IACzB;AA0wBF"}

package/dist/es/agent/utils.mjs

@@ -152,7 +152,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.6.9";
+const getMidsceneVersion = ()=>"1.6.11";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/assert.mjs

@@ -23,7 +23,7 @@ ${systemCheckPrompt}`;
     prompt += `
 
 ## 输出要求
-请以 JSON 格式输出断言结果：
+请严格以 标准JSON 格式输出断言结果,不要携带md的标签:
 {
   "pass": boolean,    // 断言是否通过
   "thought": string,  // 思考过程
@@ -46,7 +46,7 @@ async function AiAssertElement(options) {
 1. 分析操作前后的截图变化（如果提供了操作前截图）
 2. 验证用户的断言描述是否成立
 3. 检查是否存在系统级问题（如果启用）
-4. 给出详细的思考过程和判断结果`;
+4. 给出详细的思考过程和判断结果（思考不要太久）`;
     const systemCheckPrompt = enableSystemCheck ? customSystemCheckRules || DEFAULT_SYSTEM_CHECK_PROMPT : '';
     const userContent = [];
     if (beforeScreenshot) {

package/dist/es/ai-model/assert.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/assert.mjs","sources":["../../../src/ai-model/assert.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  DiffDetails,\n  IgnoreRegion,\n  ReferenceImage,\n  SystemCheckResults,\n  VideoAssertionOptions,\n  VideoDetails,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('ai:assert');\n\nexport interface AiAssertOptions {\n  beforeScreenshot?: string;\n  afterScreenshot: string;\n  assertion: string;\n  businessContext?: string;\n  enableSystemCheck?: boolean;\n  customSystemCheckRules?: string;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n  referenceImages?: ReferenceImage[];\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  systemCheckResults?: SystemCheckResults;\n  diffDetails?: DiffDetails;\n}\n\nconst DEFAULT_SYSTEM_CHECK_PROMPT = `请检查以下系统级问题：\n\n1. **白屏检测**：页面是否完全白屏或大面积空白\n4. **错误提示**：页面是否显示错误信息、异常提示\n5. **后端错误**：是否有后端请求失败的提示，如\"活动太火爆了\"、报错 Toast、错误码 等\n\n如果发现以上问题，请在 systemCheckResults 中标注对应字段为 true。`;\n\nfunction buildAssertionPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  systemCheckPrompt?: string;\n}): string {\n  const { assertion, businessContext, systemCheckPrompt } = options;\n\n  let prompt = `## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  if (systemCheckPrompt) {\n    prompt += `\n\n## 系统校验规则\n${systemCheckPrompt}`;\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出断言结果：\n{\n  \"pass\": boolean,    // 断言是否通过\n  \"thought\": string,  // 思考过程\n  \"reason\": string,   // 失败原因（如果失败）\n  \"systemCheckResults\": {  // 系统校验结果（可选）\n    \"whiteScreen\": boolean,\n    \"layoutBlocked\": boolean,\n    \"loadingContent\": boolean,\n    \"errorPrompt\": boolean,\n    \"backendError\": boolean\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertElement(options: AiAssertOptions): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  usage?: AIUsageInfo;\n  systemCheckResults?: SystemCheckResults;\n  rawResponse?: string;\n}> {\n  const {\n    beforeScreenshot,\n    afterScreenshot,\n    assertion,\n    businessContext,\n    enableSystemCheck = false,\n    customSystemCheckRules,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const systemPrompt = `你是一个自动化测试断言专家。请根据以下信息判断断言是否通过。\n\n你需要：\n1. 分析操作前后的截图变化（如果提供了操作前截图）\n2. 验证用户的断言描述是否成立\n3. 检查是否存在系统级问题（如果启用）\n4. 给出详细的思考过程和判断结果`;\n\n  const systemCheckPrompt = enableSystemCheck\n    ? customSystemCheckRules || DEFAULT_SYSTEM_CHECK_PROMPT\n    : '';\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (beforeScreenshot) {\n    userContent.push({\n      type: 'text',\n      text: '## 操作前截图（执行操作前的页面状态）',\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: beforeScreenshot, detail: 'high' },\n    });\n    userContent.push({\n      type: 'text',\n      text: '## 操作后截图（执行操作后的页面状态）',\n    });\n  } else {\n    userContent.push({\n      type: 'text',\n      text: '## 当前页面截图',\n    });\n  }\n\n  userContent.push({\n    type: 'image_url',\n    image_url: { url: afterScreenshot, detail: 'high' },\n  });\n\n  userContent.push({\n    type: 'text',\n    text: buildAssertionPrompt({\n      assertion,\n      businessContext,\n      systemCheckPrompt,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: systemPrompt },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<AIAssertionResponse>(\n      msgs,\n      modelConfig,\n      { abortSignal },\n    );\n\n    debug('assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      usage: result.usage,\n      systemCheckResults: result.content.systemCheckResults,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('assertion error:', error);\n    throw error;\n  }\n}\n\nconst DIFF_SYSTEM_PROMPT = `你是一个自动化测试的图像对比专家。你的任务是对比当前页面截图与基准图片，判断页面样式是否符合预期。\n\n你需要：\n1. 仔细对比两张图片的布局、颜色、元素位置\n2. 分析差异是否在可接受范围内\n3. 识别动态内容（如时间、日期）导致的差异\n4. 给出详细的对比分析和判断结果`;\n\nfunction buildDiffPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n}): string {\n  const {\n    assertion,\n    businessContext,\n    diffThreshold = 0.1,\n    ignoreRegions,\n    ignoreDynamicContent,\n    strictMode,\n  } = options;\n\n  let prompt = `## 图像对比任务\n\n请对比基准图片（预期样式）和当前截图（实际样式），判断页面是否符合预期。\n\n## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  prompt += `\n\n## 对比要求\n\n请从以下维度进行对比分析：\n\n1. **布局一致性**：页面布局是否与基准图片一致\n2. **颜色一致性**：主要颜色是否与基准图片一致\n3. **元素位置**：关键元素的位置是否与基准图片一致\n4. **文字内容**：文字内容是否与基准图片一致\n5. **图片资源**：图片、图标等资源是否正确加载\n\n## 差异阈值\n允许的差异阈值：${diffThreshold * 100}%`;\n\n  if (ignoreRegions && ignoreRegions.length > 0) {\n    prompt += `\n\n## 忽略对比的区域\n以下区域不参与对比：`;\n    ignoreRegions.forEach((region, index) => {\n      prompt += `\n${index + 1}. 位置 (${region.x}, ${region.y})，尺寸 ${region.width}x${region.height}`;\n    });\n  }\n\n  if (ignoreDynamicContent) {\n    prompt += `\n\n## 动态内容处理\n请忽略以下动态内容导致的差异：\n- 时间显示\n- 日期显示\n- 随机验证码\n- 动画效果`;\n  }\n\n  if (strictMode) {\n    prompt += `\n\n## 严格模式\n当前为严格模式，任何差异都视为失败。`;\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出对比结果：\n{\n  \"pass\": boolean,           // 断言是否通过\n  \"thought\": string,         // 对比分析过程\n  \"reason\": string,          // 失败原因（如果失败）\n  \"diffDetails\": {           // 差异详情\n    \"layoutMatch\": boolean,      // 布局是否匹配\n    \"colorMatch\": boolean,       // 颜色是否匹配\n    \"elementPositionMatch\": boolean, // 元素位置是否匹配\n    \"textContentMatch\": boolean, // 文字内容是否匹配\n    \"resourceMatch\": boolean,    // 资源是否匹配\n    \"acceptableDifferences\": string[], // 可接受的差异列表\n    \"unacceptableDifferences\": string[] // 不可接受的差异列表\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertDiff(options: {\n  currentScreenshot: string;\n  referenceImages: ReferenceImage[];\n  assertion: string;\n  businessContext?: string;\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  diffDetails?: DiffDetails;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const {\n    currentScreenshot,\n    referenceImages,\n    assertion,\n    businessContext,\n    diffThreshold,\n    ignoreRegions,\n    ignoreDynamicContent,\n    strictMode,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  // 添加基准图片\n  for (const refImage of referenceImages) {\n    userContent.push({\n      type: 'text',\n      text: `## 基准图片：${refImage.name}`,\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: refImage.url, detail: 'high' },\n    });\n  }\n\n  // 添加当前截图\n  userContent.push({\n    type: 'text',\n    text: '## 当前截图（实际样式）',\n  });\n  userContent.push({\n    type: 'image_url',\n    image_url: { url: currentScreenshot, detail: 'high' },\n  });\n\n  // 添加对比提示\n  userContent.push({\n    type: 'text',\n    text: buildDiffPrompt({\n      assertion,\n      businessContext,\n      diffThreshold,\n      ignoreRegions,\n      ignoreDynamicContent,\n      strictMode,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: DIFF_SYSTEM_PROMPT },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for diff assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<AIAssertionResponse>(\n      msgs,\n      modelConfig,\n      { abortSignal },\n    );\n\n    debug('diff assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      diffDetails: result.content.diffDetails,\n      usage: result.usage,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('diff assertion error:', error);\n    throw error;\n  }\n}\n\nconst VIDEO_SYSTEM_PROMPT = `你是一个自动化测试的视频/动画分析专家。你的任务是对比当前录制的视频与基准视频，判断动画效果是否符合预期。\n\n你需要：\n1. 分析视频中的动画流畅度、时长、完整性\n2. 对比基准视频与当前视频的差异\n3. 识别动画类型（过渡动画、加载动画、交互动画等）\n4. 检测动画质量问题（卡顿、跳帧等）\n5. 给出详细的分析过程和判断结果`;\n\nfunction buildVideoPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  videoOptions?: VideoAssertionOptions;\n}): string {\n  const { assertion, businessContext, videoOptions } = options;\n\n  let prompt = `## 视频/动画对比任务\n\n请对比基准视频（预期动画效果）和当前录制的视频（实际动画效果），判断动画是否符合预期。\n\n## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  prompt += `\n\n## 分析要求\n\n请从以下维度进行动画分析：\n\n1. **动画流畅度**：动画是否流畅，有无卡顿、跳帧\n2. **动画时长**：动画时长是否在预期范围内\n3. **动画完整性**：动画是否完整执行，有无中断\n4. **动画类型**：识别动画类型（过渡、加载、交互等）\n5. **关键帧匹配**：关键时间点的画面是否符合预期`;\n\n  if (videoOptions) {\n    if (videoOptions.checkSmoothness) {\n      prompt += `\n\n## 流畅度检测\n流畅度阈值：${videoOptions.smoothnessThreshold || 60}/100\n请评估动画流畅度并给出评分。`;\n    }\n\n    if (videoOptions.checkDuration && videoOptions.expectedDuration) {\n      prompt += `\n\n## 时长检测\n预期时长范围：${videoOptions.expectedDuration.min || 0}秒 - ${videoOptions.expectedDuration.max || '无限制'}秒`;\n    }\n\n    if (\n      videoOptions.keyframes &&\n      videoOptions.keyframes.timestamps.length > 0\n    ) {\n      prompt += `\n\n## 关键帧验证\n需要在以下时间点验证画面：`;\n      videoOptions.keyframes.timestamps.forEach((ts, index) => {\n        const desc = videoOptions.keyframes?.descriptions?.[index] || '未描述';\n        prompt += `\n- ${ts}秒：${desc}`;\n      });\n    }\n\n    if (videoOptions.animationType && videoOptions.animationType !== 'auto') {\n      prompt += `\n\n## 动画类型\n预期动画类型：${videoOptions.animationType}`;\n    }\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出对比结果：\n{\n  \"pass\": boolean,           // 断言是否通过\n  \"thought\": string,         // 分析过程\n  \"reason\": string,          // 失败原因（如果失败）\n  \"videoDetails\": {          // 视频详情\n    \"smoothnessScore\": number,    // 流畅度评分（0-100）\n    \"duration\": number,           // 动画时长（秒）\n    \"isComplete\": boolean,        // 动画是否完整\n    \"keyframeMatches\": [          // 关键帧匹配结果\n      {\n        \"timestamp\": number,\n        \"matched\": boolean,\n        \"description\": string\n      }\n    ],\n    \"detectedAnimationType\": string,  // 检测到的动画类型\n    \"qualityAssessment\": {            // 质量评估\n      \"hasStuttering\": boolean,       // 是否有卡顿\n      \"hasFrameDropping\": boolean,    // 是否有跳帧\n      \"averageFrameInterval\": number, // 平均帧间隔（ms）\n      \"frameIntervalStdDev\": number   // 帧间隔标准差\n    },\n    \"acceptableDifferences\": string[], // 可接受的差异列表\n    \"unacceptableDifferences\": string[] // 不可接受的差异列表\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertVideo(options: {\n  currentVideoFrames: string[];\n  assertion: string;\n  businessContext?: string;\n  videoOptions?: VideoAssertionOptions;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  videoDetails?: VideoDetails;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const {\n    currentVideoFrames,\n    assertion,\n    businessContext,\n    videoOptions,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const MAX_DURATION = 5;\n  const DEFAULT_FPS = 30;\n  const MAX_FRAMES = MAX_DURATION * DEFAULT_FPS;\n\n  if (currentVideoFrames.length > MAX_FRAMES) {\n    throw new Error(\n      `Video frames exceed maximum limit. Maximum allowed: ${MAX_FRAMES} frames (${MAX_DURATION}s at ${DEFAULT_FPS}fps), got: ${currentVideoFrames.length} frames`,\n    );\n  }\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  userContent.push({\n    type: 'text',\n    text: `## 当前视频（共 ${currentVideoFrames.length} 帧，最大限制 ${MAX_FRAMES} 帧）\n\n以下为当前录制视频的关键帧截图：`,\n  });\n\n  const frameInterval = Math.max(1, Math.floor(currentVideoFrames.length / 10));\n  for (let i = 0; i < currentVideoFrames.length; i += frameInterval) {\n    userContent.push({\n      type: 'text',\n      text: `帧 ${i + 1}/${currentVideoFrames.length}`,\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: currentVideoFrames[i], detail: 'low' },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: buildVideoPrompt({\n      assertion,\n      businessContext,\n      videoOptions,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: VIDEO_SYSTEM_PROMPT },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for video assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<{\n      pass: boolean;\n      thought: string;\n      reason?: string;\n      videoDetails?: VideoDetails;\n    }>(msgs, modelConfig, { abortSignal });\n\n    debug('video assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      videoDetails: result.content.videoDetails,\n      usage: result.usage,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('video assertion error:', error);\n    throw error;\n  }\n}\n"],"names":["debug","getDebug","DEFAULT_SYSTEM_CHECK_PROMPT","buildAssertionPrompt","options","assertion","businessContext","systemCheckPrompt","prompt","AiAssertElement","beforeScreenshot","afterScreenshot","enableSystemCheck","customSystemCheckRules","modelConfig","abortSignal","systemPrompt","userContent","msgs","result","callAIWithObjectResponse","JSON","error","DIFF_SYSTEM_PROMPT","buildDiffPrompt","diffThreshold","ignoreRegions","ignoreDynamicContent","strictMode","region","index","AiAssertDiff","currentScreenshot","referenceImages","refImage","VIDEO_SYSTEM_PROMPT","buildVideoPrompt","videoOptions","ts","desc","AiAssertVideo","currentVideoFrames","MAX_DURATION","DEFAULT_FPS","MAX_FRAMES","Error","frameInterval","Math","i"],"mappings":";;AAcA,MAAMA,QAAQC,SAAS;AA0BvB,MAAMC,8BAA8B,CAAC;;;;;;6CAMQ,CAAC;AAE9C,SAASC,qBAAqBC,OAI7B;IACC,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAEC,iBAAiB,EAAE,GAAGH;IAE1D,IAAII,SAAS,CAAC;AAChB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjB,IAAIC,mBACFC,UAAU,CAAC;;;AAGf,EAAED,mBAAmB;IAGnBC,UAAU,CAAC;;;;;;;;;;;;;;;CAeZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAeC,gBAAgBL,OAAwB;IAQ5D,MAAM,EACJM,gBAAgB,EAChBC,eAAe,EACfN,SAAS,EACTC,eAAe,EACfM,oBAAoB,KAAK,EACzBC,sBAAsB,EACtBC,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMY,eAAe,CAAC;;;;;;iBAMP,CAAC;IAEhB,MAAMT,oBAAoBK,oBACtBC,0BAA0BX,8BAC1B;IAEJ,MAAMe,cAAyD,EAAE;IAEjE,IAAIP,kBAAkB;QACpBO,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QACAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKP;gBAAkB,QAAQ;YAAO;QACrD;QACAO,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;IACF,OACEA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM;IACR;IAGFA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YAAE,KAAKN;YAAiB,QAAQ;QAAO;IACpD;IAEAM,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMd,qBAAqB;YACzBE;YACAC;YACAC;QACF;IACF;IAEA,MAAMW,OAAO;QACX;YAAE,MAAM;YAAmB,SAASF;QAAa;QACjD;YAAE,MAAM;YAAiB,SAASC;QAAY;KAC/C;IAEDjB,MAAM,6BAA6BK;IAEnC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBACnBF,MACAJ,aACA;YAAEC;QAAY;QAGhBf,MAAM,qBAAqBmB,OAAO,OAAO;QAEzC,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,OAAOA,OAAO,KAAK;YACnB,oBAAoBA,OAAO,OAAO,CAAC,kBAAkB;YACrD,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,oBAAoBsB;QAC1B,MAAMA;IACR;AACF;AAEA,MAAMC,qBAAqB,CAAC;;;;;;iBAMX,CAAC;AAElB,SAASC,gBAAgBpB,OAOxB;IACC,MAAM,EACJC,SAAS,EACTC,eAAe,EACfmB,gBAAgB,GAAG,EACnBC,aAAa,EACbC,oBAAoB,EACpBC,UAAU,EACX,GAAGxB;IAEJ,IAAII,SAAS,CAAC;;;;;AAKhB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjBE,UAAU,CAAC;;;;;;;;;;;;;QAaL,EAAEiB,AAAgB,MAAhBA,cAAoB,CAAC,CAAC;IAE9B,IAAIC,iBAAiBA,cAAc,MAAM,GAAG,GAAG;QAC7ClB,UAAU,CAAC;;;UAGL,CAAC;QACPkB,cAAc,OAAO,CAAC,CAACG,QAAQC;YAC7BtB,UAAU,CAAC;AACjB,EAAEsB,QAAQ,EAAE,MAAM,EAAED,OAAO,CAAC,CAAC,EAAE,EAAEA,OAAO,CAAC,CAAC,KAAK,EAAEA,OAAO,KAAK,CAAC,CAAC,EAAEA,OAAO,MAAM,EAAE;QAC5E;IACF;IAEA,IAAIF,sBACFnB,UAAU,CAAC;;;;;;;MAOT,CAAC;IAGL,IAAIoB,YACFpB,UAAU,CAAC;;;kBAGG,CAAC;IAGjBA,UAAU,CAAC;;;;;;;;;;;;;;;;;CAiBZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAeuB,aAAa3B,OAWlC;IAQC,MAAM,EACJ4B,iBAAiB,EACjBC,eAAe,EACf5B,SAAS,EACTC,eAAe,EACfmB,aAAa,EACbC,aAAa,EACbC,oBAAoB,EACpBC,UAAU,EACVd,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMa,cAAyD,EAAE;IAGjE,KAAK,MAAMiB,YAAYD,gBAAiB;QACtChB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM,CAAC,QAAQ,EAAEiB,SAAS,IAAI,EAAE;QAClC;QACAjB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKiB,SAAS,GAAG;gBAAE,QAAQ;YAAO;QACjD;IACF;IAGAjB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM;IACR;IACAA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YAAE,KAAKe;YAAmB,QAAQ;QAAO;IACtD;IAGAf,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMO,gBAAgB;YACpBnB;YACAC;YACAmB;YACAC;YACAC;YACAC;QACF;IACF;IAEA,MAAMV,OAAO;QACX;YAAE,MAAM;YAAmB,SAASK;QAAmB;QACvD;YAAE,MAAM;YAAiB,SAASN;QAAY;KAC/C;IAEDjB,MAAM,kCAAkCK;IAExC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBACnBF,MACAJ,aACA;YAAEC;QAAY;QAGhBf,MAAM,0BAA0BmB,OAAO,OAAO;QAE9C,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,aAAaA,OAAO,OAAO,CAAC,WAAW;YACvC,OAAOA,OAAO,KAAK;YACnB,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,yBAAyBsB;QAC/B,MAAMA;IACR;AACF;AAEA,MAAMa,sBAAsB,CAAC;;;;;;;iBAOZ,CAAC;AAElB,SAASC,iBAAiBhC,OAIzB;IACC,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE+B,YAAY,EAAE,GAAGjC;IAErD,IAAII,SAAS,CAAC;;;;;AAKhB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjBE,UAAU,CAAC;;;;;;;;;;2BAUc,CAAC;IAE1B,IAAI6B,cAAc;QAChB,IAAIA,aAAa,eAAe,EAC9B7B,UAAU,CAAC;;;MAGX,EAAE6B,aAAa,mBAAmB,IAAI,GAAG;cACjC,CAAC;QAGX,IAAIA,aAAa,aAAa,IAAIA,aAAa,gBAAgB,EAC7D7B,UAAU,CAAC;;;OAGV,EAAE6B,aAAa,gBAAgB,CAAC,GAAG,IAAI,EAAE,IAAI,EAAEA,aAAa,gBAAgB,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC;QAG/F,IACEA,aAAa,SAAS,IACtBA,aAAa,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,GAC3C;YACA7B,UAAU,CAAC;;;aAGJ,CAAC;YACR6B,aAAa,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAACC,IAAIR;gBAC7C,MAAMS,OAAOF,aAAa,SAAS,EAAE,cAAc,CAACP,MAAM,IAAI;gBAC9DtB,UAAU,CAAC;EACjB,EAAE8B,GAAG,EAAE,EAAEC,MAAM;YACX;QACF;QAEA,IAAIF,aAAa,aAAa,IAAIA,AAA+B,WAA/BA,aAAa,aAAa,EAC1D7B,UAAU,CAAC;;;OAGV,EAAE6B,aAAa,aAAa,EAAE;IAEnC;IAEA7B,UAAU,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAegC,cAAcpC,OAOnC;IAQC,MAAM,EACJqC,kBAAkB,EAClBpC,SAAS,EACTC,eAAe,EACf+B,YAAY,EACZvB,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMsC,eAAe;IACrB,MAAMC,cAAc;IACpB,MAAMC,aAAaF,eAAeC;IAElC,IAAIF,mBAAmB,MAAM,GAAGG,YAC9B,MAAM,IAAIC,MACR,CAAC,oDAAoD,EAAED,WAAW,SAAS,EAAEF,aAAa,KAAK,EAAEC,YAAY,WAAW,EAAEF,mBAAmB,MAAM,CAAC,OAAO,CAAC;IAIhK,MAAMxB,cAAyD,EAAE;IAEjEA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM,CAAC,UAAU,EAAEwB,mBAAmB,MAAM,CAAC,QAAQ,EAAEG,WAAW;;gBAEtD,CAAC;IACf;IAEA,MAAME,gBAAgBC,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACN,mBAAmB,MAAM,GAAG;IACzE,IAAK,IAAIO,IAAI,GAAGA,IAAIP,mBAAmB,MAAM,EAAEO,KAAKF,cAAe;QACjE7B,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM,CAAC,EAAE,EAAE+B,IAAI,EAAE,CAAC,EAAEP,mBAAmB,MAAM,EAAE;QACjD;QACAxB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKwB,kBAAkB,CAACO,EAAE;gBAAE,QAAQ;YAAM;QACzD;IACF;IAEA/B,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMmB,iBAAiB;YACrB/B;YACAC;YACA+B;QACF;IACF;IAEA,MAAMnB,OAAO;QACX;YAAE,MAAM;YAAmB,SAASiB;QAAoB;QACxD;YAAE,MAAM;YAAiB,SAASlB;QAAY;KAC/C;IAEDjB,MAAM,mCAAmCK;IAEzC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBAKlBF,MAAMJ,aAAa;YAAEC;QAAY;QAEpCf,MAAM,2BAA2BmB,OAAO,OAAO;QAE/C,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,cAAcA,OAAO,OAAO,CAAC,YAAY;YACzC,OAAOA,OAAO,KAAK;YACnB,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,0BAA0BsB;QAChC,MAAMA;IACR;AACF"}
+{"version":3,"file":"ai-model/assert.mjs","sources":["../../../src/ai-model/assert.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  DiffDetails,\n  IgnoreRegion,\n  ReferenceImage,\n  SystemCheckResults,\n  VideoAssertionOptions,\n  VideoDetails,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('ai:assert');\n\nexport interface AiAssertOptions {\n  beforeScreenshot?: string;\n  afterScreenshot: string;\n  assertion: string;\n  businessContext?: string;\n  enableSystemCheck?: boolean;\n  customSystemCheckRules?: string;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n  referenceImages?: ReferenceImage[];\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  systemCheckResults?: SystemCheckResults;\n  diffDetails?: DiffDetails;\n}\n\nconst DEFAULT_SYSTEM_CHECK_PROMPT = `请检查以下系统级问题：\n\n1. **白屏检测**：页面是否完全白屏或大面积空白\n4. **错误提示**：页面是否显示错误信息、异常提示\n5. **后端错误**：是否有后端请求失败的提示，如\"活动太火爆了\"、报错 Toast、错误码 等\n\n如果发现以上问题，请在 systemCheckResults 中标注对应字段为 true。`;\n\nfunction buildAssertionPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  systemCheckPrompt?: string;\n}): string {\n  const { assertion, businessContext, systemCheckPrompt } = options;\n\n  let prompt = `## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  if (systemCheckPrompt) {\n    prompt += `\n\n## 系统校验规则\n${systemCheckPrompt}`;\n  }\n\n  prompt += `\n\n## 输出要求\n请严格以 标准JSON 格式输出断言结果,不要携带md的标签:\n{\n  \"pass\": boolean,    // 断言是否通过\n  \"thought\": string,  // 思考过程\n  \"reason\": string,   // 失败原因（如果失败）\n  \"systemCheckResults\": {  // 系统校验结果（可选）\n    \"whiteScreen\": boolean,\n    \"layoutBlocked\": boolean,\n    \"loadingContent\": boolean,\n    \"errorPrompt\": boolean,\n    \"backendError\": boolean\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertElement(options: AiAssertOptions): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  usage?: AIUsageInfo;\n  systemCheckResults?: SystemCheckResults;\n  rawResponse?: string;\n}> {\n  const {\n    beforeScreenshot,\n    afterScreenshot,\n    assertion,\n    businessContext,\n    enableSystemCheck = false,\n    customSystemCheckRules,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const systemPrompt = `你是一个自动化测试断言专家。请根据以下信息判断断言是否通过。\n\n你需要：\n1. 分析操作前后的截图变化（如果提供了操作前截图）\n2. 验证用户的断言描述是否成立\n3. 检查是否存在系统级问题（如果启用）\n4. 给出详细的思考过程和判断结果（思考不要太久）`;\n\n  const systemCheckPrompt = enableSystemCheck\n    ? customSystemCheckRules || DEFAULT_SYSTEM_CHECK_PROMPT\n    : '';\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (beforeScreenshot) {\n    userContent.push({\n      type: 'text',\n      text: '## 操作前截图（执行操作前的页面状态）',\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: beforeScreenshot, detail: 'high' },\n    });\n    userContent.push({\n      type: 'text',\n      text: '## 操作后截图（执行操作后的页面状态）',\n    });\n  } else {\n    userContent.push({\n      type: 'text',\n      text: '## 当前页面截图',\n    });\n  }\n\n  userContent.push({\n    type: 'image_url',\n    image_url: { url: afterScreenshot, detail: 'high' },\n  });\n\n  userContent.push({\n    type: 'text',\n    text: buildAssertionPrompt({\n      assertion,\n      businessContext,\n      systemCheckPrompt,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: systemPrompt },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<AIAssertionResponse>(\n      msgs,\n      modelConfig,\n      { abortSignal },\n    );\n\n    debug('assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      usage: result.usage,\n      systemCheckResults: result.content.systemCheckResults,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('assertion error:', error);\n    throw error;\n  }\n}\n\nconst DIFF_SYSTEM_PROMPT = `你是一个自动化测试的图像对比专家。你的任务是对比当前页面截图与基准图片，判断页面样式是否符合预期。\n\n你需要：\n1. 仔细对比两张图片的布局、颜色、元素位置\n2. 分析差异是否在可接受范围内\n3. 识别动态内容（如时间、日期）导致的差异\n4. 给出详细的对比分析和判断结果`;\n\nfunction buildDiffPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n}): string {\n  const {\n    assertion,\n    businessContext,\n    diffThreshold = 0.1,\n    ignoreRegions,\n    ignoreDynamicContent,\n    strictMode,\n  } = options;\n\n  let prompt = `## 图像对比任务\n\n请对比基准图片（预期样式）和当前截图（实际样式），判断页面是否符合预期。\n\n## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  prompt += `\n\n## 对比要求\n\n请从以下维度进行对比分析：\n\n1. **布局一致性**：页面布局是否与基准图片一致\n2. **颜色一致性**：主要颜色是否与基准图片一致\n3. **元素位置**：关键元素的位置是否与基准图片一致\n4. **文字内容**：文字内容是否与基准图片一致\n5. **图片资源**：图片、图标等资源是否正确加载\n\n## 差异阈值\n允许的差异阈值：${diffThreshold * 100}%`;\n\n  if (ignoreRegions && ignoreRegions.length > 0) {\n    prompt += `\n\n## 忽略对比的区域\n以下区域不参与对比：`;\n    ignoreRegions.forEach((region, index) => {\n      prompt += `\n${index + 1}. 位置 (${region.x}, ${region.y})，尺寸 ${region.width}x${region.height}`;\n    });\n  }\n\n  if (ignoreDynamicContent) {\n    prompt += `\n\n## 动态内容处理\n请忽略以下动态内容导致的差异：\n- 时间显示\n- 日期显示\n- 随机验证码\n- 动画效果`;\n  }\n\n  if (strictMode) {\n    prompt += `\n\n## 严格模式\n当前为严格模式，任何差异都视为失败。`;\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出对比结果：\n{\n  \"pass\": boolean,           // 断言是否通过\n  \"thought\": string,         // 对比分析过程\n  \"reason\": string,          // 失败原因（如果失败）\n  \"diffDetails\": {           // 差异详情\n    \"layoutMatch\": boolean,      // 布局是否匹配\n    \"colorMatch\": boolean,       // 颜色是否匹配\n    \"elementPositionMatch\": boolean, // 元素位置是否匹配\n    \"textContentMatch\": boolean, // 文字内容是否匹配\n    \"resourceMatch\": boolean,    // 资源是否匹配\n    \"acceptableDifferences\": string[], // 可接受的差异列表\n    \"unacceptableDifferences\": string[] // 不可接受的差异列表\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertDiff(options: {\n  currentScreenshot: string;\n  referenceImages: ReferenceImage[];\n  assertion: string;\n  businessContext?: string;\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  diffDetails?: DiffDetails;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const {\n    currentScreenshot,\n    referenceImages,\n    assertion,\n    businessContext,\n    diffThreshold,\n    ignoreRegions,\n    ignoreDynamicContent,\n    strictMode,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  // 添加基准图片\n  for (const refImage of referenceImages) {\n    userContent.push({\n      type: 'text',\n      text: `## 基准图片：${refImage.name}`,\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: refImage.url, detail: 'high' },\n    });\n  }\n\n  // 添加当前截图\n  userContent.push({\n    type: 'text',\n    text: '## 当前截图（实际样式）',\n  });\n  userContent.push({\n    type: 'image_url',\n    image_url: { url: currentScreenshot, detail: 'high' },\n  });\n\n  // 添加对比提示\n  userContent.push({\n    type: 'text',\n    text: buildDiffPrompt({\n      assertion,\n      businessContext,\n      diffThreshold,\n      ignoreRegions,\n      ignoreDynamicContent,\n      strictMode,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: DIFF_SYSTEM_PROMPT },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for diff assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<AIAssertionResponse>(\n      msgs,\n      modelConfig,\n      { abortSignal },\n    );\n\n    debug('diff assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      diffDetails: result.content.diffDetails,\n      usage: result.usage,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('diff assertion error:', error);\n    throw error;\n  }\n}\n\nconst VIDEO_SYSTEM_PROMPT = `你是一个自动化测试的视频/动画分析专家。你的任务是对比当前录制的视频与基准视频，判断动画效果是否符合预期。\n\n你需要：\n1. 分析视频中的动画流畅度、时长、完整性\n2. 对比基准视频与当前视频的差异\n3. 识别动画类型（过渡动画、加载动画、交互动画等）\n4. 检测动画质量问题（卡顿、跳帧等）\n5. 给出详细的分析过程和判断结果`;\n\nfunction buildVideoPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  videoOptions?: VideoAssertionOptions;\n}): string {\n  const { assertion, businessContext, videoOptions } = options;\n\n  let prompt = `## 视频/动画对比任务\n\n请对比基准视频（预期动画效果）和当前录制的视频（实际动画效果），判断动画是否符合预期。\n\n## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  prompt += `\n\n## 分析要求\n\n请从以下维度进行动画分析：\n\n1. **动画流畅度**：动画是否流畅，有无卡顿、跳帧\n2. **动画时长**：动画时长是否在预期范围内\n3. **动画完整性**：动画是否完整执行，有无中断\n4. **动画类型**：识别动画类型（过渡、加载、交互等）\n5. **关键帧匹配**：关键时间点的画面是否符合预期`;\n\n  if (videoOptions) {\n    if (videoOptions.checkSmoothness) {\n      prompt += `\n\n## 流畅度检测\n流畅度阈值：${videoOptions.smoothnessThreshold || 60}/100\n请评估动画流畅度并给出评分。`;\n    }\n\n    if (videoOptions.checkDuration && videoOptions.expectedDuration) {\n      prompt += `\n\n## 时长检测\n预期时长范围：${videoOptions.expectedDuration.min || 0}秒 - ${videoOptions.expectedDuration.max || '无限制'}秒`;\n    }\n\n    if (\n      videoOptions.keyframes &&\n      videoOptions.keyframes.timestamps.length > 0\n    ) {\n      prompt += `\n\n## 关键帧验证\n需要在以下时间点验证画面：`;\n      videoOptions.keyframes.timestamps.forEach((ts, index) => {\n        const desc = videoOptions.keyframes?.descriptions?.[index] || '未描述';\n        prompt += `\n- ${ts}秒：${desc}`;\n      });\n    }\n\n    if (videoOptions.animationType && videoOptions.animationType !== 'auto') {\n      prompt += `\n\n## 动画类型\n预期动画类型：${videoOptions.animationType}`;\n    }\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出对比结果：\n{\n  \"pass\": boolean,           // 断言是否通过\n  \"thought\": string,         // 分析过程\n  \"reason\": string,          // 失败原因（如果失败）\n  \"videoDetails\": {          // 视频详情\n    \"smoothnessScore\": number,    // 流畅度评分（0-100）\n    \"duration\": number,           // 动画时长（秒）\n    \"isComplete\": boolean,        // 动画是否完整\n    \"keyframeMatches\": [          // 关键帧匹配结果\n      {\n        \"timestamp\": number,\n        \"matched\": boolean,\n        \"description\": string\n      }\n    ],\n    \"detectedAnimationType\": string,  // 检测到的动画类型\n    \"qualityAssessment\": {            // 质量评估\n      \"hasStuttering\": boolean,       // 是否有卡顿\n      \"hasFrameDropping\": boolean,    // 是否有跳帧\n      \"averageFrameInterval\": number, // 平均帧间隔（ms）\n      \"frameIntervalStdDev\": number   // 帧间隔标准差\n    },\n    \"acceptableDifferences\": string[], // 可接受的差异列表\n    \"unacceptableDifferences\": string[] // 不可接受的差异列表\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertVideo(options: {\n  currentVideoFrames: string[];\n  assertion: string;\n  businessContext?: string;\n  videoOptions?: VideoAssertionOptions;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  videoDetails?: VideoDetails;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const {\n    currentVideoFrames,\n    assertion,\n    businessContext,\n    videoOptions,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const MAX_DURATION = 5;\n  const DEFAULT_FPS = 30;\n  const MAX_FRAMES = MAX_DURATION * DEFAULT_FPS;\n\n  if (currentVideoFrames.length > MAX_FRAMES) {\n    throw new Error(\n      `Video frames exceed maximum limit. Maximum allowed: ${MAX_FRAMES} frames (${MAX_DURATION}s at ${DEFAULT_FPS}fps), got: ${currentVideoFrames.length} frames`,\n    );\n  }\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  userContent.push({\n    type: 'text',\n    text: `## 当前视频（共 ${currentVideoFrames.length} 帧，最大限制 ${MAX_FRAMES} 帧）\n\n以下为当前录制视频的关键帧截图：`,\n  });\n\n  const frameInterval = Math.max(1, Math.floor(currentVideoFrames.length / 10));\n  for (let i = 0; i < currentVideoFrames.length; i += frameInterval) {\n    userContent.push({\n      type: 'text',\n      text: `帧 ${i + 1}/${currentVideoFrames.length}`,\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: currentVideoFrames[i], detail: 'low' },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: buildVideoPrompt({\n      assertion,\n      businessContext,\n      videoOptions,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: VIDEO_SYSTEM_PROMPT },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for video assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<{\n      pass: boolean;\n      thought: string;\n      reason?: string;\n      videoDetails?: VideoDetails;\n    }>(msgs, modelConfig, { abortSignal });\n\n    debug('video assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      videoDetails: result.content.videoDetails,\n      usage: result.usage,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('video assertion error:', error);\n    throw error;\n  }\n}\n"],"names":["debug","getDebug","DEFAULT_SYSTEM_CHECK_PROMPT","buildAssertionPrompt","options","assertion","businessContext","systemCheckPrompt","prompt","AiAssertElement","beforeScreenshot","afterScreenshot","enableSystemCheck","customSystemCheckRules","modelConfig","abortSignal","systemPrompt","userContent","msgs","result","callAIWithObjectResponse","JSON","error","DIFF_SYSTEM_PROMPT","buildDiffPrompt","diffThreshold","ignoreRegions","ignoreDynamicContent","strictMode","region","index","AiAssertDiff","currentScreenshot","referenceImages","refImage","VIDEO_SYSTEM_PROMPT","buildVideoPrompt","videoOptions","ts","desc","AiAssertVideo","currentVideoFrames","MAX_DURATION","DEFAULT_FPS","MAX_FRAMES","Error","frameInterval","Math","i"],"mappings":";;AAcA,MAAMA,QAAQC,SAAS;AA0BvB,MAAMC,8BAA8B,CAAC;;;;;;6CAMQ,CAAC;AAE9C,SAASC,qBAAqBC,OAI7B;IACC,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAEC,iBAAiB,EAAE,GAAGH;IAE1D,IAAII,SAAS,CAAC;AAChB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjB,IAAIC,mBACFC,UAAU,CAAC;;;AAGf,EAAED,mBAAmB;IAGnBC,UAAU,CAAC;;;;;;;;;;;;;;;CAeZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAeC,gBAAgBL,OAAwB;IAQ5D,MAAM,EACJM,gBAAgB,EAChBC,eAAe,EACfN,SAAS,EACTC,eAAe,EACfM,oBAAoB,KAAK,EACzBC,sBAAsB,EACtBC,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMY,eAAe,CAAC;;;;;;yBAMC,CAAC;IAExB,MAAMT,oBAAoBK,oBACtBC,0BAA0BX,8BAC1B;IAEJ,MAAMe,cAAyD,EAAE;IAEjE,IAAIP,kBAAkB;QACpBO,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QACAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKP;gBAAkB,QAAQ;YAAO;QACrD;QACAO,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;IACF,OACEA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM;IACR;IAGFA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YAAE,KAAKN;YAAiB,QAAQ;QAAO;IACpD;IAEAM,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMd,qBAAqB;YACzBE;YACAC;YACAC;QACF;IACF;IAEA,MAAMW,OAAO;QACX;YAAE,MAAM;YAAmB,SAASF;QAAa;QACjD;YAAE,MAAM;YAAiB,SAASC;QAAY;KAC/C;IAEDjB,MAAM,6BAA6BK;IAEnC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBACnBF,MACAJ,aACA;YAAEC;QAAY;QAGhBf,MAAM,qBAAqBmB,OAAO,OAAO;QAEzC,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,OAAOA,OAAO,KAAK;YACnB,oBAAoBA,OAAO,OAAO,CAAC,kBAAkB;YACrD,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,oBAAoBsB;QAC1B,MAAMA;IACR;AACF;AAEA,MAAMC,qBAAqB,CAAC;;;;;;iBAMX,CAAC;AAElB,SAASC,gBAAgBpB,OAOxB;IACC,MAAM,EACJC,SAAS,EACTC,eAAe,EACfmB,gBAAgB,GAAG,EACnBC,aAAa,EACbC,oBAAoB,EACpBC,UAAU,EACX,GAAGxB;IAEJ,IAAII,SAAS,CAAC;;;;;AAKhB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjBE,UAAU,CAAC;;;;;;;;;;;;;QAaL,EAAEiB,AAAgB,MAAhBA,cAAoB,CAAC,CAAC;IAE9B,IAAIC,iBAAiBA,cAAc,MAAM,GAAG,GAAG;QAC7ClB,UAAU,CAAC;;;UAGL,CAAC;QACPkB,cAAc,OAAO,CAAC,CAACG,QAAQC;YAC7BtB,UAAU,CAAC;AACjB,EAAEsB,QAAQ,EAAE,MAAM,EAAED,OAAO,CAAC,CAAC,EAAE,EAAEA,OAAO,CAAC,CAAC,KAAK,EAAEA,OAAO,KAAK,CAAC,CAAC,EAAEA,OAAO,MAAM,EAAE;QAC5E;IACF;IAEA,IAAIF,sBACFnB,UAAU,CAAC;;;;;;;MAOT,CAAC;IAGL,IAAIoB,YACFpB,UAAU,CAAC;;;kBAGG,CAAC;IAGjBA,UAAU,CAAC;;;;;;;;;;;;;;;;;CAiBZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAeuB,aAAa3B,OAWlC;IAQC,MAAM,EACJ4B,iBAAiB,EACjBC,eAAe,EACf5B,SAAS,EACTC,eAAe,EACfmB,aAAa,EACbC,aAAa,EACbC,oBAAoB,EACpBC,UAAU,EACVd,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMa,cAAyD,EAAE;IAGjE,KAAK,MAAMiB,YAAYD,gBAAiB;QACtChB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM,CAAC,QAAQ,EAAEiB,SAAS,IAAI,EAAE;QAClC;QACAjB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKiB,SAAS,GAAG;gBAAE,QAAQ;YAAO;QACjD;IACF;IAGAjB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM;IACR;IACAA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YAAE,KAAKe;YAAmB,QAAQ;QAAO;IACtD;IAGAf,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMO,gBAAgB;YACpBnB;YACAC;YACAmB;YACAC;YACAC;YACAC;QACF;IACF;IAEA,MAAMV,OAAO;QACX;YAAE,MAAM;YAAmB,SAASK;QAAmB;QACvD;YAAE,MAAM;YAAiB,SAASN;QAAY;KAC/C;IAEDjB,MAAM,kCAAkCK;IAExC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBACnBF,MACAJ,aACA;YAAEC;QAAY;QAGhBf,MAAM,0BAA0BmB,OAAO,OAAO;QAE9C,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,aAAaA,OAAO,OAAO,CAAC,WAAW;YACvC,OAAOA,OAAO,KAAK;YACnB,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,yBAAyBsB;QAC/B,MAAMA;IACR;AACF;AAEA,MAAMa,sBAAsB,CAAC;;;;;;;iBAOZ,CAAC;AAElB,SAASC,iBAAiBhC,OAIzB;IACC,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE+B,YAAY,EAAE,GAAGjC;IAErD,IAAII,SAAS,CAAC;;;;;AAKhB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjBE,UAAU,CAAC;;;;;;;;;;2BAUc,CAAC;IAE1B,IAAI6B,cAAc;QAChB,IAAIA,aAAa,eAAe,EAC9B7B,UAAU,CAAC;;;MAGX,EAAE6B,aAAa,mBAAmB,IAAI,GAAG;cACjC,CAAC;QAGX,IAAIA,aAAa,aAAa,IAAIA,aAAa,gBAAgB,EAC7D7B,UAAU,CAAC;;;OAGV,EAAE6B,aAAa,gBAAgB,CAAC,GAAG,IAAI,EAAE,IAAI,EAAEA,aAAa,gBAAgB,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC;QAG/F,IACEA,aAAa,SAAS,IACtBA,aAAa,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,GAC3C;YACA7B,UAAU,CAAC;;;aAGJ,CAAC;YACR6B,aAAa,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAACC,IAAIR;gBAC7C,MAAMS,OAAOF,aAAa,SAAS,EAAE,cAAc,CAACP,MAAM,IAAI;gBAC9DtB,UAAU,CAAC;EACjB,EAAE8B,GAAG,EAAE,EAAEC,MAAM;YACX;QACF;QAEA,IAAIF,aAAa,aAAa,IAAIA,AAA+B,WAA/BA,aAAa,aAAa,EAC1D7B,UAAU,CAAC;;;OAGV,EAAE6B,aAAa,aAAa,EAAE;IAEnC;IAEA7B,UAAU,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAegC,cAAcpC,OAOnC;IAQC,MAAM,EACJqC,kBAAkB,EAClBpC,SAAS,EACTC,eAAe,EACf+B,YAAY,EACZvB,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMsC,eAAe;IACrB,MAAMC,cAAc;IACpB,MAAMC,aAAaF,eAAeC;IAElC,IAAIF,mBAAmB,MAAM,GAAGG,YAC9B,MAAM,IAAIC,MACR,CAAC,oDAAoD,EAAED,WAAW,SAAS,EAAEF,aAAa,KAAK,EAAEC,YAAY,WAAW,EAAEF,mBAAmB,MAAM,CAAC,OAAO,CAAC;IAIhK,MAAMxB,cAAyD,EAAE;IAEjEA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM,CAAC,UAAU,EAAEwB,mBAAmB,MAAM,CAAC,QAAQ,EAAEG,WAAW;;gBAEtD,CAAC;IACf;IAEA,MAAME,gBAAgBC,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACN,mBAAmB,MAAM,GAAG;IACzE,IAAK,IAAIO,IAAI,GAAGA,IAAIP,mBAAmB,MAAM,EAAEO,KAAKF,cAAe;QACjE7B,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM,CAAC,EAAE,EAAE+B,IAAI,EAAE,CAAC,EAAEP,mBAAmB,MAAM,EAAE;QACjD;QACAxB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKwB,kBAAkB,CAACO,EAAE;gBAAE,QAAQ;YAAM;QACzD;IACF;IAEA/B,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMmB,iBAAiB;YACrB/B;YACAC;YACA+B;QACF;IACF;IAEA,MAAMnB,OAAO;QACX;YAAE,MAAM;YAAmB,SAASiB;QAAoB;QACxD;YAAE,MAAM;YAAiB,SAASlB;QAAY;KAC/C;IAEDjB,MAAM,mCAAmCK;IAEzC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBAKlBF,MAAMJ,aAAa;YAAEC;QAAY;QAEpCf,MAAM,2BAA2BmB,OAAO,OAAO;QAE/C,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,cAAcA,OAAO,OAAO,CAAC,YAAY;YACzC,OAAOA,OAAO,KAAK;YACnB,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,0BAA0BsB;QAChC,MAAMA;IACR;AACF"}

package/dist/es/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport {\n  existsSync,\n  mkdirSync,\n  readFileSync,\n  rmSync,\n  writeFileSync,\n} from 'node:fs';\nimport { join } from 'node:path';\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport { restoreImageReferences } from './dump/image-restoration';\nimport { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface CacheVerificationResult {\n  pass: boolean;\n  description?: string;\n  reason?: string;\n}\n\nexport interface CacheValidationOptions {\n  enableCoordOffsetCheck?: boolean;\n  coordOffsetThresholdPx?: number;\n  enableVisualVerify?: boolean;\n  enableSemanticAnchor?: boolean;\n}\n\nexport interface CacheConfidenceState {\n  lastVerifiedAt: number;\n  verificationCount: number;\n  confidenceScore: number;\n}\n\nexport type VerificationLevel = 'minimal' | 'standard' | 'enhanced' | 'full';\n\nexport interface ProgressiveLocateRecord {\n  convergedCenter: [number, number];\n  convergenceRadius: number;\n  sampleCount: number;\n  lastUpdatedAt: number;\n}\n\nexport interface Landmark {\n  xpath: string;\n  description: string;\n}\n\nexport interface SemanticAnchor {\n  visualFingerprint: string;\n  contextDescription: string;\n  nearbyLandmarks: Landmark[];\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  assertionReason?: string;\n  beforeScreenshot?: string;\n  systemCheckResults?: SystemCheckResults;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface ReferenceImage {\n  name: string;\n  url: string;\n}\n\nexport interface IgnoreRegion {\n  x: number;\n  y: number;\n  width: number;\n  height: number;\n}\n\nexport interface VideoInput {\n  /** 视频帧列表（Base64 编码的图片），最多 150 帧（5秒 * 30fps） */\n  frames: string[];\n  /** 视频时长（秒），最大 5 秒 */\n  duration?: number;\n  /** 帧率，默认 30 */\n  fps?: number;\n  /** 视频格式，默认 'frames' */\n  format?: 'frames' | 'base64' | 'url';\n  /** 视频 URL（当 format 为 'url' 时使用） */\n  url?: string;\n}\n\nexport interface VideoAssertionOptions {\n  /** 是否验证动画流畅度，默认 true */\n  checkSmoothness?: boolean;\n  /** 流畅度阈值（0-100），默认 60 */\n  smoothnessThreshold?: number;\n  /** 是否验证动画时长，默认 false */\n  checkDuration?: boolean;\n  /** 预期动画时长范围（秒） */\n  expectedDuration?: {\n    min?: number;\n    max?: number;\n  };\n  /** 是否验证动画完整性，默认 true */\n  checkCompleteness?: boolean;\n  /** 关键帧验证配置 */\n  keyframes?: {\n    /** 关键帧时间点（秒） */\n    timestamps: number[];\n    /** 关键帧描述 */\n    descriptions?: string[];\n  };\n  /** 是否忽略背景变化，默认 false */\n  ignoreBackgroundChanges?: boolean;\n  /** 动画类型，用于优化分析 */\n  animationType?: 'transition' | 'loading' | 'interaction' | 'auto';\n}\n\nexport interface VideoDetails {\n  /** 动画流畅度评分（0-100） */\n  smoothnessScore: number;\n  /** 动画时长（秒） */\n  duration: number;\n  /** 动画是否完整 */\n  isComplete: boolean;\n  /** 关键帧匹配结果 */\n  keyframeMatches?: {\n    timestamp: number;\n    matched: boolean;\n    description?: string;\n  }[];\n  /** 动画类型识别 */\n  detectedAnimationType: string;\n  /** 动画质量评估 */\n  qualityAssessment: {\n    /** 是否有卡顿 */\n    hasStuttering: boolean;\n    /** 是否有跳帧 */\n    hasFrameDropping: boolean;\n    /** 平均帧间隔（ms） */\n    averageFrameInterval: number;\n    /** 帧间隔标准差 */\n    frameIntervalStdDev: number;\n  };\n  /** 可接受的差异列表 */\n  acceptableDifferences: string[];\n  /** 不可接受的差异列表 */\n  unacceptableDifferences: string[];\n}\n\nexport interface RetryOptions {\n  maxRetries: number;\n  retryInterval: number;\n}\n\nexport interface DiffDetails {\n  layoutMatch: boolean;\n  colorMatch: boolean;\n  elementPositionMatch: boolean;\n  textContentMatch: boolean;\n  resourceMatch: boolean;\n  acceptableDifferences: string[];\n  unacceptableDifferences: string[];\n}\n\nexport interface AssertSnapshot {\n  beforeScreenshot: string;\n  afterScreenshot: string;\n  timestamp: number;\n  assertion: string;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n  /** 启用系统级校验（白屏、遮挡等），默认 true */\n  enableSystemCheck?: boolean;\n  /** 自定义系统校验规则 */\n  customSystemCheckRules?: string;\n  /** 业务知识上下文 */\n  businessContext?: string;\n  /** 手动传入操作前截图（Base64） */\n  beforeScreenshot?: string;\n  /** 手动传入操作后截图（Base64） */\n  afterScreenshot?: string;\n  /**\n   * 截图模式\n   * - 'auto': 自动判断（默认）\n   * - 'lastAction': 使用最后一次 action 的前后截图\n   * - 'flow': 使用整个流程的开始和结束截图\n   * - 'manual': 使用手动传入的截图\n   * - 'currentOnly': 仅使用当前截图（无操作前截图）\n   * - 'diff': 与基准图片进行 Diff 对比\n   * - 'video': 与基准视频进行动画对比\n   */\n  screenshotMode?:\n    | 'auto'\n    | 'lastAction'\n    | 'flow'\n    | 'manual'\n    | 'currentOnly'\n    | 'diff'\n    | 'video';\n  /** 基准图片列表（仅 Diff 模式） */\n  referenceImages?: ReferenceImage[];\n  /** Diff 差异阈值（0-1），默认 0.1 */\n  diffThreshold?: number;\n  /** 忽略对比的区域 */\n  ignoreRegions?: IgnoreRegion[];\n  /** 是否忽略动态内容，默认 false */\n  ignoreDynamicContent?: boolean;\n  /** 严格模式，任何差异都视为失败，默认 false */\n  strictMode?: boolean;\n  /** 视频断言配置（仅 Video 模式） */\n  videoOptions?: VideoAssertionOptions;\n  /** 当前视频输入（仅 Video 模式），用户传入的视频帧或视频 URL */\n  currentVideo?: VideoInput;\n  /** 断言重试配置 */\n  retryOptions?: RetryOptions;\n  /** 是否保存快照，默认 false */\n  saveSnapshot?: boolean;\n  /** 快照保存路径 */\n  snapshotPath?: string;\n}\n\nexport interface ActionScreenshotContext {\n  beforeScreenshot: string;\n  afterScreenshot: string;\n  actionType: string;\n  actionParam?: any;\n  timestamp: number;\n}\n\nexport interface SystemCheckResults {\n  whiteScreen?: boolean;\n  layoutBlocked?: boolean;\n  loadingContent?: boolean;\n  errorPrompt?: boolean;\n  backendError?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/**\n * Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem\n */\nfunction replacerForDumpSerialization(_key: string, value: any): any {\n  if (value && value.constructor?.name === 'Page') {\n    return '[Page object]';\n  }\n  if (value && value.constructor?.name === 'Browser') {\n    return '[Browser object]';\n  }\n  // Handle ScreenshotItem serialization\n  if (value && typeof value.toSerializable === 'function') {\n    return value.toSerializable();\n  }\n  return value;\n}\n\n/**\n * Reviver function for JSON deserialization that handles ScreenshotItem formats.\n *\n * BEHAVIOR:\n * - For { $screenshot: \"id\" } format: Left as-is (plain object)\n *   Consumer must use imageMap to restore base64 data\n * - For { base64: \"...\" } format: Creates ScreenshotItem from base64 data\n *\n * @param key - JSON key being processed\n * @param value - JSON value being processed\n * @returns Restored value\n */\nfunction reviverForDumpDeserialization(key: string, value: any): any {\n  // Only process screenshot fields\n  if (key !== 'screenshot' || typeof value !== 'object' || value === null) {\n    return value;\n  }\n\n  // Handle serialized format: { $screenshot: \"id\" }\n  // Leave as plain object — consumer uses imageMap to restore\n  if (ScreenshotItem.isSerialized(value)) {\n    return value;\n  }\n\n  // Handle inline base64 format: { base64: \"...\" }\n  if ('base64' in value && typeof value.base64 === 'string') {\n    return value;\n  }\n\n  return value;\n}\n\n/**\n * ExecutionDump class for serializing and deserializing execution dumps\n */\nexport class ExecutionDump implements IExecutionDump {\n  logTime: number;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n\n  constructor(data: IExecutionDump) {\n    this.logTime = data.logTime;\n    this.name = data.name;\n    this.description = data.description;\n    this.tasks = data.tasks;\n    this.aiActContext = data.aiActContext;\n  }\n\n  /**\n   * Serialize the ExecutionDump to a JSON string\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IExecutionDump {\n    return {\n      logTime: this.logTime,\n      name: this.name,\n      description: this.description,\n      tasks: this.tasks.map((task) => ({\n        ...task,\n        recorder: task.recorder || [],\n      })),\n      aiActContext: this.aiActContext,\n    };\n  }\n\n  /**\n   * Create an ExecutionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): ExecutionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IExecutionDump;\n    return new ExecutionDump(parsed);\n  }\n\n  /**\n   * Create an ExecutionDump instance from a plain object\n   */\n  static fromJSON(data: IExecutionDump): ExecutionDump {\n    return new ExecutionDump(data);\n  }\n\n  /**\n   * Collect all ScreenshotItem instances from tasks.\n   * Scans through uiContext and recorder items to find screenshots.\n   *\n   * @returns Array of ScreenshotItem instances\n   */\n  collectScreenshots(): ScreenshotItem[] {\n    const screenshots: ScreenshotItem[] = [];\n\n    for (const task of this.tasks) {\n      // Collect uiContext.screenshot if present\n      if (task.uiContext?.screenshot instanceof ScreenshotItem) {\n        screenshots.push(task.uiContext.screenshot);\n      }\n\n      // Collect recorder screenshots\n      if (task.recorder) {\n        for (const record of task.recorder) {\n          if (record.screenshot instanceof ScreenshotItem) {\n            screenshots.push(record.screenshot);\n          }\n        }\n      }\n    }\n\n    return screenshots;\n  }\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    aiActContext?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nGrouped dump\n*/\nexport interface IGroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n/**\n * GroupedActionDump class for serializing and deserializing grouped action dumps\n */\nexport class GroupedActionDump implements IGroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n  deviceType?: string;\n\n  constructor(data: IGroupedActionDump) {\n    this.sdkVersion = data.sdkVersion;\n    this.groupName = data.groupName;\n    this.groupDescription = data.groupDescription;\n    this.modelBriefs = data.modelBriefs;\n    this.executions = data.executions.map((exec) =>\n      exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec),\n    );\n    this.deviceType = data.deviceType;\n  }\n\n  /**\n   * Serialize the GroupedActionDump to a JSON string\n   * Uses compact { $screenshot: id } format\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Serialize the GroupedActionDump with inline screenshots to a JSON string.\n   * Each ScreenshotItem is replaced with { base64: \"...\", capturedAt }.\n   */\n  serializeWithInlineScreenshots(indents?: number): string {\n    const processValue = (obj: unknown): unknown => {\n      if (obj instanceof ScreenshotItem) {\n        return { base64: obj.base64, capturedAt: obj.capturedAt };\n      }\n      if (Array.isArray(obj)) {\n        return obj.map(processValue);\n      }\n      if (obj && typeof obj === 'object') {\n        const entries = Object.entries(obj).map(([key, value]) => [\n          key,\n          processValue(value),\n        ]);\n        return Object.fromEntries(entries);\n      }\n      return obj;\n    };\n\n    const data = processValue(this.toJSON());\n    return JSON.stringify(data, null, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IGroupedActionDump {\n    return {\n      sdkVersion: this.sdkVersion,\n      groupName: this.groupName,\n      groupDescription: this.groupDescription,\n      modelBriefs: this.modelBriefs,\n      executions: this.executions.map((exec) => exec.toJSON()),\n      deviceType: this.deviceType,\n    };\n  }\n\n  /**\n   * Create a GroupedActionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): GroupedActionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IGroupedActionDump;\n    return new GroupedActionDump(parsed);\n  }\n\n  /**\n   * Create a GroupedActionDump instance from a plain object\n   */\n  static fromJSON(data: IGroupedActionDump): GroupedActionDump {\n    return new GroupedActionDump(data);\n  }\n\n  /**\n   * Collect all ScreenshotItem instances from all executions.\n   *\n   * @returns Array of all ScreenshotItem instances across all executions\n   */\n  collectAllScreenshots(): ScreenshotItem[] {\n    const screenshots: ScreenshotItem[] = [];\n    for (const execution of this.executions) {\n      screenshots.push(...execution.collectScreenshots());\n    }\n    return screenshots;\n  }\n\n  /**\n   * Serialize the dump to files with screenshots as separate PNG files.\n   * Creates:\n   * - {basePath} - dump JSON with { $screenshot: id } references\n   * - {basePath}.screenshots/ - PNG files\n   * - {basePath}.screenshots.json - ID to path mapping\n   *\n   * @param basePath - Base path for the dump file\n   */\n  serializeToFiles(basePath: string): void {\n    const screenshotsDir = `${basePath}.screenshots`;\n    if (!existsSync(screenshotsDir)) {\n      mkdirSync(screenshotsDir, { recursive: true });\n    }\n\n    // Write screenshots to separate files\n    const screenshotMap: Record<string, string> = {};\n    const screenshots = this.collectAllScreenshots();\n\n    for (const screenshot of screenshots) {\n      if (screenshot.hasBase64()) {\n        const imagePath = join(\n          screenshotsDir,\n          `${screenshot.id}.${screenshot.extension}`,\n        );\n        const rawBase64 = screenshot.rawBase64;\n        writeFileSync(imagePath, Buffer.from(rawBase64, 'base64'));\n        screenshotMap[screenshot.id] = imagePath;\n      }\n    }\n\n    // Write screenshot map file\n    writeFileSync(\n      `${basePath}.screenshots.json`,\n      JSON.stringify(screenshotMap),\n      'utf-8',\n    );\n\n    // Write dump JSON with references\n    writeFileSync(basePath, this.serialize(), 'utf-8');\n  }\n\n  /**\n   * Read dump from files and return JSON string with inline screenshots.\n   * Reads the dump JSON and screenshot files, then inlines the base64 data.\n   *\n   * @param basePath - Base path for the dump file\n   * @returns JSON string with inline screenshots ({ base64: \"...\" } format)\n   */\n  static fromFilesAsInlineJson(basePath: string): string {\n    const dumpString = readFileSync(basePath, 'utf-8');\n    const screenshotsMapPath = `${basePath}.screenshots.json`;\n\n    if (!existsSync(screenshotsMapPath)) {\n      return dumpString;\n    }\n\n    // Read screenshot map and build imageMap from files\n    const screenshotMap: Record<string, string> = JSON.parse(\n      readFileSync(screenshotsMapPath, 'utf-8'),\n    );\n\n    const imageMap: Record<string, string> = {};\n    for (const [id, filePath] of Object.entries(screenshotMap)) {\n      if (existsSync(filePath)) {\n        const data = readFileSync(filePath);\n        const mime =\n          filePath.endsWith('.jpeg') || filePath.endsWith('.jpg')\n            ? 'jpeg'\n            : 'png';\n        imageMap[id] = `data:image/${mime};base64,${data.toString('base64')}`;\n      }\n    }\n\n    // Restore image references\n    const dumpData = JSON.parse(dumpString);\n    const processedData = restoreImageReferences(\n      dumpData,\n      (id) => imageMap[id] ?? '',\n    );\n    return JSON.stringify(processedData);\n  }\n\n  /**\n   * Clean up all files associated with a serialized dump.\n   *\n   * @param basePath - Base path for the dump file\n   */\n  static cleanupFiles(basePath: string): void {\n    const filesToClean = [\n      basePath,\n      `${basePath}.screenshots.json`,\n      `${basePath}.screenshots`,\n    ];\n\n    for (const filePath of filesToClean) {\n      try {\n        rmSync(filePath, { force: true, recursive: true });\n      } catch {\n        // Ignore errors - file may already be deleted\n      }\n    }\n  }\n\n  /**\n   * Get all file paths associated with a serialized dump.\n   *\n   * @param basePath - Base path for the dump file\n   * @returns Array of all associated file paths\n   */\n  static getFilePaths(basePath: string): string[] {\n    return [\n      basePath,\n      `${basePath}.screenshots.json`,\n      `${basePath}.screenshots`,\n    ];\n  }\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } will automatically get bbox injected when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n  cacheAdapter?: import('./agent/cache-adapter').CacheAdapter;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * When set to true, Midscene will use the target device's time (Android/iOS)\n   * instead of the system time. Useful when the device time differs from the\n   * host machine. Default: false\n   */\n  useDeviceTimestamp?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["UIContext","ServiceError","Error","message","dump","replacerForDumpSerialization","_key","value","reviverForDumpDeserialization","key","ScreenshotItem","ExecutionDump","indents","JSON","task","serialized","parsed","data","screenshots","record","GroupedActionDump","processValue","obj","Array","entries","Object","exec","execution","basePath","screenshotsDir","existsSync","mkdirSync","screenshotMap","screenshot","imagePath","join","rawBase64","writeFileSync","Buffer","dumpString","readFileSync","screenshotsMapPath","imageMap","id","filePath","mime","dumpData","processedData","restoreImageReferences","filesToClean","rmSync"],"mappings":";;;;;AAAqD;;;;;;;;;;AAsJ9C,MAAeA;AA4BtB;AA+EO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF;AA0YA,SAASC,6BAA6BC,IAAY,EAAEC,KAAU;IAC5D,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,QACvC,OAAO;IAET,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,WACvC,OAAO;IAGT,IAAIA,SAAS,AAAgC,cAAhC,OAAOA,MAAM,cAAc,EACtC,OAAOA,MAAM,cAAc;IAE7B,OAAOA;AACT;AAcA,SAASC,8BAA8BC,GAAW,EAAEF,KAAU;IAE5D,IAAIE,AAAQ,iBAARA,OAAwB,AAAiB,YAAjB,OAAOF,SAAsBA,AAAU,SAAVA,OACvD,OAAOA;IAKT,IAAIG,eAAe,YAAY,CAACH,QAC9B,OAAOA;IAIL,YAAYA,SAAgBA,MAAM,MAAM;IAI5C,OAAOA;AACT;AAKO,MAAMI;IAkBX,UAAUC,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAAyB;QACvB,OAAO;YACL,SAAS,IAAI,CAAC,OAAO;YACrB,MAAM,IAAI,CAAC,IAAI;YACf,aAAa,IAAI,CAAC,WAAW;YAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAACE,OAAU;oBAC/B,GAAGA,IAAI;oBACP,UAAUA,KAAK,QAAQ,IAAI,EAAE;gBAC/B;YACA,cAAc,IAAI,CAAC,YAAY;QACjC;IACF;IAKA,OAAO,qBAAqBC,UAAkB,EAAiB;QAC7D,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIG,cAAcK;IAC3B;IAKA,OAAO,SAASC,IAAoB,EAAiB;QACnD,OAAO,IAAIN,cAAcM;IAC3B;IAQA,qBAAuC;QACrC,MAAMC,cAAgC,EAAE;QAExC,KAAK,MAAMJ,QAAQ,IAAI,CAAC,KAAK,CAAE;YAE7B,IAAIA,KAAK,SAAS,EAAE,sBAAsBJ,gBACxCQ,YAAY,IAAI,CAACJ,KAAK,SAAS,CAAC,UAAU;YAI5C,IAAIA,KAAK,QAAQ,EACf;gBAAA,KAAK,MAAMK,UAAUL,KAAK,QAAQ,CAChC,IAAIK,OAAO,UAAU,YAAYT,gBAC/BQ,YAAY,IAAI,CAACC,OAAO,UAAU;YAEtC;QAEJ;QAEA,OAAOD;IACT;IA3EA,YAAYD,IAAoB,CAAE;QANlC;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,OAAO,GAAGA,KAAK,OAAO;QAC3B,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI;QACrB,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,YAAY,GAAGA,KAAK,YAAY;IACvC;AAsEF;AAwIO,MAAMG;IAuBX,UAAUR,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAMA,+BAA+BA,OAAgB,EAAU;QACvD,MAAMS,eAAe,CAACC;YACpB,IAAIA,eAAeZ,gBACjB,OAAO;gBAAE,QAAQY,IAAI,MAAM;gBAAE,YAAYA,IAAI,UAAU;YAAC;YAE1D,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAACD;YAEjB,IAAIC,OAAO,AAAe,YAAf,OAAOA,KAAkB;gBAClC,MAAME,UAAUC,OAAO,OAAO,CAACH,KAAK,GAAG,CAAC,CAAC,CAACb,KAAKF,MAAM,GAAK;wBACxDE;wBACAY,aAAad;qBACd;gBACD,OAAOkB,OAAO,WAAW,CAACD;YAC5B;YACA,OAAOF;QACT;QAEA,MAAML,OAAOI,aAAa,IAAI,CAAC,MAAM;QACrC,OAAOR,KAAK,SAAS,CAACI,MAAM,MAAML;IACpC;IAKA,SAA6B;QAC3B,OAAO;YACL,YAAY,IAAI,CAAC,UAAU;YAC3B,WAAW,IAAI,CAAC,SAAS;YACzB,kBAAkB,IAAI,CAAC,gBAAgB;YACvC,aAAa,IAAI,CAAC,WAAW;YAC7B,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAACc,OAASA,KAAK,MAAM;YACrD,YAAY,IAAI,CAAC,UAAU;QAC7B;IACF;IAKA,OAAO,qBAAqBX,UAAkB,EAAqB;QACjE,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIY,kBAAkBJ;IAC/B;IAKA,OAAO,SAASC,IAAwB,EAAqB;QAC3D,OAAO,IAAIG,kBAAkBH;IAC/B;IAOA,wBAA0C;QACxC,MAAMC,cAAgC,EAAE;QACxC,KAAK,MAAMS,aAAa,IAAI,CAAC,UAAU,CACrCT,YAAY,IAAI,IAAIS,UAAU,kBAAkB;QAElD,OAAOT;IACT;IAWA,iBAAiBU,QAAgB,EAAQ;QACvC,MAAMC,iBAAiB,GAAGD,SAAS,YAAY,CAAC;QAChD,IAAI,CAACE,WAAWD,iBACdE,UAAUF,gBAAgB;YAAE,WAAW;QAAK;QAI9C,MAAMG,gBAAwC,CAAC;QAC/C,MAAMd,cAAc,IAAI,CAAC,qBAAqB;QAE9C,KAAK,MAAMe,cAAcf,YACvB,IAAIe,WAAW,SAAS,IAAI;YAC1B,MAAMC,YAAYC,KAChBN,gBACA,GAAGI,WAAW,EAAE,CAAC,CAAC,EAAEA,WAAW,SAAS,EAAE;YAE5C,MAAMG,YAAYH,WAAW,SAAS;YACtCI,cAAcH,WAAWI,OAAO,IAAI,CAACF,WAAW;YAChDJ,aAAa,CAACC,WAAW,EAAE,CAAC,GAAGC;QACjC;QAIFG,cACE,GAAGT,SAAS,iBAAiB,CAAC,EAC9Bf,KAAK,SAAS,CAACmB,gBACf;QAIFK,cAAcT,UAAU,IAAI,CAAC,SAAS,IAAI;IAC5C;IASA,OAAO,sBAAsBA,QAAgB,EAAU;QACrD,MAAMW,aAAaC,aAAaZ,UAAU;QAC1C,MAAMa,qBAAqB,GAAGb,SAAS,iBAAiB,CAAC;QAEzD,IAAI,CAACE,WAAWW,qBACd,OAAOF;QAIT,MAAMP,gBAAwCnB,KAAK,KAAK,CACtD2B,aAAaC,oBAAoB;QAGnC,MAAMC,WAAmC,CAAC;QAC1C,KAAK,MAAM,CAACC,IAAIC,SAAS,IAAInB,OAAO,OAAO,CAACO,eAC1C,IAAIF,WAAWc,WAAW;YACxB,MAAM3B,OAAOuB,aAAaI;YAC1B,MAAMC,OACJD,SAAS,QAAQ,CAAC,YAAYA,SAAS,QAAQ,CAAC,UAC5C,SACA;YACNF,QAAQ,CAACC,GAAG,GAAG,CAAC,WAAW,EAAEE,KAAK,QAAQ,EAAE5B,KAAK,QAAQ,CAAC,WAAW;QACvE;QAIF,MAAM6B,WAAWjC,KAAK,KAAK,CAAC0B;QAC5B,MAAMQ,gBAAgBC,uBACpBF,UACA,CAACH,KAAOD,QAAQ,CAACC,GAAG,IAAI;QAE1B,OAAO9B,KAAK,SAAS,CAACkC;IACxB;IAOA,OAAO,aAAanB,QAAgB,EAAQ;QAC1C,MAAMqB,eAAe;YACnBrB;YACA,GAAGA,SAAS,iBAAiB,CAAC;YAC9B,GAAGA,SAAS,YAAY,CAAC;SAC1B;QAED,KAAK,MAAMgB,YAAYK,aACrB,IAAI;YACFC,OAAON,UAAU;gBAAE,OAAO;gBAAM,WAAW;YAAK;QAClD,EAAE,OAAM,CAER;IAEJ;IAQA,OAAO,aAAahB,QAAgB,EAAY;QAC9C,OAAO;YACLA;YACA,GAAGA,SAAS,iBAAiB,CAAC;YAC9B,GAAGA,SAAS,YAAY,CAAC;SAC1B;IACH;IA9MA,YAAYX,IAAwB,CAAE;QAPtC;QACA;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU;QACjC,IAAI,CAAC,SAAS,GAAGA,KAAK,SAAS;QAC/B,IAAI,CAAC,gBAAgB,GAAGA,KAAK,gBAAgB;QAC7C,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU,CAAC,GAAG,CAAC,CAACS,OACrCA,gBAAgBf,gBAAgBe,OAAOf,cAAc,QAAQ,CAACe;QAEhE,IAAI,CAAC,UAAU,GAAGT,KAAK,UAAU;IACnC;AAsMF"}
+{"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport {\n  existsSync,\n  mkdirSync,\n  readFileSync,\n  rmSync,\n  writeFileSync,\n} from 'node:fs';\nimport { join } from 'node:path';\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport { restoreImageReferences } from './dump/image-restoration';\nimport { ScreenshotItem } from './screenshot-item';\nimport type {\n  DetailedLocateParam,\n  MidsceneYamlFlowItem,\n  ServiceExtractOption,\n} from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n  request_id: string | undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface CacheVerificationResult {\n  pass: boolean;\n  description?: string;\n  reason?: string;\n}\n\nexport interface CacheValidationOptions {\n  enableCoordOffsetCheck?: boolean;\n  coordOffsetThresholdPx?: number;\n  enableVisualVerify?: boolean;\n  enableSemanticAnchor?: boolean;\n}\n\nexport interface CacheConfidenceState {\n  lastVerifiedAt: number;\n  verificationCount: number;\n  confidenceScore: number;\n}\n\nexport type VerificationLevel = 'minimal' | 'standard' | 'enhanced' | 'full';\n\nexport interface ProgressiveLocateRecord {\n  convergedCenter: [number, number];\n  convergenceRadius: number;\n  sampleCount: number;\n  lastUpdatedAt: number;\n}\n\nexport interface Landmark {\n  xpath: string;\n  description: string;\n}\n\nexport interface SemanticAnchor {\n  visualFingerprint: string;\n  contextDescription: string;\n  nearbyLandmarks: Landmark[];\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepLocate: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  /**\n   * screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n   */\n  abstract screenshot: ScreenshotItem;\n\n  /**\n   * screenshot size after shrinking\n   */\n  abstract shotSize: Size;\n\n  /**\n   * The ratio for converting shrunk screenshot coordinates to logical coordinates.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - User-defined screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n   * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n   */\n  abstract shrunkShotToLogicalRatio: number;\n\n  abstract _isFrozen?: boolean;\n\n  // @deprecated - backward compatibility for aiLocate\n  abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\n\nexport type DeepThinkOption = 'unset' | true | false;\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepLocate?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  assertionReason?: string;\n  beforeScreenshot?: string;\n  systemCheckResults?: SystemCheckResults;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface ReferenceImage {\n  name: string;\n  url: string;\n}\n\nexport interface IgnoreRegion {\n  x: number;\n  y: number;\n  width: number;\n  height: number;\n}\n\nexport interface VideoInput {\n  /** 视频帧列表（Base64 编码的图片），最多 150 帧（5秒 * 30fps） */\n  frames: string[];\n  /** 视频时长（秒），最大 5 秒 */\n  duration?: number;\n  /** 帧率，默认 30 */\n  fps?: number;\n  /** 视频格式，默认 'frames' */\n  format?: 'frames' | 'base64' | 'url';\n  /** 视频 URL（当 format 为 'url' 时使用） */\n  url?: string;\n}\n\nexport interface VideoAssertionOptions {\n  /** 是否验证动画流畅度，默认 true */\n  checkSmoothness?: boolean;\n  /** 流畅度阈值（0-100），默认 60 */\n  smoothnessThreshold?: number;\n  /** 是否验证动画时长，默认 false */\n  checkDuration?: boolean;\n  /** 预期动画时长范围（秒） */\n  expectedDuration?: {\n    min?: number;\n    max?: number;\n  };\n  /** 是否验证动画完整性，默认 true */\n  checkCompleteness?: boolean;\n  /** 关键帧验证配置 */\n  keyframes?: {\n    /** 关键帧时间点（秒） */\n    timestamps: number[];\n    /** 关键帧描述 */\n    descriptions?: string[];\n  };\n  /** 是否忽略背景变化，默认 false */\n  ignoreBackgroundChanges?: boolean;\n  /** 动画类型，用于优化分析 */\n  animationType?: 'transition' | 'loading' | 'interaction' | 'auto';\n}\n\nexport interface VideoDetails {\n  /** 动画流畅度评分（0-100） */\n  smoothnessScore: number;\n  /** 动画时长（秒） */\n  duration: number;\n  /** 动画是否完整 */\n  isComplete: boolean;\n  /** 关键帧匹配结果 */\n  keyframeMatches?: {\n    timestamp: number;\n    matched: boolean;\n    description?: string;\n  }[];\n  /** 动画类型识别 */\n  detectedAnimationType: string;\n  /** 动画质量评估 */\n  qualityAssessment: {\n    /** 是否有卡顿 */\n    hasStuttering: boolean;\n    /** 是否有跳帧 */\n    hasFrameDropping: boolean;\n    /** 平均帧间隔（ms） */\n    averageFrameInterval: number;\n    /** 帧间隔标准差 */\n    frameIntervalStdDev: number;\n  };\n  /** 可接受的差异列表 */\n  acceptableDifferences: string[];\n  /** 不可接受的差异列表 */\n  unacceptableDifferences: string[];\n}\n\nexport interface RetryOptions {\n  maxRetries: number;\n  retryInterval: number;\n}\n\nexport interface DiffDetails {\n  layoutMatch: boolean;\n  colorMatch: boolean;\n  elementPositionMatch: boolean;\n  textContentMatch: boolean;\n  resourceMatch: boolean;\n  acceptableDifferences: string[];\n  unacceptableDifferences: string[];\n}\n\nexport interface AssertSnapshot {\n  beforeScreenshot: string;\n  afterScreenshot: string;\n  timestamp: number;\n  assertion: string;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n  /** 启用系统级校验（白屏、遮挡等），默认 false */\n  enableSystemCheck?: boolean;\n  /** 自定义系统校验规则 */\n  customSystemCheckRules?: string;\n  /** 业务知识上下文 */\n  businessContext?: string;\n  /** 手动传入操作前截图（Base64） */\n  beforeScreenshot?: string;\n  /** 手动传入操作后截图（Base64） */\n  afterScreenshot?: string;\n  /**\n   * 截图模式\n   * - 'currentOnly': 仅使用当前截图（默认）\n   * - 'auto': 自动判断\n   * - 'lastAction': 使用最后一次 action 的前后截图\n   * - 'flow': 使用整个流程的开始和结束截图\n   * - 'manual': 使用手动传入的截图\n   * - 'diff': 与基准图片进行 Diff 对比\n   * - 'video': 与基准视频进行动画对比\n   */\n  screenshotMode?:\n    | 'auto'\n    | 'lastAction'\n    | 'flow'\n    | 'manual'\n    | 'currentOnly'\n    | 'diff'\n    | 'video';\n  /** 基准图片列表（仅 Diff 模式） */\n  referenceImages?: ReferenceImage[];\n  /** Diff 差异阈值（0-1），默认 0.1 */\n  diffThreshold?: number;\n  /** 忽略对比的区域 */\n  ignoreRegions?: IgnoreRegion[];\n  /** 是否忽略动态内容，默认 false */\n  ignoreDynamicContent?: boolean;\n  /** 严格模式，任何差异都视为失败，默认 false */\n  strictMode?: boolean;\n  /** 视频断言配置（仅 Video 模式） */\n  videoOptions?: VideoAssertionOptions;\n  /** 当前视频输入（仅 Video 模式），用户传入的视频帧或视频 URL */\n  currentVideo?: VideoInput;\n  /** 断言重试配置 */\n  retryOptions?: RetryOptions;\n  /** 是否保存快照，默认 false */\n  saveSnapshot?: boolean;\n  /** 快照保存路径 */\n  snapshotPath?: string;\n}\n\nexport interface ActionScreenshotContext {\n  beforeScreenshot: string;\n  afterScreenshot: string;\n  actionType: string;\n  actionParam?: any;\n  timestamp: number;\n}\n\nexport interface SystemCheckResults {\n  whiteScreen?: boolean;\n  layoutBlocked?: boolean;\n  loadingContent?: boolean;\n  errorPrompt?: boolean;\n  backendError?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  log?: string; // a brief preamble to the user explaining what you’re about to do\n  type: string;\n  param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' | 'running' | 'finished';\n\nexport interface SubGoal {\n  index: number;\n  status: SubGoalStatus;\n  description: string;\n  logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  thought?: string;\n  log: string;\n  memory?: string;\n  error?: string;\n  finalizeMessage?: string;\n  finalizeSuccess?: boolean;\n  updateSubGoals?: SubGoal[];\n  markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n  error?: string;\n  reasoning_content?: string;\n  shouldContinuePlanning: boolean;\n  output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: ScreenshotItem;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    taskId: string;\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      getUiContextStart?: number;\n      getUiContextEnd?: number;\n      callAiStart?: number;\n      callAiEnd?: number;\n      beforeInvokeActionHookStart?: number;\n      beforeInvokeActionHookEnd?: number;\n      callActionStart?: number;\n      callActionEnd?: number;\n      afterInvokeActionHookStart?: number;\n      afterInvokeActionHookEnd?: number;\n      captureAfterCallingSnapshotStart?: number;\n      captureAfterCallingSnapshotEnd?: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n    reasoning_content?: string;\n  };\n\nexport interface IExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n}\n\n/**\n * Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem\n */\nfunction replacerForDumpSerialization(_key: string, value: any): any {\n  if (value && value.constructor?.name === 'Page') {\n    return '[Page object]';\n  }\n  if (value && value.constructor?.name === 'Browser') {\n    return '[Browser object]';\n  }\n  // Handle ScreenshotItem serialization\n  if (value && typeof value.toSerializable === 'function') {\n    return value.toSerializable();\n  }\n  return value;\n}\n\n/**\n * Reviver function for JSON deserialization that handles ScreenshotItem formats.\n *\n * BEHAVIOR:\n * - For { $screenshot: \"id\" } format: Left as-is (plain object)\n *   Consumer must use imageMap to restore base64 data\n * - For { base64: \"...\" } format: Creates ScreenshotItem from base64 data\n *\n * @param key - JSON key being processed\n * @param value - JSON value being processed\n * @returns Restored value\n */\nfunction reviverForDumpDeserialization(key: string, value: any): any {\n  // Only process screenshot fields\n  if (key !== 'screenshot' || typeof value !== 'object' || value === null) {\n    return value;\n  }\n\n  // Handle serialized format: { $screenshot: \"id\" }\n  // Leave as plain object — consumer uses imageMap to restore\n  if (ScreenshotItem.isSerialized(value)) {\n    return value;\n  }\n\n  // Handle inline base64 format: { base64: \"...\" }\n  if ('base64' in value && typeof value.base64 === 'string') {\n    return value;\n  }\n\n  return value;\n}\n\n/**\n * ExecutionDump class for serializing and deserializing execution dumps\n */\nexport class ExecutionDump implements IExecutionDump {\n  logTime: number;\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActContext?: string;\n\n  constructor(data: IExecutionDump) {\n    this.logTime = data.logTime;\n    this.name = data.name;\n    this.description = data.description;\n    this.tasks = data.tasks;\n    this.aiActContext = data.aiActContext;\n  }\n\n  /**\n   * Serialize the ExecutionDump to a JSON string\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IExecutionDump {\n    return {\n      logTime: this.logTime,\n      name: this.name,\n      description: this.description,\n      tasks: this.tasks.map((task) => ({\n        ...task,\n        recorder: task.recorder || [],\n      })),\n      aiActContext: this.aiActContext,\n    };\n  }\n\n  /**\n   * Create an ExecutionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): ExecutionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IExecutionDump;\n    return new ExecutionDump(parsed);\n  }\n\n  /**\n   * Create an ExecutionDump instance from a plain object\n   */\n  static fromJSON(data: IExecutionDump): ExecutionDump {\n    return new ExecutionDump(data);\n  }\n\n  /**\n   * Collect all ScreenshotItem instances from tasks.\n   * Scans through uiContext and recorder items to find screenshots.\n   *\n   * @returns Array of ScreenshotItem instances\n   */\n  collectScreenshots(): ScreenshotItem[] {\n    const screenshots: ScreenshotItem[] = [];\n\n    for (const task of this.tasks) {\n      // Collect uiContext.screenshot if present\n      if (task.uiContext?.screenshot instanceof ScreenshotItem) {\n        screenshots.push(task.uiContext.screenshot);\n      }\n\n      // Collect recorder screenshots\n      if (task.recorder) {\n        for (const record of task.recorder) {\n          if (record.screenshot instanceof ScreenshotItem) {\n            screenshots.push(record.screenshot);\n          }\n        }\n      }\n    }\n\n    return screenshots;\n  }\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    aiActContext?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nGrouped dump\n*/\nexport interface IGroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: IExecutionDump[];\n  deviceType?: string;\n}\n\n/**\n * GroupedActionDump class for serializing and deserializing grouped action dumps\n */\nexport class GroupedActionDump implements IGroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n  deviceType?: string;\n\n  constructor(data: IGroupedActionDump) {\n    this.sdkVersion = data.sdkVersion;\n    this.groupName = data.groupName;\n    this.groupDescription = data.groupDescription;\n    this.modelBriefs = data.modelBriefs;\n    this.executions = data.executions.map((exec) =>\n      exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec),\n    );\n    this.deviceType = data.deviceType;\n  }\n\n  /**\n   * Serialize the GroupedActionDump to a JSON string\n   * Uses compact { $screenshot: id } format\n   */\n  serialize(indents?: number): string {\n    return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n  }\n\n  /**\n   * Serialize the GroupedActionDump with inline screenshots to a JSON string.\n   * Each ScreenshotItem is replaced with { base64: \"...\", capturedAt }.\n   */\n  serializeWithInlineScreenshots(indents?: number): string {\n    const processValue = (obj: unknown): unknown => {\n      if (obj instanceof ScreenshotItem) {\n        return { base64: obj.base64, capturedAt: obj.capturedAt };\n      }\n      if (Array.isArray(obj)) {\n        return obj.map(processValue);\n      }\n      if (obj && typeof obj === 'object') {\n        const entries = Object.entries(obj).map(([key, value]) => [\n          key,\n          processValue(value),\n        ]);\n        return Object.fromEntries(entries);\n      }\n      return obj;\n    };\n\n    const data = processValue(this.toJSON());\n    return JSON.stringify(data, null, indents);\n  }\n\n  /**\n   * Convert to a plain object for JSON serialization\n   */\n  toJSON(): IGroupedActionDump {\n    return {\n      sdkVersion: this.sdkVersion,\n      groupName: this.groupName,\n      groupDescription: this.groupDescription,\n      modelBriefs: this.modelBriefs,\n      executions: this.executions.map((exec) => exec.toJSON()),\n      deviceType: this.deviceType,\n    };\n  }\n\n  /**\n   * Create a GroupedActionDump instance from a serialized JSON string\n   */\n  static fromSerializedString(serialized: string): GroupedActionDump {\n    const parsed = JSON.parse(\n      serialized,\n      reviverForDumpDeserialization,\n    ) as IGroupedActionDump;\n    return new GroupedActionDump(parsed);\n  }\n\n  /**\n   * Create a GroupedActionDump instance from a plain object\n   */\n  static fromJSON(data: IGroupedActionDump): GroupedActionDump {\n    return new GroupedActionDump(data);\n  }\n\n  /**\n   * Collect all ScreenshotItem instances from all executions.\n   *\n   * @returns Array of all ScreenshotItem instances across all executions\n   */\n  collectAllScreenshots(): ScreenshotItem[] {\n    const screenshots: ScreenshotItem[] = [];\n    for (const execution of this.executions) {\n      screenshots.push(...execution.collectScreenshots());\n    }\n    return screenshots;\n  }\n\n  /**\n   * Serialize the dump to files with screenshots as separate PNG files.\n   * Creates:\n   * - {basePath} - dump JSON with { $screenshot: id } references\n   * - {basePath}.screenshots/ - PNG files\n   * - {basePath}.screenshots.json - ID to path mapping\n   *\n   * @param basePath - Base path for the dump file\n   */\n  serializeToFiles(basePath: string): void {\n    const screenshotsDir = `${basePath}.screenshots`;\n    if (!existsSync(screenshotsDir)) {\n      mkdirSync(screenshotsDir, { recursive: true });\n    }\n\n    // Write screenshots to separate files\n    const screenshotMap: Record<string, string> = {};\n    const screenshots = this.collectAllScreenshots();\n\n    for (const screenshot of screenshots) {\n      if (screenshot.hasBase64()) {\n        const imagePath = join(\n          screenshotsDir,\n          `${screenshot.id}.${screenshot.extension}`,\n        );\n        const rawBase64 = screenshot.rawBase64;\n        writeFileSync(imagePath, Buffer.from(rawBase64, 'base64'));\n        screenshotMap[screenshot.id] = imagePath;\n      }\n    }\n\n    // Write screenshot map file\n    writeFileSync(\n      `${basePath}.screenshots.json`,\n      JSON.stringify(screenshotMap),\n      'utf-8',\n    );\n\n    // Write dump JSON with references\n    writeFileSync(basePath, this.serialize(), 'utf-8');\n  }\n\n  /**\n   * Read dump from files and return JSON string with inline screenshots.\n   * Reads the dump JSON and screenshot files, then inlines the base64 data.\n   *\n   * @param basePath - Base path for the dump file\n   * @returns JSON string with inline screenshots ({ base64: \"...\" } format)\n   */\n  static fromFilesAsInlineJson(basePath: string): string {\n    const dumpString = readFileSync(basePath, 'utf-8');\n    const screenshotsMapPath = `${basePath}.screenshots.json`;\n\n    if (!existsSync(screenshotsMapPath)) {\n      return dumpString;\n    }\n\n    // Read screenshot map and build imageMap from files\n    const screenshotMap: Record<string, string> = JSON.parse(\n      readFileSync(screenshotsMapPath, 'utf-8'),\n    );\n\n    const imageMap: Record<string, string> = {};\n    for (const [id, filePath] of Object.entries(screenshotMap)) {\n      if (existsSync(filePath)) {\n        const data = readFileSync(filePath);\n        const mime =\n          filePath.endsWith('.jpeg') || filePath.endsWith('.jpg')\n            ? 'jpeg'\n            : 'png';\n        imageMap[id] = `data:image/${mime};base64,${data.toString('base64')}`;\n      }\n    }\n\n    // Restore image references\n    const dumpData = JSON.parse(dumpString);\n    const processedData = restoreImageReferences(\n      dumpData,\n      (id) => imageMap[id] ?? '',\n    );\n    return JSON.stringify(processedData);\n  }\n\n  /**\n   * Clean up all files associated with a serialized dump.\n   *\n   * @param basePath - Base path for the dump file\n   */\n  static cleanupFiles(basePath: string): void {\n    const filesToClean = [\n      basePath,\n      `${basePath}.screenshots.json`,\n      `${basePath}.screenshots`,\n    ];\n\n    for (const filePath of filesToClean) {\n      try {\n        rmSync(filePath, { force: true, recursive: true });\n      } catch {\n        // Ignore errors - file may already be deleted\n      }\n    }\n  }\n\n  /**\n   * Get all file paths associated with a serialized dump.\n   *\n   * @param basePath - Base path for the dump file\n   * @returns Array of all associated file paths\n   */\n  static getFilePaths(basePath: string): string[] {\n    return [\n      basePath,\n      `${basePath}.screenshots.json`,\n      `${basePath}.screenshots`,\n    ];\n  }\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;\n  delayAfterRunner?: number;\n  /**\n   * An example param object for this action.\n   * Locate fields with { prompt } will automatically get bbox injected when needed.\n   */\n  sample?: { [K in keyof TParam]?: any };\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n  cacheAdapter?: import('./agent/cache-adapter').CacheAdapter;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n\n  /**\n   * Use directory-based report format with separate image files.\n   *\n   * When enabled:\n   * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n   * - Report is generated as `index.html` with relative image paths\n   * - Reduces memory usage and report file size\n   *\n   * IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n   * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n   * work due to browser CORS restrictions.\n   *\n   * @default 'single-html'\n   */\n  outputFormat?: 'single-html' | 'html-and-external-assets';\n\n  onTaskStartTip?: OnTaskStartTip;\n  aiActContext?: string;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Wait time in milliseconds after each action execution.\n   * This allows the UI to settle and stabilize before the next action.\n   * Defaults to 300ms when not provided.\n   */\n  waitAfterAction?: number;\n\n  /**\n   * When set to true, Midscene will use the target device's time (Android/iOS)\n   * instead of the system time. Useful when the device time differs from the\n   * host machine. Default: false\n   */\n  useDeviceTimestamp?: boolean;\n\n  /**\n   * Custom screenshot shrink factor to reduce AI token usage.\n   * When set, the screenshot will be scaled down by this factor from the physical resolution.\n   *\n   * Example:\n   * - Physical screen width: 3000px, dpr=6\n   * - Logical width: 500px\n   * - screenshotShrinkFactor: 2\n   * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n   * - AI analyzes the 1500px screenshot\n   * - Coordinates are transformed back to logical (500px) before actions execute\n   *\n   * Benefits:\n   * - Reduces token usage for high-resolution screenshots\n   * - Maintains accuracy by scaling coordinates appropriately\n   *\n   * Must be >= 1 (shrinking only, enlarging is not supported).\n   *\n   * @default 1 (no shrinking, uses original physical screenshot)\n   */\n  screenshotShrinkFactor?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["UIContext","ServiceError","Error","message","dump","replacerForDumpSerialization","_key","value","reviverForDumpDeserialization","key","ScreenshotItem","ExecutionDump","indents","JSON","task","serialized","parsed","data","screenshots","record","GroupedActionDump","processValue","obj","Array","entries","Object","exec","execution","basePath","screenshotsDir","existsSync","mkdirSync","screenshotMap","screenshot","imagePath","join","rawBase64","writeFileSync","Buffer","dumpString","readFileSync","screenshotsMapPath","imageMap","id","filePath","mime","dumpData","processedData","restoreImageReferences","filesToClean","rmSync"],"mappings":";;;;;AAAqD;;;;;;;;;;AAsJ9C,MAAeA;AA4BtB;AA+EO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF;AA0YA,SAASC,6BAA6BC,IAAY,EAAEC,KAAU;IAC5D,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,QACvC,OAAO;IAET,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,WACvC,OAAO;IAGT,IAAIA,SAAS,AAAgC,cAAhC,OAAOA,MAAM,cAAc,EACtC,OAAOA,MAAM,cAAc;IAE7B,OAAOA;AACT;AAcA,SAASC,8BAA8BC,GAAW,EAAEF,KAAU;IAE5D,IAAIE,AAAQ,iBAARA,OAAwB,AAAiB,YAAjB,OAAOF,SAAsBA,AAAU,SAAVA,OACvD,OAAOA;IAKT,IAAIG,eAAe,YAAY,CAACH,QAC9B,OAAOA;IAIL,YAAYA,SAAgBA,MAAM,MAAM;IAI5C,OAAOA;AACT;AAKO,MAAMI;IAkBX,UAAUC,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAAyB;QACvB,OAAO;YACL,SAAS,IAAI,CAAC,OAAO;YACrB,MAAM,IAAI,CAAC,IAAI;YACf,aAAa,IAAI,CAAC,WAAW;YAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAACE,OAAU;oBAC/B,GAAGA,IAAI;oBACP,UAAUA,KAAK,QAAQ,IAAI,EAAE;gBAC/B;YACA,cAAc,IAAI,CAAC,YAAY;QACjC;IACF;IAKA,OAAO,qBAAqBC,UAAkB,EAAiB;QAC7D,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIG,cAAcK;IAC3B;IAKA,OAAO,SAASC,IAAoB,EAAiB;QACnD,OAAO,IAAIN,cAAcM;IAC3B;IAQA,qBAAuC;QACrC,MAAMC,cAAgC,EAAE;QAExC,KAAK,MAAMJ,QAAQ,IAAI,CAAC,KAAK,CAAE;YAE7B,IAAIA,KAAK,SAAS,EAAE,sBAAsBJ,gBACxCQ,YAAY,IAAI,CAACJ,KAAK,SAAS,CAAC,UAAU;YAI5C,IAAIA,KAAK,QAAQ,EACf;gBAAA,KAAK,MAAMK,UAAUL,KAAK,QAAQ,CAChC,IAAIK,OAAO,UAAU,YAAYT,gBAC/BQ,YAAY,IAAI,CAACC,OAAO,UAAU;YAEtC;QAEJ;QAEA,OAAOD;IACT;IA3EA,YAAYD,IAAoB,CAAE;QANlC;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,OAAO,GAAGA,KAAK,OAAO;QAC3B,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI;QACrB,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,YAAY,GAAGA,KAAK,YAAY;IACvC;AAsEF;AAwIO,MAAMG;IAuBX,UAAUR,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAMA,+BAA+BA,OAAgB,EAAU;QACvD,MAAMS,eAAe,CAACC;YACpB,IAAIA,eAAeZ,gBACjB,OAAO;gBAAE,QAAQY,IAAI,MAAM;gBAAE,YAAYA,IAAI,UAAU;YAAC;YAE1D,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAACD;YAEjB,IAAIC,OAAO,AAAe,YAAf,OAAOA,KAAkB;gBAClC,MAAME,UAAUC,OAAO,OAAO,CAACH,KAAK,GAAG,CAAC,CAAC,CAACb,KAAKF,MAAM,GAAK;wBACxDE;wBACAY,aAAad;qBACd;gBACD,OAAOkB,OAAO,WAAW,CAACD;YAC5B;YACA,OAAOF;QACT;QAEA,MAAML,OAAOI,aAAa,IAAI,CAAC,MAAM;QACrC,OAAOR,KAAK,SAAS,CAACI,MAAM,MAAML;IACpC;IAKA,SAA6B;QAC3B,OAAO;YACL,YAAY,IAAI,CAAC,UAAU;YAC3B,WAAW,IAAI,CAAC,SAAS;YACzB,kBAAkB,IAAI,CAAC,gBAAgB;YACvC,aAAa,IAAI,CAAC,WAAW;YAC7B,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAACc,OAASA,KAAK,MAAM;YACrD,YAAY,IAAI,CAAC,UAAU;QAC7B;IACF;IAKA,OAAO,qBAAqBX,UAAkB,EAAqB;QACjE,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIY,kBAAkBJ;IAC/B;IAKA,OAAO,SAASC,IAAwB,EAAqB;QAC3D,OAAO,IAAIG,kBAAkBH;IAC/B;IAOA,wBAA0C;QACxC,MAAMC,cAAgC,EAAE;QACxC,KAAK,MAAMS,aAAa,IAAI,CAAC,UAAU,CACrCT,YAAY,IAAI,IAAIS,UAAU,kBAAkB;QAElD,OAAOT;IACT;IAWA,iBAAiBU,QAAgB,EAAQ;QACvC,MAAMC,iBAAiB,GAAGD,SAAS,YAAY,CAAC;QAChD,IAAI,CAACE,WAAWD,iBACdE,UAAUF,gBAAgB;YAAE,WAAW;QAAK;QAI9C,MAAMG,gBAAwC,CAAC;QAC/C,MAAMd,cAAc,IAAI,CAAC,qBAAqB;QAE9C,KAAK,MAAMe,cAAcf,YACvB,IAAIe,WAAW,SAAS,IAAI;YAC1B,MAAMC,YAAYC,KAChBN,gBACA,GAAGI,WAAW,EAAE,CAAC,CAAC,EAAEA,WAAW,SAAS,EAAE;YAE5C,MAAMG,YAAYH,WAAW,SAAS;YACtCI,cAAcH,WAAWI,OAAO,IAAI,CAACF,WAAW;YAChDJ,aAAa,CAACC,WAAW,EAAE,CAAC,GAAGC;QACjC;QAIFG,cACE,GAAGT,SAAS,iBAAiB,CAAC,EAC9Bf,KAAK,SAAS,CAACmB,gBACf;QAIFK,cAAcT,UAAU,IAAI,CAAC,SAAS,IAAI;IAC5C;IASA,OAAO,sBAAsBA,QAAgB,EAAU;QACrD,MAAMW,aAAaC,aAAaZ,UAAU;QAC1C,MAAMa,qBAAqB,GAAGb,SAAS,iBAAiB,CAAC;QAEzD,IAAI,CAACE,WAAWW,qBACd,OAAOF;QAIT,MAAMP,gBAAwCnB,KAAK,KAAK,CACtD2B,aAAaC,oBAAoB;QAGnC,MAAMC,WAAmC,CAAC;QAC1C,KAAK,MAAM,CAACC,IAAIC,SAAS,IAAInB,OAAO,OAAO,CAACO,eAC1C,IAAIF,WAAWc,WAAW;YACxB,MAAM3B,OAAOuB,aAAaI;YAC1B,MAAMC,OACJD,SAAS,QAAQ,CAAC,YAAYA,SAAS,QAAQ,CAAC,UAC5C,SACA;YACNF,QAAQ,CAACC,GAAG,GAAG,CAAC,WAAW,EAAEE,KAAK,QAAQ,EAAE5B,KAAK,QAAQ,CAAC,WAAW;QACvE;QAIF,MAAM6B,WAAWjC,KAAK,KAAK,CAAC0B;QAC5B,MAAMQ,gBAAgBC,uBACpBF,UACA,CAACH,KAAOD,QAAQ,CAACC,GAAG,IAAI;QAE1B,OAAO9B,KAAK,SAAS,CAACkC;IACxB;IAOA,OAAO,aAAanB,QAAgB,EAAQ;QAC1C,MAAMqB,eAAe;YACnBrB;YACA,GAAGA,SAAS,iBAAiB,CAAC;YAC9B,GAAGA,SAAS,YAAY,CAAC;SAC1B;QAED,KAAK,MAAMgB,YAAYK,aACrB,IAAI;YACFC,OAAON,UAAU;gBAAE,OAAO;gBAAM,WAAW;YAAK;QAClD,EAAE,OAAM,CAER;IAEJ;IAQA,OAAO,aAAahB,QAAgB,EAAY;QAC9C,OAAO;YACLA;YACA,GAAGA,SAAS,iBAAiB,CAAC;YAC9B,GAAGA,SAAS,YAAY,CAAC;SAC1B;IACH;IA9MA,YAAYX,IAAwB,CAAE;QAPtC;QACA;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU;QACjC,IAAI,CAAC,SAAS,GAAGA,KAAK,SAAS;QAC/B,IAAI,CAAC,gBAAgB,GAAGA,KAAK,gBAAgB;QAC7C,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU,CAAC,GAAG,CAAC,CAACS,OACrCA,gBAAgBf,gBAAgBe,OAAOf,cAAc,QAAQ,CAACe;QAEhE,IAAI,CAAC,UAAU,GAAGT,KAAK,UAAU;IACnC;AAsMF"}

package/dist/es/utils.mjs

@@ -159,7 +159,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "1.6.9";
+    return "1.6.11";
 }
 function debugLog(...message) {
     const debugMode = process.env[MIDSCENE_DEBUG_MODE];