@donggui/core 1.6.7 → 1.6.9

package/dist/es/agent/task-builder.mjs

@@ -3,7 +3,7 @@ import { findAllMidsceneLocatorField, parseActionParam } from "../ai-model/index
 import { setTimingFieldOnce } from "../task-timing.mjs";
 import { ServiceError } from "../types.mjs";
 import { sleep } from "../utils.mjs";
-import { MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD, MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR } from "@midscene/shared/env/constants";
+import { MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD, MIDSCENE_CACHE_ENABLE_COORD_CHECK, MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR, MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY } from "@midscene/shared/env/constants";
 import { generateElementByRect } from "@midscene/shared/extractor";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
@@ -239,99 +239,103 @@ class TaskBuilder {
                     const cacheFeature = cacheEntry;
                     const cachedCenter = cacheFeature?.cachedCenter;
                     isLegacyCache = !cachedCenter;
+                    const enableCoordCheck = 'true' === process.env[MIDSCENE_CACHE_ENABLE_COORD_CHECK];
+                    const enableVisualVerify = 'true' === process.env[MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY];
                     const coordOffsetThreshold = Number.parseInt(process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] || '16', 10) || 16;
-                    const confidenceState = cacheFeature?.confidenceState || createInitialConfidenceState();
-                    const confidence = isLegacyCache ? 0.35 : calculateConfidence(confidenceState);
-                    const level = determineVerificationLevel(confidence);
-                    const actions = getVerificationActions(level);
-                    debug('cache confidence assessment', {
-                        confidence: confidence.toFixed(3),
-                        level,
-                        actions,
-                        verificationCount: confidenceState.verificationCount,
-                        isLegacyCache
-                    });
-                    if (actions.skipCache) {
-                        debug('cache confidence too low, skipping cache entirely', {
-                            confidence,
-                            level
+                    if (enableCoordCheck || enableVisualVerify) {
+                        const confidenceState = cacheFeature?.confidenceState || createInitialConfidenceState();
+                        const confidence = isLegacyCache ? 0.35 : calculateConfidence(confidenceState);
+                        const level = determineVerificationLevel(confidence);
+                        const actions = getVerificationActions(level);
+                        debug('cache confidence assessment', {
+                            confidence: confidence.toFixed(3),
+                            level,
+                            actions,
+                            verificationCount: confidenceState.verificationCount,
+                            isLegacyCache
                         });
-                        isCacheHit = false;
-                    }
-                    try {
-                        if (isCacheHit && actions.coordCheck && cachedCenter) {
-                            const offset = Math.sqrt((elementFromCache.center[0] - cachedCenter[0]) ** 2 + (elementFromCache.center[1] - cachedCenter[1]) ** 2);
-                            debug('cache coord offset check', {
-                                cachedCenter,
-                                currentCenter: elementFromCache.center,
-                                offset: Math.round(offset),
-                                threshold: coordOffsetThreshold
+                        if (actions.skipCache) {
+                            debug('cache confidence too low, skipping cache entirely', {
+                                confidence,
+                                level
                             });
-                            if (offset > coordOffsetThreshold) {
-                                debug('cache coord offset exceeded threshold, fallback to AI locate', {
-                                    offset,
+                            isCacheHit = false;
+                        }
+                        try {
+                            if (isCacheHit && enableCoordCheck && actions.coordCheck && cachedCenter) {
+                                const offset = Math.sqrt((elementFromCache.center[0] - cachedCenter[0]) ** 2 + (elementFromCache.center[1] - cachedCenter[1]) ** 2);
+                                debug('cache coord offset check', {
+                                    cachedCenter,
+                                    currentCenter: elementFromCache.center,
+                                    offset: Math.round(offset),
                                     threshold: coordOffsetThreshold
                                 });
-                                isCacheHit = false;
+                                if (offset > coordOffsetThreshold) {
+                                    debug('cache coord offset exceeded threshold, fallback to AI locate', {
+                                        offset,
+                                        threshold: coordOffsetThreshold
+                                    });
+                                    isCacheHit = false;
+                                }
                             }
-                        }
-                        if (isCacheHit && actions.visualVerify) {
-                            const verification = await this.service.verifyCachedElement(elementFromCache.center, cachePrompt, modelConfigForDefaultIntent, uiContext);
-                            if (verification.pass) debug('cache hit and visual verification passed', {
-                                description: verification.description
-                            });
-                            else {
-                                debug('cache hit but visual verification failed, fallback to AI locate', {
-                                    reason: verification.reason,
-                                    description: verification.description,
-                                    prompt: cachePrompt
+                            if (isCacheHit && enableVisualVerify && actions.visualVerify) {
+                                const verification = await this.service.verifyCachedElement(elementFromCache.center, cachePrompt, modelConfigForDefaultIntent, uiContext);
+                                if (verification.pass) debug('cache hit and visual verification passed', {
+                                    description: verification.description
                                 });
-                                isCacheHit = false;
+                                else {
+                                    debug('cache hit but visual verification failed, fallback to AI locate', {
+                                        reason: verification.reason,
+                                        description: verification.description,
+                                        prompt: cachePrompt
+                                    });
+                                    isCacheHit = false;
+                                }
                             }
+                        } catch (verifyError) {
+                            debug('cache verification error, fallback to AI locate', verifyError);
+                            isCacheHit = false;
                         }
-                    } catch (verifyError) {
-                        debug('cache verification error, fallback to AI locate', verifyError);
-                        isCacheHit = false;
-                    }
-                    if (isCacheHit) {
-                        const updatedState = updateConfidenceOnVerify(confidenceState, true);
-                        cacheFeature.confidenceState = updatedState;
-                        if (isLegacyCache) {
-                            cacheFeature.cachedCenter = elementFromCache.center;
-                            cacheFeature.progressiveRecord = createInitialProgressiveRecord(elementFromCache.center);
-                            debug('legacy cache upgraded with new fields', {
-                                cachedCenter: cacheFeature.cachedCenter
-                            });
-                        }
-                        const progressiveRecord = cacheFeature.progressiveRecord;
-                        if (progressiveRecord) {
-                            const updated = updateProgressiveConvergence(progressiveRecord, elementFromCache.center, updatedState.confidenceScore);
-                            cacheFeature.progressiveRecord = updated;
-                            if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {
-                                debug('using converged center instead of single-result center', {
-                                    convergedCenter: updated.convergedCenter.map((v)=>v.toFixed(1)),
-                                    singleCenter: elementFromCache.center,
-                                    convergenceRadius: updated.convergenceRadius.toFixed(1),
-                                    sampleCount: updated.sampleCount
+                        if (isCacheHit) {
+                            const updatedState = updateConfidenceOnVerify(confidenceState, true);
+                            cacheFeature.confidenceState = updatedState;
+                            if (isLegacyCache) {
+                                cacheFeature.cachedCenter = elementFromCache.center;
+                                cacheFeature.progressiveRecord = createInitialProgressiveRecord(elementFromCache.center);
+                                debug('legacy cache upgraded with new fields', {
+                                    cachedCenter: cacheFeature.cachedCenter
                                 });
-                                elementFromCache = {
-                                    ...elementFromCache,
-                                    center: [
-                                        Math.round(updated.convergedCenter[0]),
-                                        Math.round(updated.convergedCenter[1])
-                                    ]
-                                };
                             }
+                            const progressiveRecord = cacheFeature.progressiveRecord;
+                            if (progressiveRecord) {
+                                const updated = updateProgressiveConvergence(progressiveRecord, elementFromCache.center, updatedState.confidenceScore);
+                                cacheFeature.progressiveRecord = updated;
+                                if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {
+                                    debug('using converged center instead of single-result center', {
+                                        convergedCenter: updated.convergedCenter.map((v)=>v.toFixed(1)),
+                                        singleCenter: elementFromCache.center,
+                                        convergenceRadius: updated.convergenceRadius.toFixed(1),
+                                        sampleCount: updated.sampleCount
+                                    });
+                                    elementFromCache = {
+                                        ...elementFromCache,
+                                        center: [
+                                            Math.round(updated.convergedCenter[0]),
+                                            Math.round(updated.convergedCenter[1])
+                                        ]
+                                    };
+                                }
+                            }
+                        } else {
+                            const updatedState = updateConfidenceOnVerify(confidenceState, false);
+                            cacheFeature.confidenceState = updatedState;
                         }
-                    } else {
-                        const updatedState = updateConfidenceOnVerify(confidenceState, false);
-                        cacheFeature.confidenceState = updatedState;
                     }
                 }
                 if (!isXpathHit && !isCacheHit && !isPlanHit) {
                     const cacheFeature = cacheEntry;
                     const semanticAnchor = cacheFeature?.semanticAnchor;
-                    if (semanticAnchor && 'false' !== process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR]) try {
+                    if (semanticAnchor && 'true' === process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR]) try {
                         const anchorResult = await this.service.locateBySemanticAnchor(semanticAnchor, modelConfigForDefaultIntent, this.interface, uiContext);
                         if (anchorResult) {
                             elementFromAiLocate = anchorResult;
@@ -375,7 +379,7 @@ class TaskBuilder {
                         feature.confidenceState = createInitialConfidenceState();
                         feature.progressiveRecord = createInitialProgressiveRecord(pointForCache);
                         debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
-                        const enableSemanticAnchor = 'false' !== process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR];
+                        const enableSemanticAnchor = 'true' === process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR];
                         if (enableSemanticAnchor) try {
                             const anchor = await this.service.generateSemanticAnchor(pointForCache, modelConfigForDefaultIntent, uiContext);
                             if (anchor) {

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import {\n  calculateConfidence,\n  createInitialConfidenceState,\n  createInitialProgressiveRecord,\n  determineVerificationLevel,\n  getVerificationActions,\n  updateConfidenceOnVerify,\n  updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  CacheConfidenceState,\n  CacheValidationOptions,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  ProgressiveLocateRecord,\n  Rect,\n  SemanticAnchor,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n  MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n  MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n  MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord =\n          await this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        let elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        let isCacheHit = !!elementFromCache;\n        let isLegacyCache = false;\n        const timing = taskContext.task.timing;\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n\n        if (isCacheHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const cachedCenter = cacheFeature?.cachedCenter as\n            | [number, number]\n            | undefined;\n\n          isLegacyCache = !cachedCenter;\n\n          const coordOffsetThreshold =\n            Number.parseInt(\n              process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] || '16',\n              10,\n            ) || 16;\n\n          const confidenceState = (cacheFeature?.confidenceState ||\n            createInitialConfidenceState()) as CacheConfidenceState;\n          const confidence = isLegacyCache\n            ? 0.35\n            : calculateConfidence(confidenceState);\n          const level = determineVerificationLevel(confidence);\n          const actions = getVerificationActions(level);\n\n          debug('cache confidence assessment', {\n            confidence: confidence.toFixed(3),\n            level,\n            actions,\n            verificationCount: confidenceState.verificationCount,\n            isLegacyCache,\n          });\n\n          if (actions.skipCache) {\n            debug('cache confidence too low, skipping cache entirely', {\n              confidence,\n              level,\n            });\n            isCacheHit = false;\n          }\n\n          try {\n            if (isCacheHit && actions.coordCheck && cachedCenter) {\n              const offset = Math.sqrt(\n                (elementFromCache!.center[0] - cachedCenter[0]) ** 2 +\n                  (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n              );\n\n              debug('cache coord offset check', {\n                cachedCenter,\n                currentCenter: elementFromCache!.center,\n                offset: Math.round(offset),\n                threshold: coordOffsetThreshold,\n              });\n\n              if (offset > coordOffsetThreshold) {\n                debug(\n                  'cache coord offset exceeded threshold, fallback to AI locate',\n                  { offset, threshold: coordOffsetThreshold },\n                );\n                isCacheHit = false;\n              }\n            }\n\n            if (isCacheHit && actions.visualVerify) {\n              const verification = await this.service.verifyCachedElement(\n                elementFromCache!.center,\n                cachePrompt,\n                modelConfigForDefaultIntent,\n                uiContext,\n              );\n              if (!verification.pass) {\n                debug(\n                  'cache hit but visual verification failed, fallback to AI locate',\n                  {\n                    reason: verification.reason,\n                    description: verification.description,\n                    prompt: cachePrompt,\n                  },\n                );\n                isCacheHit = false;\n              } else {\n                debug('cache hit and visual verification passed', {\n                  description: verification.description,\n                });\n              }\n            }\n          } catch (verifyError) {\n            debug(\n              'cache verification error, fallback to AI locate',\n              verifyError,\n            );\n            isCacheHit = false;\n          }\n\n          if (isCacheHit) {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              true,\n            );\n            cacheFeature.confidenceState = updatedState;\n\n            // Upgrade legacy cache: fill missing fields\n            if (isLegacyCache) {\n              cacheFeature.cachedCenter = elementFromCache!.center;\n              cacheFeature.progressiveRecord = createInitialProgressiveRecord(\n                elementFromCache!.center,\n              );\n              debug('legacy cache upgraded with new fields', {\n                cachedCenter: cacheFeature.cachedCenter,\n              });\n            }\n\n            const progressiveRecord = cacheFeature.progressiveRecord as\n              | ProgressiveLocateRecord\n              | undefined;\n            if (progressiveRecord) {\n              const updated = updateProgressiveConvergence(\n                progressiveRecord,\n                elementFromCache!.center,\n                updatedState.confidenceScore,\n              );\n              cacheFeature.progressiveRecord = updated;\n\n              if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n                debug(\n                  'using converged center instead of single-result center',\n                  {\n                    convergedCenter: updated.convergedCenter.map((v) =>\n                      v.toFixed(1),\n                    ),\n                    singleCenter: elementFromCache!.center,\n                    convergenceRadius: updated.convergenceRadius.toFixed(1),\n                    sampleCount: updated.sampleCount,\n                  },\n                );\n                elementFromCache = {\n                  ...elementFromCache!,\n                  center: [\n                    Math.round(updated.convergedCenter[0]),\n                    Math.round(updated.convergedCenter[1]),\n                  ],\n                };\n              }\n            }\n          } else {\n            const updatedState = updateConfidenceOnVerify(\n              confidenceState,\n              false,\n            );\n            cacheFeature.confidenceState = updatedState;\n          }\n        }\n\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const semanticAnchor = cacheFeature?.semanticAnchor as\n            | SemanticAnchor\n            | undefined;\n\n          if (\n            semanticAnchor &&\n            process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !== 'false'\n          ) {\n            try {\n              const anchorResult = await this.service.locateBySemanticAnchor(\n                semanticAnchor,\n                modelConfigForDefaultIntent,\n                this.interface,\n                uiContext,\n              );\n              if (anchorResult) {\n                elementFromAiLocate = anchorResult;\n                debug(\n                  'semantic anchor locate succeeded, skipping full AI locate',\n                );\n              }\n            } catch (anchorError) {\n              debug('semantic anchor locate failed:', anchorError);\n            }\n          }\n\n          if (!elementFromAiLocate) {\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              locateResult = await this.service.locate(\n                param,\n                {\n                  context: uiContext,\n                },\n                modelConfigForDefaultIntent,\n                abortSignal,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n            } catch (error) {\n              if (error instanceof ServiceError) {\n                applyDump(error.dump);\n              }\n              throw error;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read), OR cache hit but legacy cache needs upgrade\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          (!isCacheHit || isLegacyCache) &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                feature.cachedCenter = pointForCache;\n                feature.confidenceState = createInitialConfidenceState();\n                feature.progressiveRecord =\n                  createInitialProgressiveRecord(pointForCache);\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n\n                const enableSemanticAnchor =\n                  process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] !==\n                  'false';\n                if (enableSemanticAnchor) {\n                  try {\n                    const anchor = await this.service.generateSemanticAnchor(\n                      pointForCache,\n                      modelConfigForDefaultIntent,\n                      uiContext,\n                    );\n                    if (anchor) {\n                      feature.semanticAnchor = anchor;\n                      debug(\n                        'semantic anchor generated for prompt: %s',\n                        cachePrompt,\n                      );\n                    }\n                  } catch (anchorError) {\n                    debug('generateSemanticAnchor failed:', anchorError);\n                  }\n                }\n\n                currentCacheEntry = feature;\n                await this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","isLegacyCache","elementFromAiLocate","cacheFeature","cachedCenter","coordOffsetThreshold","Number","process","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","createInitialProgressiveRecord","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqDA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC3C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,MAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,iBAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,mBAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,wBAAwB5B,SAC5C4D,qBAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,oCACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,IAAIG,gBAAgB;gBACpB,MAAMzC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI2C;gBAEJ,IAAIF,YAAY;oBACd,MAAMG,eAAeR;oBACrB,MAAMS,eAAeD,cAAc;oBAInCF,gBAAgB,CAACG;oBAEjB,MAAMC,uBACJC,OAAO,QAAQ,CACbC,QAAQ,GAAG,CAACC,sCAAsC,IAAI,MACtD,OACG;oBAEP,MAAMC,kBAAmBN,cAAc,mBACrCO;oBACF,MAAMC,aAAaV,gBACf,OACAW,oBAAoBH;oBACxB,MAAMI,QAAQC,2BAA2BH;oBACzC,MAAMI,UAAUC,uBAAuBH;oBAEvC7F,MAAM,+BAA+B;wBACnC,YAAY2F,WAAW,OAAO,CAAC;wBAC/BE;wBACAE;wBACA,mBAAmBN,gBAAgB,iBAAiB;wBACpDR;oBACF;oBAEA,IAAIc,QAAQ,SAAS,EAAE;wBACrB/F,MAAM,qDAAqD;4BACzD2F;4BACAE;wBACF;wBACAb,aAAa;oBACf;oBAEA,IAAI;wBACF,IAAIA,cAAce,QAAQ,UAAU,IAAIX,cAAc;4BACpD,MAAMa,SAASC,KAAK,IAAI,CACrBpB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM,IAChDN,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM;4BAGvDpF,MAAM,4BAA4B;gCAChCoF;gCACA,eAAeN,iBAAkB,MAAM;gCACvC,QAAQoB,KAAK,KAAK,CAACD;gCACnB,WAAWZ;4BACb;4BAEA,IAAIY,SAASZ,sBAAsB;gCACjCrF,MACE,gEACA;oCAAEiG;oCAAQ,WAAWZ;gCAAqB;gCAE5CL,aAAa;4BACf;wBACF;wBAEA,IAAIA,cAAce,QAAQ,YAAY,EAAE;4BACtC,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDrB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;4BAEF,IAAK0D,aAAa,IAAI,EAWpBnG,MAAM,4CAA4C;gCAChD,aAAamG,aAAa,WAAW;4BACvC;iCAbsB;gCACtBnG,MACE,mEACA;oCACE,QAAQmG,aAAa,MAAM;oCAC3B,aAAaA,aAAa,WAAW;oCACrC,QAAQ1B;gCACV;gCAEFO,aAAa;4BACf;wBAKF;oBACF,EAAE,OAAOoB,aAAa;wBACpBpG,MACE,mDACAoG;wBAEFpB,aAAa;oBACf;oBAEA,IAAIA,YAAY;wBACd,MAAMqB,eAAeC,yBACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;wBAG/B,IAAIpB,eAAe;4BACjBE,aAAa,YAAY,GAAGL,iBAAkB,MAAM;4BACpDK,aAAa,iBAAiB,GAAGoB,+BAC/BzB,iBAAkB,MAAM;4BAE1B9E,MAAM,yCAAyC;gCAC7C,cAAcmF,aAAa,YAAY;4BACzC;wBACF;wBAEA,MAAMqB,oBAAoBrB,aAAa,iBAAiB;wBAGxD,IAAIqB,mBAAmB;4BACrB,MAAMC,UAAUC,6BACdF,mBACA1B,iBAAkB,MAAM,EACxBuB,aAAa,eAAe;4BAE9BlB,aAAa,iBAAiB,GAAGsB;4BAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;gCAC7DzG,MACE,0DACA;oCACE,iBAAiByG,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;oCAEZ,cAAc7B,iBAAkB,MAAM;oCACtC,mBAAmB2B,QAAQ,iBAAiB,CAAC,OAAO,CAAC;oCACrD,aAAaA,QAAQ,WAAW;gCAClC;gCAEF3B,mBAAmB;oCACjB,GAAGA,gBAAgB;oCACnB,QAAQ;wCACNoB,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;wCACrCP,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;qCACtC;gCACH;4BACF;wBACF;oBACF,OAAO;wBACL,MAAMJ,eAAeC,yBACnBb,iBACA;wBAEFN,aAAa,eAAe,GAAGkB;oBACjC;gBACF;gBAEA,IAAI,CAAC7B,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMgB,eAAeR;oBACrB,MAAMiC,iBAAiBzB,cAAc;oBAIrC,IACEyB,kBACArB,AAAuD,YAAvDA,QAAQ,GAAG,CAACsB,sCAAsC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACAhG,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAIqE,cAAc;4BAChB5B,sBAAsB4B;4BACtB9G,MACE;wBAEJ;oBACF,EAAE,OAAO+G,aAAa;wBACpB/G,MAAM,kCAAkC+G;oBAC1C;oBAGF,IAAI,CAAC7B,qBACH,IAAI;wBACFxC,mBAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BoB,sBAAsBpB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiB6D,cACnBjD,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,mBAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAMyE,UACJhD,mBACAI,oBACAS,oBACAI;gBAGF,MAAMgC,2BAA2BhH,iBAC/BwE,mBAAmB,cAAc;gBAGnC,IAAIyC;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACb,EAACjC,cAAcC,aAAY,KAC3B,EAACd,aAAa,CAAC+C,wBAAuB,KACvC5G,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAI8G,gBAAkCH,QAAQ,MAAM;oBACpD,IAAIjE,AAA6B,MAA7BA,0BAAgC;wBAClCoE,gBAAgB;4BACdlB,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGjE;4BAC/BkD,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGjE;yBAChC;wBACDhD,MACE,8DACAiH,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAO9G,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiBmH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG3B;wBAC1B2B,QAAQ,iBAAiB,GACvBd,+BAA+Ba;wBACjCpH,MACE,uCACAyE,aACA4C;wBAGF,MAAMC,uBACJ/B,AACA,YADAA,QAAQ,GAAG,CAACsB,sCAAsC;wBAEpD,IAAIS,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDH,eACAxG,6BACA6B;4BAEF,IAAI8E,QAAQ;gCACVF,QAAQ,cAAc,GAAGE;gCACzBvH,MACE,4CACAyE;4BAEJ;wBACF,EAAE,OAAOsC,aAAa;4BACpB/G,MAAM,kCAAkC+G;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQ5C;4BACR,OAAO4C;wBACT,GACA3C;oBAEJ,OACE1E,MACE,yDACAyE;gBAGN,EAAE,OAAOtB,OAAO;oBACdnD,MAAM,mCAAmCmD;gBAC3C;qBAEAnD,MAAM;gBAIV,IAAI,CAACiH,SAAS;oBACZ,IAAIpD,YACF,MAAM,IAAImD,aACR,CAAC,oBAAoB,EAAE1G,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIkH;gBAEJ,IAAIrD,WACFqD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMlH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACTgD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOlH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACTwC,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP7C;wBACA,aAAawC;oBACf;gBACF;gBAGF3D,WAAWyD;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAKxE,UAAU,aAAa;wBAC9B;oBACF;oBACA+E;gBACF;YACF;QACF;QAEA,OAAO5D;IACT;IA/vBA,YAAY,EACV6D,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTlG,WAAW,EACXmG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGlG;QACnB,IAAI,CAAC,eAAe,GAAGmG;IACzB;AAovBF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import {\n  calculateConfidence,\n  createInitialConfidenceState,\n  createInitialProgressiveRecord,\n  determineVerificationLevel,\n  getVerificationActions,\n  updateConfidenceOnVerify,\n  updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  CacheConfidenceState,\n  CacheValidationOptions,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  ProgressiveLocateRecord,\n  Rect,\n  SemanticAnchor,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n  MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n  MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n  MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n  ifPlanLocateParamIsBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = typeof param === 'string' ? { prompt: param } : param;\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  modelConfigForPlanning: IModelConfig;\n  modelConfigForDefaultIntent: IModelConfig;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = actionSpace.find((item) => item.name === planType);\n    const param = plan.param;\n\n    if (!action) {\n      throw new Error(`Action type '${planType}' not found`);\n    }\n\n    const locateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema)\n      : [];\n\n    const requiredLocateFields = action\n      ? findAllMidsceneLocatorField(action.paramSchema, true)\n      : [];\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params (including bbox)\n        // This ensures cache writing happens even when bbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            sleep(200),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n      context;\n\n    let locateParam = detailedLocateParam;\n\n    if (typeof locateParam === 'string') {\n      locateParam = {\n        prompt: locateParam,\n      };\n    }\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n\n        assert(\n          param?.prompt || param?.bbox,\n          `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // from bbox (plan hit)\n        const elementFromBbox = ifPlanLocateParamIsBbox(param)\n          ? matchElementFromPlan(param)\n          : undefined;\n        const isPlanHit = !!elementFromBbox;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord =\n          await this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        let elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        let isCacheHit = !!elementFromCache;\n        let isLegacyCache = false;\n        const timing = taskContext.task.timing;\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n\n        if (isCacheHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const cachedCenter = cacheFeature?.cachedCenter as\n            | [number, number]\n            | undefined;\n\n          isLegacyCache = !cachedCenter;\n\n          const enableCoordCheck =\n            process.env[MIDSCENE_CACHE_ENABLE_COORD_CHECK] === 'true';\n          const enableVisualVerify =\n            process.env[MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY] === 'true';\n\n          const coordOffsetThreshold =\n            Number.parseInt(\n              process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] || '16',\n              10,\n            ) || 16;\n\n          if (enableCoordCheck || enableVisualVerify) {\n            const confidenceState = (cacheFeature?.confidenceState ||\n              createInitialConfidenceState()) as CacheConfidenceState;\n            const confidence = isLegacyCache\n              ? 0.35\n              : calculateConfidence(confidenceState);\n            const level = determineVerificationLevel(confidence);\n            const actions = getVerificationActions(level);\n\n            debug('cache confidence assessment', {\n              confidence: confidence.toFixed(3),\n              level,\n              actions,\n              verificationCount: confidenceState.verificationCount,\n              isLegacyCache,\n            });\n\n            if (actions.skipCache) {\n              debug('cache confidence too low, skipping cache entirely', {\n                confidence,\n                level,\n              });\n              isCacheHit = false;\n            }\n\n            try {\n              if (\n                isCacheHit &&\n                enableCoordCheck &&\n                actions.coordCheck &&\n                cachedCenter\n              ) {\n                const offset = Math.sqrt(\n                  (elementFromCache!.center[0] - cachedCenter[0]) ** 2 +\n                    (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n                );\n\n                debug('cache coord offset check', {\n                  cachedCenter,\n                  currentCenter: elementFromCache!.center,\n                  offset: Math.round(offset),\n                  threshold: coordOffsetThreshold,\n                });\n\n                if (offset > coordOffsetThreshold) {\n                  debug(\n                    'cache coord offset exceeded threshold, fallback to AI locate',\n                    { offset, threshold: coordOffsetThreshold },\n                  );\n                  isCacheHit = false;\n                }\n              }\n\n              if (isCacheHit && enableVisualVerify && actions.visualVerify) {\n                const verification = await this.service.verifyCachedElement(\n                  elementFromCache!.center,\n                  cachePrompt,\n                  modelConfigForDefaultIntent,\n                  uiContext,\n                );\n                if (!verification.pass) {\n                  debug(\n                    'cache hit but visual verification failed, fallback to AI locate',\n                    {\n                      reason: verification.reason,\n                      description: verification.description,\n                      prompt: cachePrompt,\n                    },\n                  );\n                  isCacheHit = false;\n                } else {\n                  debug('cache hit and visual verification passed', {\n                    description: verification.description,\n                  });\n                }\n              }\n            } catch (verifyError) {\n              debug(\n                'cache verification error, fallback to AI locate',\n                verifyError,\n              );\n              isCacheHit = false;\n            }\n\n            if (isCacheHit) {\n              const updatedState = updateConfidenceOnVerify(\n                confidenceState,\n                true,\n              );\n              cacheFeature.confidenceState = updatedState;\n\n              if (isLegacyCache) {\n                cacheFeature.cachedCenter = elementFromCache!.center;\n                cacheFeature.progressiveRecord = createInitialProgressiveRecord(\n                  elementFromCache!.center,\n                );\n                debug('legacy cache upgraded with new fields', {\n                  cachedCenter: cacheFeature.cachedCenter,\n                });\n              }\n\n              const progressiveRecord = cacheFeature.progressiveRecord as\n                | ProgressiveLocateRecord\n                | undefined;\n              if (progressiveRecord) {\n                const updated = updateProgressiveConvergence(\n                  progressiveRecord,\n                  elementFromCache!.center,\n                  updatedState.confidenceScore,\n                );\n                cacheFeature.progressiveRecord = updated;\n\n                if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n                  debug(\n                    'using converged center instead of single-result center',\n                    {\n                      convergedCenter: updated.convergedCenter.map((v) =>\n                        v.toFixed(1),\n                      ),\n                      singleCenter: elementFromCache!.center,\n                      convergenceRadius: updated.convergenceRadius.toFixed(1),\n                      sampleCount: updated.sampleCount,\n                    },\n                  );\n                  elementFromCache = {\n                    ...elementFromCache!,\n                    center: [\n                      Math.round(updated.convergedCenter[0]),\n                      Math.round(updated.convergedCenter[1]),\n                    ],\n                  };\n                }\n              }\n            } else {\n              const updatedState = updateConfidenceOnVerify(\n                confidenceState,\n                false,\n              );\n              cacheFeature.confidenceState = updatedState;\n            }\n          }\n        }\n\n        if (!isXpathHit && !isCacheHit && !isPlanHit) {\n          const cacheFeature = cacheEntry as ElementCacheFeature;\n          const semanticAnchor = cacheFeature?.semanticAnchor as\n            | SemanticAnchor\n            | undefined;\n\n          if (\n            semanticAnchor &&\n            process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true'\n          ) {\n            try {\n              const anchorResult = await this.service.locateBySemanticAnchor(\n                semanticAnchor,\n                modelConfigForDefaultIntent,\n                this.interface,\n                uiContext,\n              );\n              if (anchorResult) {\n                elementFromAiLocate = anchorResult;\n                debug(\n                  'semantic anchor locate succeeded, skipping full AI locate',\n                );\n              }\n            } catch (anchorError) {\n              debug('semantic anchor locate failed:', anchorError);\n            }\n          }\n\n          if (!elementFromAiLocate) {\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              locateResult = await this.service.locate(\n                param,\n                {\n                  context: uiContext,\n                },\n                modelConfigForDefaultIntent,\n                abortSignal,\n              );\n              applyDump(locateResult.dump);\n              elementFromAiLocate = locateResult.element;\n            } catch (error) {\n              if (error instanceof ServiceError) {\n                applyDump(error.dump);\n              }\n              throw error;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n          }\n        }\n\n        const element =\n          elementFromBbox ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read), OR cache hit but legacy cache needs upgrade\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          (!isCacheHit || isLegacyCache) &&\n          (!isPlanHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelConfig: modelConfigForDefaultIntent,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                feature.cachedCenter = pointForCache;\n                feature.confidenceState = createInitialConfidenceState();\n                feature.progressiveRecord =\n                  createInitialProgressiveRecord(pointForCache);\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n\n                const enableSemanticAnchor =\n                  process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true';\n                if (enableSemanticAnchor) {\n                  try {\n                    const anchor = await this.service.generateSemanticAnchor(\n                      pointForCache,\n                      modelConfigForDefaultIntent,\n                      uiContext,\n                    );\n                    if (anchor) {\n                      feature.semanticAnchor = anchor;\n                      debug(\n                        'semantic anchor generated for prompt: %s',\n                        cachePrompt,\n                      );\n                    }\n                  } catch (anchorError) {\n                    debug('generateSemanticAnchor failed:', anchorError);\n                  }\n                }\n\n                currentCacheEntry = feature;\n                await this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanHit) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              bbox: param.bbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","isLegacyCache","elementFromAiLocate","cacheFeature","cachedCenter","enableCoordCheck","process","MIDSCENE_CACHE_ENABLE_COORD_CHECK","enableVisualVerify","MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY","coordOffsetThreshold","Number","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","createInitialProgressiveRecord","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqDA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC3C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,MAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,iBAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,mBAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,wBAAwB5B,SAC5C4D,qBAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,oCACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,IAAIG,gBAAgB;gBACpB,MAAMzC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI2C;gBAEJ,IAAIF,YAAY;oBACd,MAAMG,eAAeR;oBACrB,MAAMS,eAAeD,cAAc;oBAInCF,gBAAgB,CAACG;oBAEjB,MAAMC,mBACJC,AAAmD,WAAnDA,QAAQ,GAAG,CAACC,kCAAkC;oBAChD,MAAMC,qBACJF,AAAqD,WAArDA,QAAQ,GAAG,CAACG,oCAAoC;oBAElD,MAAMC,uBACJC,OAAO,QAAQ,CACbL,QAAQ,GAAG,CAACM,sCAAsC,IAAI,MACtD,OACG;oBAEP,IAAIP,oBAAoBG,oBAAoB;wBAC1C,MAAMK,kBAAmBV,cAAc,mBACrCW;wBACF,MAAMC,aAAad,gBACf,OACAe,oBAAoBH;wBACxB,MAAMI,QAAQC,2BAA2BH;wBACzC,MAAMI,UAAUC,uBAAuBH;wBAEvCjG,MAAM,+BAA+B;4BACnC,YAAY+F,WAAW,OAAO,CAAC;4BAC/BE;4BACAE;4BACA,mBAAmBN,gBAAgB,iBAAiB;4BACpDZ;wBACF;wBAEA,IAAIkB,QAAQ,SAAS,EAAE;4BACrBnG,MAAM,qDAAqD;gCACzD+F;gCACAE;4BACF;4BACAjB,aAAa;wBACf;wBAEA,IAAI;4BACF,IACEA,cACAK,oBACAc,QAAQ,UAAU,IAClBf,cACA;gCACA,MAAMiB,SAASC,KAAK,IAAI,CACrBxB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM,IAChDN,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM;gCAGvDpF,MAAM,4BAA4B;oCAChCoF;oCACA,eAAeN,iBAAkB,MAAM;oCACvC,QAAQwB,KAAK,KAAK,CAACD;oCACnB,WAAWX;gCACb;gCAEA,IAAIW,SAASX,sBAAsB;oCACjC1F,MACE,gEACA;wCAAEqG;wCAAQ,WAAWX;oCAAqB;oCAE5CV,aAAa;gCACf;4BACF;4BAEA,IAAIA,cAAcQ,sBAAsBW,QAAQ,YAAY,EAAE;gCAC5D,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDzB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;gCAEF,IAAK8D,aAAa,IAAI,EAWpBvG,MAAM,4CAA4C;oCAChD,aAAauG,aAAa,WAAW;gCACvC;qCAbsB;oCACtBvG,MACE,mEACA;wCACE,QAAQuG,aAAa,MAAM;wCAC3B,aAAaA,aAAa,WAAW;wCACrC,QAAQ9B;oCACV;oCAEFO,aAAa;gCACf;4BAKF;wBACF,EAAE,OAAOwB,aAAa;4BACpBxG,MACE,mDACAwG;4BAEFxB,aAAa;wBACf;wBAEA,IAAIA,YAAY;4BACd,MAAMyB,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;4BAE/B,IAAIxB,eAAe;gCACjBE,aAAa,YAAY,GAAGL,iBAAkB,MAAM;gCACpDK,aAAa,iBAAiB,GAAGwB,+BAC/B7B,iBAAkB,MAAM;gCAE1B9E,MAAM,yCAAyC;oCAC7C,cAAcmF,aAAa,YAAY;gCACzC;4BACF;4BAEA,MAAMyB,oBAAoBzB,aAAa,iBAAiB;4BAGxD,IAAIyB,mBAAmB;gCACrB,MAAMC,UAAUC,6BACdF,mBACA9B,iBAAkB,MAAM,EACxB2B,aAAa,eAAe;gCAE9BtB,aAAa,iBAAiB,GAAG0B;gCAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;oCAC7D7G,MACE,0DACA;wCACE,iBAAiB6G,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;wCAEZ,cAAcjC,iBAAkB,MAAM;wCACtC,mBAAmB+B,QAAQ,iBAAiB,CAAC,OAAO,CAAC;wCACrD,aAAaA,QAAQ,WAAW;oCAClC;oCAEF/B,mBAAmB;wCACjB,GAAGA,gBAAgB;wCACnB,QAAQ;4CACNwB,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;4CACrCP,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;yCACtC;oCACH;gCACF;4BACF;wBACF,OAAO;4BACL,MAAMJ,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;wBACjC;oBACF;gBACF;gBAEA,IAAI,CAACjC,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMgB,eAAeR;oBACrB,MAAMqC,iBAAiB7B,cAAc;oBAIrC,IACE6B,kBACA1B,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACApG,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAIyE,cAAc;4BAChBhC,sBAAsBgC;4BACtBlH,MACE;wBAEJ;oBACF,EAAE,OAAOmH,aAAa;wBACpBnH,MAAM,kCAAkCmH;oBAC1C;oBAGF,IAAI,CAACjC,qBACH,IAAI;wBACFxC,mBAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BoB,sBAAsBpB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiBiE,cACnBrD,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,mBAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAM6E,UACJpD,mBACAI,oBACAS,oBACAI;gBAGF,MAAMoC,2BAA2BpH,iBAC/BwE,mBAAmB,cAAc;gBAGnC,IAAI6C;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACb,EAACrC,cAAcC,aAAY,KAC3B,EAACd,aAAa,CAACmD,wBAAuB,KACvChH,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIkH,gBAAkCH,QAAQ,MAAM;oBACpD,IAAIrE,AAA6B,MAA7BA,0BAAgC;wBAClCwE,gBAAgB;4BACdlB,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;4BAC/BsD,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;yBAChC;wBACDhD,MACE,8DACAqH,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOlH,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiBuH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG3B;wBAC1B2B,QAAQ,iBAAiB,GACvBd,+BAA+Ba;wBACjCxH,MACE,uCACAyE,aACAgD;wBAGF,MAAMC,uBACJpC,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC;wBACpD,IAAIS,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDH,eACA5G,6BACA6B;4BAEF,IAAIkF,QAAQ;gCACVF,QAAQ,cAAc,GAAGE;gCACzB3H,MACE,4CACAyE;4BAEJ;wBACF,EAAE,OAAO0C,aAAa;4BACpBnH,MAAM,kCAAkCmH;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQhD;4BACR,OAAOgD;wBACT,GACA/C;oBAEJ,OACE1E,MACE,yDACAyE;gBAGN,EAAE,OAAOtB,OAAO;oBACdnD,MAAM,mCAAmCmD;gBAC3C;qBAEAnD,MAAM;gBAIV,IAAI,CAACqH,SAAS;oBACZ,IAAIxD,YACF,MAAM,IAAIuD,aACR,CAAC,oBAAoB,EAAE9G,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIsH;gBAEJ,IAAIzD,WACFyD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMtH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACToD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOtH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACT4C,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPjD;wBACA,aAAa4C;oBACf;gBACF;gBAGF/D,WAAW6D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK5E,UAAU,aAAa;wBAC9B;oBACF;oBACAmF;gBACF;YACF;QACF;QAEA,OAAOhE;IACT;IAzwBA,YAAY,EACViE,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTtG,WAAW,EACXuG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGtG;QACnB,IAAI,CAAC,eAAe,GAAGuG;IACzB;AA8vBF"}

package/dist/es/agent/utils.mjs

@@ -152,7 +152,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.6.7";
+const getMidsceneVersion = ()=>"1.6.9";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/assert.mjs

@@ -4,8 +4,6 @@ const debug = getDebug('ai:assert');
 const DEFAULT_SYSTEM_CHECK_PROMPT = `请检查以下系统级问题：
 
 1. **白屏检测**：页面是否完全白屏或大面积空白
-2. **布局遮挡**：重要元素是否存在组件间重叠遮挡
-3. **未加载内容**：页面是否有加载中的占位符、骨架屏
 4. **错误提示**：页面是否显示错误信息、异常提示
 5. **后端错误**：是否有后端请求失败的提示，如"活动太火爆了"、报错 Toast、错误码 等
 
@@ -41,7 +39,7 @@ ${systemCheckPrompt}`;
     return prompt;
 }
 async function AiAssertElement(options) {
-    const { beforeScreenshot, afterScreenshot, assertion, businessContext, enableSystemCheck = true, customSystemCheckRules, modelConfig, abortSignal } = options;
+    const { beforeScreenshot, afterScreenshot, assertion, businessContext, enableSystemCheck = false, customSystemCheckRules, modelConfig, abortSignal } = options;
     const systemPrompt = `你是一个自动化测试断言专家。请根据以下信息判断断言是否通过。
 
 你需要：

package/dist/es/ai-model/assert.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/assert.mjs","sources":["../../../src/ai-model/assert.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  DiffDetails,\n  IgnoreRegion,\n  ReferenceImage,\n  SystemCheckResults,\n  VideoAssertionOptions,\n  VideoDetails,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('ai:assert');\n\nexport interface AiAssertOptions {\n  beforeScreenshot?: string;\n  afterScreenshot: string;\n  assertion: string;\n  businessContext?: string;\n  enableSystemCheck?: boolean;\n  customSystemCheckRules?: string;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n  referenceImages?: ReferenceImage[];\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  systemCheckResults?: SystemCheckResults;\n  diffDetails?: DiffDetails;\n}\n\nconst DEFAULT_SYSTEM_CHECK_PROMPT = `请检查以下系统级问题：\n\n1. **白屏检测**：页面是否完全白屏或大面积空白\n2. **布局遮挡**：重要元素是否存在组件间重叠遮挡\n3. **未加载内容**：页面是否有加载中的占位符、骨架屏\n4. **错误提示**：页面是否显示错误信息、异常提示\n5. **后端错误**：是否有后端请求失败的提示，如\"活动太火爆了\"、报错 Toast、错误码 等\n\n如果发现以上问题，请在 systemCheckResults 中标注对应字段为 true。`;\n\nfunction buildAssertionPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  systemCheckPrompt?: string;\n}): string {\n  const { assertion, businessContext, systemCheckPrompt } = options;\n\n  let prompt = `## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  if (systemCheckPrompt) {\n    prompt += `\n\n## 系统校验规则\n${systemCheckPrompt}`;\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出断言结果：\n{\n  \"pass\": boolean,    // 断言是否通过\n  \"thought\": string,  // 思考过程\n  \"reason\": string,   // 失败原因（如果失败）\n  \"systemCheckResults\": {  // 系统校验结果（可选）\n    \"whiteScreen\": boolean,\n    \"layoutBlocked\": boolean,\n    \"loadingContent\": boolean,\n    \"errorPrompt\": boolean,\n    \"backendError\": boolean\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertElement(options: AiAssertOptions): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  usage?: AIUsageInfo;\n  systemCheckResults?: SystemCheckResults;\n  rawResponse?: string;\n}> {\n  const {\n    beforeScreenshot,\n    afterScreenshot,\n    assertion,\n    businessContext,\n    enableSystemCheck = true,\n    customSystemCheckRules,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const systemPrompt = `你是一个自动化测试断言专家。请根据以下信息判断断言是否通过。\n\n你需要：\n1. 分析操作前后的截图变化（如果提供了操作前截图）\n2. 验证用户的断言描述是否成立\n3. 检查是否存在系统级问题（如果启用）\n4. 给出详细的思考过程和判断结果`;\n\n  const systemCheckPrompt = enableSystemCheck\n    ? customSystemCheckRules || DEFAULT_SYSTEM_CHECK_PROMPT\n    : '';\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (beforeScreenshot) {\n    userContent.push({\n      type: 'text',\n      text: '## 操作前截图（执行操作前的页面状态）',\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: beforeScreenshot, detail: 'high' },\n    });\n    userContent.push({\n      type: 'text',\n      text: '## 操作后截图（执行操作后的页面状态）',\n    });\n  } else {\n    userContent.push({\n      type: 'text',\n      text: '## 当前页面截图',\n    });\n  }\n\n  userContent.push({\n    type: 'image_url',\n    image_url: { url: afterScreenshot, detail: 'high' },\n  });\n\n  userContent.push({\n    type: 'text',\n    text: buildAssertionPrompt({\n      assertion,\n      businessContext,\n      systemCheckPrompt,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: systemPrompt },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<AIAssertionResponse>(\n      msgs,\n      modelConfig,\n      { abortSignal },\n    );\n\n    debug('assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      usage: result.usage,\n      systemCheckResults: result.content.systemCheckResults,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('assertion error:', error);\n    throw error;\n  }\n}\n\nconst DIFF_SYSTEM_PROMPT = `你是一个自动化测试的图像对比专家。你的任务是对比当前页面截图与基准图片，判断页面样式是否符合预期。\n\n你需要：\n1. 仔细对比两张图片的布局、颜色、元素位置\n2. 分析差异是否在可接受范围内\n3. 识别动态内容（如时间、日期）导致的差异\n4. 给出详细的对比分析和判断结果`;\n\nfunction buildDiffPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n}): string {\n  const {\n    assertion,\n    businessContext,\n    diffThreshold = 0.1,\n    ignoreRegions,\n    ignoreDynamicContent,\n    strictMode,\n  } = options;\n\n  let prompt = `## 图像对比任务\n\n请对比基准图片（预期样式）和当前截图（实际样式），判断页面是否符合预期。\n\n## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  prompt += `\n\n## 对比要求\n\n请从以下维度进行对比分析：\n\n1. **布局一致性**：页面布局是否与基准图片一致\n2. **颜色一致性**：主要颜色是否与基准图片一致\n3. **元素位置**：关键元素的位置是否与基准图片一致\n4. **文字内容**：文字内容是否与基准图片一致\n5. **图片资源**：图片、图标等资源是否正确加载\n\n## 差异阈值\n允许的差异阈值：${diffThreshold * 100}%`;\n\n  if (ignoreRegions && ignoreRegions.length > 0) {\n    prompt += `\n\n## 忽略对比的区域\n以下区域不参与对比：`;\n    ignoreRegions.forEach((region, index) => {\n      prompt += `\n${index + 1}. 位置 (${region.x}, ${region.y})，尺寸 ${region.width}x${region.height}`;\n    });\n  }\n\n  if (ignoreDynamicContent) {\n    prompt += `\n\n## 动态内容处理\n请忽略以下动态内容导致的差异：\n- 时间显示\n- 日期显示\n- 随机验证码\n- 动画效果`;\n  }\n\n  if (strictMode) {\n    prompt += `\n\n## 严格模式\n当前为严格模式，任何差异都视为失败。`;\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出对比结果：\n{\n  \"pass\": boolean,           // 断言是否通过\n  \"thought\": string,         // 对比分析过程\n  \"reason\": string,          // 失败原因（如果失败）\n  \"diffDetails\": {           // 差异详情\n    \"layoutMatch\": boolean,      // 布局是否匹配\n    \"colorMatch\": boolean,       // 颜色是否匹配\n    \"elementPositionMatch\": boolean, // 元素位置是否匹配\n    \"textContentMatch\": boolean, // 文字内容是否匹配\n    \"resourceMatch\": boolean,    // 资源是否匹配\n    \"acceptableDifferences\": string[], // 可接受的差异列表\n    \"unacceptableDifferences\": string[] // 不可接受的差异列表\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertDiff(options: {\n  currentScreenshot: string;\n  referenceImages: ReferenceImage[];\n  assertion: string;\n  businessContext?: string;\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  diffDetails?: DiffDetails;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const {\n    currentScreenshot,\n    referenceImages,\n    assertion,\n    businessContext,\n    diffThreshold,\n    ignoreRegions,\n    ignoreDynamicContent,\n    strictMode,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  // 添加基准图片\n  for (const refImage of referenceImages) {\n    userContent.push({\n      type: 'text',\n      text: `## 基准图片：${refImage.name}`,\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: refImage.url, detail: 'high' },\n    });\n  }\n\n  // 添加当前截图\n  userContent.push({\n    type: 'text',\n    text: '## 当前截图（实际样式）',\n  });\n  userContent.push({\n    type: 'image_url',\n    image_url: { url: currentScreenshot, detail: 'high' },\n  });\n\n  // 添加对比提示\n  userContent.push({\n    type: 'text',\n    text: buildDiffPrompt({\n      assertion,\n      businessContext,\n      diffThreshold,\n      ignoreRegions,\n      ignoreDynamicContent,\n      strictMode,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: DIFF_SYSTEM_PROMPT },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for diff assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<AIAssertionResponse>(\n      msgs,\n      modelConfig,\n      { abortSignal },\n    );\n\n    debug('diff assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      diffDetails: result.content.diffDetails,\n      usage: result.usage,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('diff assertion error:', error);\n    throw error;\n  }\n}\n\nconst VIDEO_SYSTEM_PROMPT = `你是一个自动化测试的视频/动画分析专家。你的任务是对比当前录制的视频与基准视频，判断动画效果是否符合预期。\n\n你需要：\n1. 分析视频中的动画流畅度、时长、完整性\n2. 对比基准视频与当前视频的差异\n3. 识别动画类型（过渡动画、加载动画、交互动画等）\n4. 检测动画质量问题（卡顿、跳帧等）\n5. 给出详细的分析过程和判断结果`;\n\nfunction buildVideoPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  videoOptions?: VideoAssertionOptions;\n}): string {\n  const { assertion, businessContext, videoOptions } = options;\n\n  let prompt = `## 视频/动画对比任务\n\n请对比基准视频（预期动画效果）和当前录制的视频（实际动画效果），判断动画是否符合预期。\n\n## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  prompt += `\n\n## 分析要求\n\n请从以下维度进行动画分析：\n\n1. **动画流畅度**：动画是否流畅，有无卡顿、跳帧\n2. **动画时长**：动画时长是否在预期范围内\n3. **动画完整性**：动画是否完整执行，有无中断\n4. **动画类型**：识别动画类型（过渡、加载、交互等）\n5. **关键帧匹配**：关键时间点的画面是否符合预期`;\n\n  if (videoOptions) {\n    if (videoOptions.checkSmoothness) {\n      prompt += `\n\n## 流畅度检测\n流畅度阈值：${videoOptions.smoothnessThreshold || 60}/100\n请评估动画流畅度并给出评分。`;\n    }\n\n    if (videoOptions.checkDuration && videoOptions.expectedDuration) {\n      prompt += `\n\n## 时长检测\n预期时长范围：${videoOptions.expectedDuration.min || 0}秒 - ${videoOptions.expectedDuration.max || '无限制'}秒`;\n    }\n\n    if (\n      videoOptions.keyframes &&\n      videoOptions.keyframes.timestamps.length > 0\n    ) {\n      prompt += `\n\n## 关键帧验证\n需要在以下时间点验证画面：`;\n      videoOptions.keyframes.timestamps.forEach((ts, index) => {\n        const desc = videoOptions.keyframes?.descriptions?.[index] || '未描述';\n        prompt += `\n- ${ts}秒：${desc}`;\n      });\n    }\n\n    if (videoOptions.animationType && videoOptions.animationType !== 'auto') {\n      prompt += `\n\n## 动画类型\n预期动画类型：${videoOptions.animationType}`;\n    }\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出对比结果：\n{\n  \"pass\": boolean,           // 断言是否通过\n  \"thought\": string,         // 分析过程\n  \"reason\": string,          // 失败原因（如果失败）\n  \"videoDetails\": {          // 视频详情\n    \"smoothnessScore\": number,    // 流畅度评分（0-100）\n    \"duration\": number,           // 动画时长（秒）\n    \"isComplete\": boolean,        // 动画是否完整\n    \"keyframeMatches\": [          // 关键帧匹配结果\n      {\n        \"timestamp\": number,\n        \"matched\": boolean,\n        \"description\": string\n      }\n    ],\n    \"detectedAnimationType\": string,  // 检测到的动画类型\n    \"qualityAssessment\": {            // 质量评估\n      \"hasStuttering\": boolean,       // 是否有卡顿\n      \"hasFrameDropping\": boolean,    // 是否有跳帧\n      \"averageFrameInterval\": number, // 平均帧间隔（ms）\n      \"frameIntervalStdDev\": number   // 帧间隔标准差\n    },\n    \"acceptableDifferences\": string[], // 可接受的差异列表\n    \"unacceptableDifferences\": string[] // 不可接受的差异列表\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertVideo(options: {\n  currentVideoFrames: string[];\n  assertion: string;\n  businessContext?: string;\n  videoOptions?: VideoAssertionOptions;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  videoDetails?: VideoDetails;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const {\n    currentVideoFrames,\n    assertion,\n    businessContext,\n    videoOptions,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const MAX_DURATION = 5;\n  const DEFAULT_FPS = 30;\n  const MAX_FRAMES = MAX_DURATION * DEFAULT_FPS;\n\n  if (currentVideoFrames.length > MAX_FRAMES) {\n    throw new Error(\n      `Video frames exceed maximum limit. Maximum allowed: ${MAX_FRAMES} frames (${MAX_DURATION}s at ${DEFAULT_FPS}fps), got: ${currentVideoFrames.length} frames`,\n    );\n  }\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  userContent.push({\n    type: 'text',\n    text: `## 当前视频（共 ${currentVideoFrames.length} 帧，最大限制 ${MAX_FRAMES} 帧）\n\n以下为当前录制视频的关键帧截图：`,\n  });\n\n  const frameInterval = Math.max(1, Math.floor(currentVideoFrames.length / 10));\n  for (let i = 0; i < currentVideoFrames.length; i += frameInterval) {\n    userContent.push({\n      type: 'text',\n      text: `帧 ${i + 1}/${currentVideoFrames.length}`,\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: currentVideoFrames[i], detail: 'low' },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: buildVideoPrompt({\n      assertion,\n      businessContext,\n      videoOptions,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: VIDEO_SYSTEM_PROMPT },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for video assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<{\n      pass: boolean;\n      thought: string;\n      reason?: string;\n      videoDetails?: VideoDetails;\n    }>(msgs, modelConfig, { abortSignal });\n\n    debug('video assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      videoDetails: result.content.videoDetails,\n      usage: result.usage,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('video assertion error:', error);\n    throw error;\n  }\n}\n"],"names":["debug","getDebug","DEFAULT_SYSTEM_CHECK_PROMPT","buildAssertionPrompt","options","assertion","businessContext","systemCheckPrompt","prompt","AiAssertElement","beforeScreenshot","afterScreenshot","enableSystemCheck","customSystemCheckRules","modelConfig","abortSignal","systemPrompt","userContent","msgs","result","callAIWithObjectResponse","JSON","error","DIFF_SYSTEM_PROMPT","buildDiffPrompt","diffThreshold","ignoreRegions","ignoreDynamicContent","strictMode","region","index","AiAssertDiff","currentScreenshot","referenceImages","refImage","VIDEO_SYSTEM_PROMPT","buildVideoPrompt","videoOptions","ts","desc","AiAssertVideo","currentVideoFrames","MAX_DURATION","DEFAULT_FPS","MAX_FRAMES","Error","frameInterval","Math","i"],"mappings":";;AAcA,MAAMA,QAAQC,SAAS;AA0BvB,MAAMC,8BAA8B,CAAC;;;;;;;;6CAQQ,CAAC;AAE9C,SAASC,qBAAqBC,OAI7B;IACC,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAEC,iBAAiB,EAAE,GAAGH;IAE1D,IAAII,SAAS,CAAC;AAChB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjB,IAAIC,mBACFC,UAAU,CAAC;;;AAGf,EAAED,mBAAmB;IAGnBC,UAAU,CAAC;;;;;;;;;;;;;;;CAeZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAeC,gBAAgBL,OAAwB;IAQ5D,MAAM,EACJM,gBAAgB,EAChBC,eAAe,EACfN,SAAS,EACTC,eAAe,EACfM,oBAAoB,IAAI,EACxBC,sBAAsB,EACtBC,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMY,eAAe,CAAC;;;;;;iBAMP,CAAC;IAEhB,MAAMT,oBAAoBK,oBACtBC,0BAA0BX,8BAC1B;IAEJ,MAAMe,cAAyD,EAAE;IAEjE,IAAIP,kBAAkB;QACpBO,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QACAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKP;gBAAkB,QAAQ;YAAO;QACrD;QACAO,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;IACF,OACEA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM;IACR;IAGFA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YAAE,KAAKN;YAAiB,QAAQ;QAAO;IACpD;IAEAM,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMd,qBAAqB;YACzBE;YACAC;YACAC;QACF;IACF;IAEA,MAAMW,OAAO;QACX;YAAE,MAAM;YAAmB,SAASF;QAAa;QACjD;YAAE,MAAM;YAAiB,SAASC;QAAY;KAC/C;IAEDjB,MAAM,6BAA6BK;IAEnC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBACnBF,MACAJ,aACA;YAAEC;QAAY;QAGhBf,MAAM,qBAAqBmB,OAAO,OAAO;QAEzC,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,OAAOA,OAAO,KAAK;YACnB,oBAAoBA,OAAO,OAAO,CAAC,kBAAkB;YACrD,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,oBAAoBsB;QAC1B,MAAMA;IACR;AACF;AAEA,MAAMC,qBAAqB,CAAC;;;;;;iBAMX,CAAC;AAElB,SAASC,gBAAgBpB,OAOxB;IACC,MAAM,EACJC,SAAS,EACTC,eAAe,EACfmB,gBAAgB,GAAG,EACnBC,aAAa,EACbC,oBAAoB,EACpBC,UAAU,EACX,GAAGxB;IAEJ,IAAII,SAAS,CAAC;;;;;AAKhB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjBE,UAAU,CAAC;;;;;;;;;;;;;QAaL,EAAEiB,AAAgB,MAAhBA,cAAoB,CAAC,CAAC;IAE9B,IAAIC,iBAAiBA,cAAc,MAAM,GAAG,GAAG;QAC7ClB,UAAU,CAAC;;;UAGL,CAAC;QACPkB,cAAc,OAAO,CAAC,CAACG,QAAQC;YAC7BtB,UAAU,CAAC;AACjB,EAAEsB,QAAQ,EAAE,MAAM,EAAED,OAAO,CAAC,CAAC,EAAE,EAAEA,OAAO,CAAC,CAAC,KAAK,EAAEA,OAAO,KAAK,CAAC,CAAC,EAAEA,OAAO,MAAM,EAAE;QAC5E;IACF;IAEA,IAAIF,sBACFnB,UAAU,CAAC;;;;;;;MAOT,CAAC;IAGL,IAAIoB,YACFpB,UAAU,CAAC;;;kBAGG,CAAC;IAGjBA,UAAU,CAAC;;;;;;;;;;;;;;;;;CAiBZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAeuB,aAAa3B,OAWlC;IAQC,MAAM,EACJ4B,iBAAiB,EACjBC,eAAe,EACf5B,SAAS,EACTC,eAAe,EACfmB,aAAa,EACbC,aAAa,EACbC,oBAAoB,EACpBC,UAAU,EACVd,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMa,cAAyD,EAAE;IAGjE,KAAK,MAAMiB,YAAYD,gBAAiB;QACtChB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM,CAAC,QAAQ,EAAEiB,SAAS,IAAI,EAAE;QAClC;QACAjB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKiB,SAAS,GAAG;gBAAE,QAAQ;YAAO;QACjD;IACF;IAGAjB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM;IACR;IACAA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YAAE,KAAKe;YAAmB,QAAQ;QAAO;IACtD;IAGAf,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMO,gBAAgB;YACpBnB;YACAC;YACAmB;YACAC;YACAC;YACAC;QACF;IACF;IAEA,MAAMV,OAAO;QACX;YAAE,MAAM;YAAmB,SAASK;QAAmB;QACvD;YAAE,MAAM;YAAiB,SAASN;QAAY;KAC/C;IAEDjB,MAAM,kCAAkCK;IAExC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBACnBF,MACAJ,aACA;YAAEC;QAAY;QAGhBf,MAAM,0BAA0BmB,OAAO,OAAO;QAE9C,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,aAAaA,OAAO,OAAO,CAAC,WAAW;YACvC,OAAOA,OAAO,KAAK;YACnB,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,yBAAyBsB;QAC/B,MAAMA;IACR;AACF;AAEA,MAAMa,sBAAsB,CAAC;;;;;;;iBAOZ,CAAC;AAElB,SAASC,iBAAiBhC,OAIzB;IACC,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE+B,YAAY,EAAE,GAAGjC;IAErD,IAAII,SAAS,CAAC;;;;;AAKhB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjBE,UAAU,CAAC;;;;;;;;;;2BAUc,CAAC;IAE1B,IAAI6B,cAAc;QAChB,IAAIA,aAAa,eAAe,EAC9B7B,UAAU,CAAC;;;MAGX,EAAE6B,aAAa,mBAAmB,IAAI,GAAG;cACjC,CAAC;QAGX,IAAIA,aAAa,aAAa,IAAIA,aAAa,gBAAgB,EAC7D7B,UAAU,CAAC;;;OAGV,EAAE6B,aAAa,gBAAgB,CAAC,GAAG,IAAI,EAAE,IAAI,EAAEA,aAAa,gBAAgB,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC;QAG/F,IACEA,aAAa,SAAS,IACtBA,aAAa,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,GAC3C;YACA7B,UAAU,CAAC;;;aAGJ,CAAC;YACR6B,aAAa,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAACC,IAAIR;gBAC7C,MAAMS,OAAOF,aAAa,SAAS,EAAE,cAAc,CAACP,MAAM,IAAI;gBAC9DtB,UAAU,CAAC;EACjB,EAAE8B,GAAG,EAAE,EAAEC,MAAM;YACX;QACF;QAEA,IAAIF,aAAa,aAAa,IAAIA,AAA+B,WAA/BA,aAAa,aAAa,EAC1D7B,UAAU,CAAC;;;OAGV,EAAE6B,aAAa,aAAa,EAAE;IAEnC;IAEA7B,UAAU,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAegC,cAAcpC,OAOnC;IAQC,MAAM,EACJqC,kBAAkB,EAClBpC,SAAS,EACTC,eAAe,EACf+B,YAAY,EACZvB,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMsC,eAAe;IACrB,MAAMC,cAAc;IACpB,MAAMC,aAAaF,eAAeC;IAElC,IAAIF,mBAAmB,MAAM,GAAGG,YAC9B,MAAM,IAAIC,MACR,CAAC,oDAAoD,EAAED,WAAW,SAAS,EAAEF,aAAa,KAAK,EAAEC,YAAY,WAAW,EAAEF,mBAAmB,MAAM,CAAC,OAAO,CAAC;IAIhK,MAAMxB,cAAyD,EAAE;IAEjEA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM,CAAC,UAAU,EAAEwB,mBAAmB,MAAM,CAAC,QAAQ,EAAEG,WAAW;;gBAEtD,CAAC;IACf;IAEA,MAAME,gBAAgBC,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACN,mBAAmB,MAAM,GAAG;IACzE,IAAK,IAAIO,IAAI,GAAGA,IAAIP,mBAAmB,MAAM,EAAEO,KAAKF,cAAe;QACjE7B,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM,CAAC,EAAE,EAAE+B,IAAI,EAAE,CAAC,EAAEP,mBAAmB,MAAM,EAAE;QACjD;QACAxB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKwB,kBAAkB,CAACO,EAAE;gBAAE,QAAQ;YAAM;QACzD;IACF;IAEA/B,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMmB,iBAAiB;YACrB/B;YACAC;YACA+B;QACF;IACF;IAEA,MAAMnB,OAAO;QACX;YAAE,MAAM;YAAmB,SAASiB;QAAoB;QACxD;YAAE,MAAM;YAAiB,SAASlB;QAAY;KAC/C;IAEDjB,MAAM,mCAAmCK;IAEzC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBAKlBF,MAAMJ,aAAa;YAAEC;QAAY;QAEpCf,MAAM,2BAA2BmB,OAAO,OAAO;QAE/C,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,cAAcA,OAAO,OAAO,CAAC,YAAY;YACzC,OAAOA,OAAO,KAAK;YACnB,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,0BAA0BsB;QAChC,MAAMA;IACR;AACF"}
+{"version":3,"file":"ai-model/assert.mjs","sources":["../../../src/ai-model/assert.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  DiffDetails,\n  IgnoreRegion,\n  ReferenceImage,\n  SystemCheckResults,\n  VideoAssertionOptions,\n  VideoDetails,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { ChatCompletionUserMessageParam } from 'openai/resources/index';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nconst debug = getDebug('ai:assert');\n\nexport interface AiAssertOptions {\n  beforeScreenshot?: string;\n  afterScreenshot: string;\n  assertion: string;\n  businessContext?: string;\n  enableSystemCheck?: boolean;\n  customSystemCheckRules?: string;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n  referenceImages?: ReferenceImage[];\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  systemCheckResults?: SystemCheckResults;\n  diffDetails?: DiffDetails;\n}\n\nconst DEFAULT_SYSTEM_CHECK_PROMPT = `请检查以下系统级问题：\n\n1. **白屏检测**：页面是否完全白屏或大面积空白\n4. **错误提示**：页面是否显示错误信息、异常提示\n5. **后端错误**：是否有后端请求失败的提示，如\"活动太火爆了\"、报错 Toast、错误码 等\n\n如果发现以上问题，请在 systemCheckResults 中标注对应字段为 true。`;\n\nfunction buildAssertionPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  systemCheckPrompt?: string;\n}): string {\n  const { assertion, businessContext, systemCheckPrompt } = options;\n\n  let prompt = `## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  if (systemCheckPrompt) {\n    prompt += `\n\n## 系统校验规则\n${systemCheckPrompt}`;\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出断言结果：\n{\n  \"pass\": boolean,    // 断言是否通过\n  \"thought\": string,  // 思考过程\n  \"reason\": string,   // 失败原因（如果失败）\n  \"systemCheckResults\": {  // 系统校验结果（可选）\n    \"whiteScreen\": boolean,\n    \"layoutBlocked\": boolean,\n    \"loadingContent\": boolean,\n    \"errorPrompt\": boolean,\n    \"backendError\": boolean\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertElement(options: AiAssertOptions): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  usage?: AIUsageInfo;\n  systemCheckResults?: SystemCheckResults;\n  rawResponse?: string;\n}> {\n  const {\n    beforeScreenshot,\n    afterScreenshot,\n    assertion,\n    businessContext,\n    enableSystemCheck = false,\n    customSystemCheckRules,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const systemPrompt = `你是一个自动化测试断言专家。请根据以下信息判断断言是否通过。\n\n你需要：\n1. 分析操作前后的截图变化（如果提供了操作前截图）\n2. 验证用户的断言描述是否成立\n3. 检查是否存在系统级问题（如果启用）\n4. 给出详细的思考过程和判断结果`;\n\n  const systemCheckPrompt = enableSystemCheck\n    ? customSystemCheckRules || DEFAULT_SYSTEM_CHECK_PROMPT\n    : '';\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (beforeScreenshot) {\n    userContent.push({\n      type: 'text',\n      text: '## 操作前截图（执行操作前的页面状态）',\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: beforeScreenshot, detail: 'high' },\n    });\n    userContent.push({\n      type: 'text',\n      text: '## 操作后截图（执行操作后的页面状态）',\n    });\n  } else {\n    userContent.push({\n      type: 'text',\n      text: '## 当前页面截图',\n    });\n  }\n\n  userContent.push({\n    type: 'image_url',\n    image_url: { url: afterScreenshot, detail: 'high' },\n  });\n\n  userContent.push({\n    type: 'text',\n    text: buildAssertionPrompt({\n      assertion,\n      businessContext,\n      systemCheckPrompt,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: systemPrompt },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<AIAssertionResponse>(\n      msgs,\n      modelConfig,\n      { abortSignal },\n    );\n\n    debug('assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      usage: result.usage,\n      systemCheckResults: result.content.systemCheckResults,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('assertion error:', error);\n    throw error;\n  }\n}\n\nconst DIFF_SYSTEM_PROMPT = `你是一个自动化测试的图像对比专家。你的任务是对比当前页面截图与基准图片，判断页面样式是否符合预期。\n\n你需要：\n1. 仔细对比两张图片的布局、颜色、元素位置\n2. 分析差异是否在可接受范围内\n3. 识别动态内容（如时间、日期）导致的差异\n4. 给出详细的对比分析和判断结果`;\n\nfunction buildDiffPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n}): string {\n  const {\n    assertion,\n    businessContext,\n    diffThreshold = 0.1,\n    ignoreRegions,\n    ignoreDynamicContent,\n    strictMode,\n  } = options;\n\n  let prompt = `## 图像对比任务\n\n请对比基准图片（预期样式）和当前截图（实际样式），判断页面是否符合预期。\n\n## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  prompt += `\n\n## 对比要求\n\n请从以下维度进行对比分析：\n\n1. **布局一致性**：页面布局是否与基准图片一致\n2. **颜色一致性**：主要颜色是否与基准图片一致\n3. **元素位置**：关键元素的位置是否与基准图片一致\n4. **文字内容**：文字内容是否与基准图片一致\n5. **图片资源**：图片、图标等资源是否正确加载\n\n## 差异阈值\n允许的差异阈值：${diffThreshold * 100}%`;\n\n  if (ignoreRegions && ignoreRegions.length > 0) {\n    prompt += `\n\n## 忽略对比的区域\n以下区域不参与对比：`;\n    ignoreRegions.forEach((region, index) => {\n      prompt += `\n${index + 1}. 位置 (${region.x}, ${region.y})，尺寸 ${region.width}x${region.height}`;\n    });\n  }\n\n  if (ignoreDynamicContent) {\n    prompt += `\n\n## 动态内容处理\n请忽略以下动态内容导致的差异：\n- 时间显示\n- 日期显示\n- 随机验证码\n- 动画效果`;\n  }\n\n  if (strictMode) {\n    prompt += `\n\n## 严格模式\n当前为严格模式，任何差异都视为失败。`;\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出对比结果：\n{\n  \"pass\": boolean,           // 断言是否通过\n  \"thought\": string,         // 对比分析过程\n  \"reason\": string,          // 失败原因（如果失败）\n  \"diffDetails\": {           // 差异详情\n    \"layoutMatch\": boolean,      // 布局是否匹配\n    \"colorMatch\": boolean,       // 颜色是否匹配\n    \"elementPositionMatch\": boolean, // 元素位置是否匹配\n    \"textContentMatch\": boolean, // 文字内容是否匹配\n    \"resourceMatch\": boolean,    // 资源是否匹配\n    \"acceptableDifferences\": string[], // 可接受的差异列表\n    \"unacceptableDifferences\": string[] // 不可接受的差异列表\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertDiff(options: {\n  currentScreenshot: string;\n  referenceImages: ReferenceImage[];\n  assertion: string;\n  businessContext?: string;\n  diffThreshold?: number;\n  ignoreRegions?: IgnoreRegion[];\n  ignoreDynamicContent?: boolean;\n  strictMode?: boolean;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  diffDetails?: DiffDetails;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const {\n    currentScreenshot,\n    referenceImages,\n    assertion,\n    businessContext,\n    diffThreshold,\n    ignoreRegions,\n    ignoreDynamicContent,\n    strictMode,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  // 添加基准图片\n  for (const refImage of referenceImages) {\n    userContent.push({\n      type: 'text',\n      text: `## 基准图片：${refImage.name}`,\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: refImage.url, detail: 'high' },\n    });\n  }\n\n  // 添加当前截图\n  userContent.push({\n    type: 'text',\n    text: '## 当前截图（实际样式）',\n  });\n  userContent.push({\n    type: 'image_url',\n    image_url: { url: currentScreenshot, detail: 'high' },\n  });\n\n  // 添加对比提示\n  userContent.push({\n    type: 'text',\n    text: buildDiffPrompt({\n      assertion,\n      businessContext,\n      diffThreshold,\n      ignoreRegions,\n      ignoreDynamicContent,\n      strictMode,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: DIFF_SYSTEM_PROMPT },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for diff assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<AIAssertionResponse>(\n      msgs,\n      modelConfig,\n      { abortSignal },\n    );\n\n    debug('diff assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      diffDetails: result.content.diffDetails,\n      usage: result.usage,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('diff assertion error:', error);\n    throw error;\n  }\n}\n\nconst VIDEO_SYSTEM_PROMPT = `你是一个自动化测试的视频/动画分析专家。你的任务是对比当前录制的视频与基准视频，判断动画效果是否符合预期。\n\n你需要：\n1. 分析视频中的动画流畅度、时长、完整性\n2. 对比基准视频与当前视频的差异\n3. 识别动画类型（过渡动画、加载动画、交互动画等）\n4. 检测动画质量问题（卡顿、跳帧等）\n5. 给出详细的分析过程和判断结果`;\n\nfunction buildVideoPrompt(options: {\n  assertion: string;\n  businessContext?: string;\n  videoOptions?: VideoAssertionOptions;\n}): string {\n  const { assertion, businessContext, videoOptions } = options;\n\n  let prompt = `## 视频/动画对比任务\n\n请对比基准视频（预期动画效果）和当前录制的视频（实际动画效果），判断动画是否符合预期。\n\n## 用户断言描述\n${assertion}`;\n\n  if (businessContext) {\n    prompt += `\n\n## 业务知识上下文\n${businessContext}`;\n  }\n\n  prompt += `\n\n## 分析要求\n\n请从以下维度进行动画分析：\n\n1. **动画流畅度**：动画是否流畅，有无卡顿、跳帧\n2. **动画时长**：动画时长是否在预期范围内\n3. **动画完整性**：动画是否完整执行，有无中断\n4. **动画类型**：识别动画类型（过渡、加载、交互等）\n5. **关键帧匹配**：关键时间点的画面是否符合预期`;\n\n  if (videoOptions) {\n    if (videoOptions.checkSmoothness) {\n      prompt += `\n\n## 流畅度检测\n流畅度阈值：${videoOptions.smoothnessThreshold || 60}/100\n请评估动画流畅度并给出评分。`;\n    }\n\n    if (videoOptions.checkDuration && videoOptions.expectedDuration) {\n      prompt += `\n\n## 时长检测\n预期时长范围：${videoOptions.expectedDuration.min || 0}秒 - ${videoOptions.expectedDuration.max || '无限制'}秒`;\n    }\n\n    if (\n      videoOptions.keyframes &&\n      videoOptions.keyframes.timestamps.length > 0\n    ) {\n      prompt += `\n\n## 关键帧验证\n需要在以下时间点验证画面：`;\n      videoOptions.keyframes.timestamps.forEach((ts, index) => {\n        const desc = videoOptions.keyframes?.descriptions?.[index] || '未描述';\n        prompt += `\n- ${ts}秒：${desc}`;\n      });\n    }\n\n    if (videoOptions.animationType && videoOptions.animationType !== 'auto') {\n      prompt += `\n\n## 动画类型\n预期动画类型：${videoOptions.animationType}`;\n    }\n  }\n\n  prompt += `\n\n## 输出要求\n请以 JSON 格式输出对比结果：\n{\n  \"pass\": boolean,           // 断言是否通过\n  \"thought\": string,         // 分析过程\n  \"reason\": string,          // 失败原因（如果失败）\n  \"videoDetails\": {          // 视频详情\n    \"smoothnessScore\": number,    // 流畅度评分（0-100）\n    \"duration\": number,           // 动画时长（秒）\n    \"isComplete\": boolean,        // 动画是否完整\n    \"keyframeMatches\": [          // 关键帧匹配结果\n      {\n        \"timestamp\": number,\n        \"matched\": boolean,\n        \"description\": string\n      }\n    ],\n    \"detectedAnimationType\": string,  // 检测到的动画类型\n    \"qualityAssessment\": {            // 质量评估\n      \"hasStuttering\": boolean,       // 是否有卡顿\n      \"hasFrameDropping\": boolean,    // 是否有跳帧\n      \"averageFrameInterval\": number, // 平均帧间隔（ms）\n      \"frameIntervalStdDev\": number   // 帧间隔标准差\n    },\n    \"acceptableDifferences\": string[], // 可接受的差异列表\n    \"unacceptableDifferences\": string[] // 不可接受的差异列表\n  }\n}`;\n\n  return prompt;\n}\n\nexport async function AiAssertVideo(options: {\n  currentVideoFrames: string[];\n  assertion: string;\n  businessContext?: string;\n  videoOptions?: VideoAssertionOptions;\n  modelConfig: IModelConfig;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  pass: boolean;\n  thought: string;\n  reason?: string;\n  videoDetails?: VideoDetails;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const {\n    currentVideoFrames,\n    assertion,\n    businessContext,\n    videoOptions,\n    modelConfig,\n    abortSignal,\n  } = options;\n\n  const MAX_DURATION = 5;\n  const DEFAULT_FPS = 30;\n  const MAX_FRAMES = MAX_DURATION * DEFAULT_FPS;\n\n  if (currentVideoFrames.length > MAX_FRAMES) {\n    throw new Error(\n      `Video frames exceed maximum limit. Maximum allowed: ${MAX_FRAMES} frames (${MAX_DURATION}s at ${DEFAULT_FPS}fps), got: ${currentVideoFrames.length} frames`,\n    );\n  }\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  userContent.push({\n    type: 'text',\n    text: `## 当前视频（共 ${currentVideoFrames.length} 帧，最大限制 ${MAX_FRAMES} 帧）\n\n以下为当前录制视频的关键帧截图：`,\n  });\n\n  const frameInterval = Math.max(1, Math.floor(currentVideoFrames.length / 10));\n  for (let i = 0; i < currentVideoFrames.length; i += frameInterval) {\n    userContent.push({\n      type: 'text',\n      text: `帧 ${i + 1}/${currentVideoFrames.length}`,\n    });\n    userContent.push({\n      type: 'image_url',\n      image_url: { url: currentVideoFrames[i], detail: 'low' },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: buildVideoPrompt({\n      assertion,\n      businessContext,\n      videoOptions,\n    }),\n  });\n\n  const msgs = [\n    { role: 'system' as const, content: VIDEO_SYSTEM_PROMPT },\n    { role: 'user' as const, content: userContent },\n  ];\n\n  debug('calling AI for video assertion:', assertion);\n\n  try {\n    const result = await callAIWithObjectResponse<{\n      pass: boolean;\n      thought: string;\n      reason?: string;\n      videoDetails?: VideoDetails;\n    }>(msgs, modelConfig, { abortSignal });\n\n    debug('video assertion result:', result.content);\n\n    return {\n      pass: result.content.pass,\n      thought: result.content.thought || '',\n      reason: result.content.reason,\n      videoDetails: result.content.videoDetails,\n      usage: result.usage,\n      rawResponse: JSON.stringify(result.content),\n    };\n  } catch (error) {\n    debug('video assertion error:', error);\n    throw error;\n  }\n}\n"],"names":["debug","getDebug","DEFAULT_SYSTEM_CHECK_PROMPT","buildAssertionPrompt","options","assertion","businessContext","systemCheckPrompt","prompt","AiAssertElement","beforeScreenshot","afterScreenshot","enableSystemCheck","customSystemCheckRules","modelConfig","abortSignal","systemPrompt","userContent","msgs","result","callAIWithObjectResponse","JSON","error","DIFF_SYSTEM_PROMPT","buildDiffPrompt","diffThreshold","ignoreRegions","ignoreDynamicContent","strictMode","region","index","AiAssertDiff","currentScreenshot","referenceImages","refImage","VIDEO_SYSTEM_PROMPT","buildVideoPrompt","videoOptions","ts","desc","AiAssertVideo","currentVideoFrames","MAX_DURATION","DEFAULT_FPS","MAX_FRAMES","Error","frameInterval","Math","i"],"mappings":";;AAcA,MAAMA,QAAQC,SAAS;AA0BvB,MAAMC,8BAA8B,CAAC;;;;;;6CAMQ,CAAC;AAE9C,SAASC,qBAAqBC,OAI7B;IACC,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAEC,iBAAiB,EAAE,GAAGH;IAE1D,IAAII,SAAS,CAAC;AAChB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjB,IAAIC,mBACFC,UAAU,CAAC;;;AAGf,EAAED,mBAAmB;IAGnBC,UAAU,CAAC;;;;;;;;;;;;;;;CAeZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAeC,gBAAgBL,OAAwB;IAQ5D,MAAM,EACJM,gBAAgB,EAChBC,eAAe,EACfN,SAAS,EACTC,eAAe,EACfM,oBAAoB,KAAK,EACzBC,sBAAsB,EACtBC,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMY,eAAe,CAAC;;;;;;iBAMP,CAAC;IAEhB,MAAMT,oBAAoBK,oBACtBC,0BAA0BX,8BAC1B;IAEJ,MAAMe,cAAyD,EAAE;IAEjE,IAAIP,kBAAkB;QACpBO,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QACAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKP;gBAAkB,QAAQ;YAAO;QACrD;QACAO,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;IACF,OACEA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM;IACR;IAGFA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YAAE,KAAKN;YAAiB,QAAQ;QAAO;IACpD;IAEAM,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMd,qBAAqB;YACzBE;YACAC;YACAC;QACF;IACF;IAEA,MAAMW,OAAO;QACX;YAAE,MAAM;YAAmB,SAASF;QAAa;QACjD;YAAE,MAAM;YAAiB,SAASC;QAAY;KAC/C;IAEDjB,MAAM,6BAA6BK;IAEnC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBACnBF,MACAJ,aACA;YAAEC;QAAY;QAGhBf,MAAM,qBAAqBmB,OAAO,OAAO;QAEzC,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,OAAOA,OAAO,KAAK;YACnB,oBAAoBA,OAAO,OAAO,CAAC,kBAAkB;YACrD,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,oBAAoBsB;QAC1B,MAAMA;IACR;AACF;AAEA,MAAMC,qBAAqB,CAAC;;;;;;iBAMX,CAAC;AAElB,SAASC,gBAAgBpB,OAOxB;IACC,MAAM,EACJC,SAAS,EACTC,eAAe,EACfmB,gBAAgB,GAAG,EACnBC,aAAa,EACbC,oBAAoB,EACpBC,UAAU,EACX,GAAGxB;IAEJ,IAAII,SAAS,CAAC;;;;;AAKhB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjBE,UAAU,CAAC;;;;;;;;;;;;;QAaL,EAAEiB,AAAgB,MAAhBA,cAAoB,CAAC,CAAC;IAE9B,IAAIC,iBAAiBA,cAAc,MAAM,GAAG,GAAG;QAC7ClB,UAAU,CAAC;;;UAGL,CAAC;QACPkB,cAAc,OAAO,CAAC,CAACG,QAAQC;YAC7BtB,UAAU,CAAC;AACjB,EAAEsB,QAAQ,EAAE,MAAM,EAAED,OAAO,CAAC,CAAC,EAAE,EAAEA,OAAO,CAAC,CAAC,KAAK,EAAEA,OAAO,KAAK,CAAC,CAAC,EAAEA,OAAO,MAAM,EAAE;QAC5E;IACF;IAEA,IAAIF,sBACFnB,UAAU,CAAC;;;;;;;MAOT,CAAC;IAGL,IAAIoB,YACFpB,UAAU,CAAC;;;kBAGG,CAAC;IAGjBA,UAAU,CAAC;;;;;;;;;;;;;;;;;CAiBZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAeuB,aAAa3B,OAWlC;IAQC,MAAM,EACJ4B,iBAAiB,EACjBC,eAAe,EACf5B,SAAS,EACTC,eAAe,EACfmB,aAAa,EACbC,aAAa,EACbC,oBAAoB,EACpBC,UAAU,EACVd,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMa,cAAyD,EAAE;IAGjE,KAAK,MAAMiB,YAAYD,gBAAiB;QACtChB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM,CAAC,QAAQ,EAAEiB,SAAS,IAAI,EAAE;QAClC;QACAjB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKiB,SAAS,GAAG;gBAAE,QAAQ;YAAO;QACjD;IACF;IAGAjB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM;IACR;IACAA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YAAE,KAAKe;YAAmB,QAAQ;QAAO;IACtD;IAGAf,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMO,gBAAgB;YACpBnB;YACAC;YACAmB;YACAC;YACAC;YACAC;QACF;IACF;IAEA,MAAMV,OAAO;QACX;YAAE,MAAM;YAAmB,SAASK;QAAmB;QACvD;YAAE,MAAM;YAAiB,SAASN;QAAY;KAC/C;IAEDjB,MAAM,kCAAkCK;IAExC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBACnBF,MACAJ,aACA;YAAEC;QAAY;QAGhBf,MAAM,0BAA0BmB,OAAO,OAAO;QAE9C,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,aAAaA,OAAO,OAAO,CAAC,WAAW;YACvC,OAAOA,OAAO,KAAK;YACnB,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,yBAAyBsB;QAC/B,MAAMA;IACR;AACF;AAEA,MAAMa,sBAAsB,CAAC;;;;;;;iBAOZ,CAAC;AAElB,SAASC,iBAAiBhC,OAIzB;IACC,MAAM,EAAEC,SAAS,EAAEC,eAAe,EAAE+B,YAAY,EAAE,GAAGjC;IAErD,IAAII,SAAS,CAAC;;;;;AAKhB,EAAEH,WAAW;IAEX,IAAIC,iBACFE,UAAU,CAAC;;;AAGf,EAAEF,iBAAiB;IAGjBE,UAAU,CAAC;;;;;;;;;;2BAUc,CAAC;IAE1B,IAAI6B,cAAc;QAChB,IAAIA,aAAa,eAAe,EAC9B7B,UAAU,CAAC;;;MAGX,EAAE6B,aAAa,mBAAmB,IAAI,GAAG;cACjC,CAAC;QAGX,IAAIA,aAAa,aAAa,IAAIA,aAAa,gBAAgB,EAC7D7B,UAAU,CAAC;;;OAGV,EAAE6B,aAAa,gBAAgB,CAAC,GAAG,IAAI,EAAE,IAAI,EAAEA,aAAa,gBAAgB,CAAC,GAAG,IAAI,MAAM,CAAC,CAAC;QAG/F,IACEA,aAAa,SAAS,IACtBA,aAAa,SAAS,CAAC,UAAU,CAAC,MAAM,GAAG,GAC3C;YACA7B,UAAU,CAAC;;;aAGJ,CAAC;YACR6B,aAAa,SAAS,CAAC,UAAU,CAAC,OAAO,CAAC,CAACC,IAAIR;gBAC7C,MAAMS,OAAOF,aAAa,SAAS,EAAE,cAAc,CAACP,MAAM,IAAI;gBAC9DtB,UAAU,CAAC;EACjB,EAAE8B,GAAG,EAAE,EAAEC,MAAM;YACX;QACF;QAEA,IAAIF,aAAa,aAAa,IAAIA,AAA+B,WAA/BA,aAAa,aAAa,EAC1D7B,UAAU,CAAC;;;OAGV,EAAE6B,aAAa,aAAa,EAAE;IAEnC;IAEA7B,UAAU,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6BZ,CAAC;IAEA,OAAOA;AACT;AAEO,eAAegC,cAAcpC,OAOnC;IAQC,MAAM,EACJqC,kBAAkB,EAClBpC,SAAS,EACTC,eAAe,EACf+B,YAAY,EACZvB,WAAW,EACXC,WAAW,EACZ,GAAGX;IAEJ,MAAMsC,eAAe;IACrB,MAAMC,cAAc;IACpB,MAAMC,aAAaF,eAAeC;IAElC,IAAIF,mBAAmB,MAAM,GAAGG,YAC9B,MAAM,IAAIC,MACR,CAAC,oDAAoD,EAAED,WAAW,SAAS,EAAEF,aAAa,KAAK,EAAEC,YAAY,WAAW,EAAEF,mBAAmB,MAAM,CAAC,OAAO,CAAC;IAIhK,MAAMxB,cAAyD,EAAE;IAEjEA,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAM,CAAC,UAAU,EAAEwB,mBAAmB,MAAM,CAAC,QAAQ,EAAEG,WAAW;;gBAEtD,CAAC;IACf;IAEA,MAAME,gBAAgBC,KAAK,GAAG,CAAC,GAAGA,KAAK,KAAK,CAACN,mBAAmB,MAAM,GAAG;IACzE,IAAK,IAAIO,IAAI,GAAGA,IAAIP,mBAAmB,MAAM,EAAEO,KAAKF,cAAe;QACjE7B,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM,CAAC,EAAE,EAAE+B,IAAI,EAAE,CAAC,EAAEP,mBAAmB,MAAM,EAAE;QACjD;QACAxB,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBAAE,KAAKwB,kBAAkB,CAACO,EAAE;gBAAE,QAAQ;YAAM;QACzD;IACF;IAEA/B,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMmB,iBAAiB;YACrB/B;YACAC;YACA+B;QACF;IACF;IAEA,MAAMnB,OAAO;QACX;YAAE,MAAM;YAAmB,SAASiB;QAAoB;QACxD;YAAE,MAAM;YAAiB,SAASlB;QAAY;KAC/C;IAEDjB,MAAM,mCAAmCK;IAEzC,IAAI;QACF,MAAMc,SAAS,MAAMC,yBAKlBF,MAAMJ,aAAa;YAAEC;QAAY;QAEpCf,MAAM,2BAA2BmB,OAAO,OAAO;QAE/C,OAAO;YACL,MAAMA,OAAO,OAAO,CAAC,IAAI;YACzB,SAASA,OAAO,OAAO,CAAC,OAAO,IAAI;YACnC,QAAQA,OAAO,OAAO,CAAC,MAAM;YAC7B,cAAcA,OAAO,OAAO,CAAC,YAAY;YACzC,OAAOA,OAAO,KAAK;YACnB,aAAaE,KAAK,SAAS,CAACF,OAAO,OAAO;QAC5C;IACF,EAAE,OAAOG,OAAO;QACdtB,MAAM,0BAA0BsB;QAChC,MAAMA;IACR;AACF"}

package/dist/es/utils.mjs

@@ -159,7 +159,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "1.6.7";
+    return "1.6.9";
 }
 function debugLog(...message) {
     const debugMode = process.env[MIDSCENE_DEBUG_MODE];

package/dist/lib/agent/agent.js

@@ -582,7 +582,7 @@ class Agent {
                 afterScreenshot,
                 assertion: textPrompt,
                 businessContext: opt?.businessContext,
-                enableSystemCheck: opt?.enableSystemCheck ?? true,
+                enableSystemCheck: opt?.enableSystemCheck ?? false,
                 customSystemCheckRules: opt?.customSystemCheckRules,
                 modelConfig
             });