npm - @donggui/core - Versions diffs - 1.6.10 → 1.6.11 - Mend

@donggui/core 1.6.10 → 1.6.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (16) hide show

package/dist/es/agent/agent.mjs +1 -1
package/dist/es/agent/agent.mjs.map +1 -1
package/dist/es/agent/task-builder.mjs +9 -0
package/dist/es/agent/task-builder.mjs.map +1 -1
package/dist/es/agent/utils.mjs +1 -1
package/dist/es/types.mjs.map +1 -1
package/dist/es/utils.mjs +1 -1
package/dist/lib/agent/agent.js +1 -1
package/dist/lib/agent/agent.js.map +1 -1
package/dist/lib/agent/task-builder.js +9 -0
package/dist/lib/agent/task-builder.js.map +1 -1
package/dist/lib/agent/utils.js +1 -1
package/dist/lib/types.js.map +1 -1
package/dist/lib/utils.js +1 -1
package/dist/types/types.d.ts +2 -2
package/package.json +3 -2

package/dist/es/agent/task-builder.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import {\n calculateConfidence,\n createInitialConfidenceState,\n createInitialProgressiveRecord,\n determineVerificationLevel,\n getVerificationActions,\n updateConfidenceOnVerify,\n updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n CacheConfidenceState,\n CacheValidationOptions,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n PlanningLocateParam,\n ProgressiveLocateRecord,\n Rect,\n SemanticAnchor,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n ifPlanLocateParamIsBbox,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n /\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = typeof param === 'string' ? { prompt: param } : param;\n const locatePlan: PlanningAction<PlanningLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n modelConfigForPlanning: IModelConfig;\n modelConfigForDefaultIntent: IModelConfig;\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n\n constructor({\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n }\n\n public async build(\n plans: PlanningAction[],\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n cacheable,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<PlanningLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<PlanningLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = actionSpace.find((item) => item.name === planType);\n const param = plan.param;\n\n if (!action) {\n throw new Error(`Action type '${planType}' not found`);\n }\n\n const locateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema)\n : [];\n\n const requiredLocateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema, true)\n : [];\n\n locateFields.forEach((field) => {\n if (param[field]) {\n // Always use createLocateTask for all locate params (including bbox)\n // This ensures cache writing happens even when bbox is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n executor: async (param, taskContext) => {\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug(\n `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.beforeInvokeAction(action.name, param);\n debug(\n `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n })(),\n sleep(200),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n\n if (action.paramSchema) {\n try {\n param = parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n }\n\n setTimingFieldOnce(timing, 'callActionStart');\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(param, taskContext);\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug(\n `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.afterInvokeAction(action.name, param);\n debug(\n `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<PlanningLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n context;\n\n let locateParam = detailedLocateParam;\n\n if (typeof locateParam === 'string') {\n locateParam = {\n prompt: locateParam,\n };\n }\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n\n assert(\n param?.prompt \|\| param?.bbox,\n `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n param,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n };\n task.usage = dump.taskInfo?.usage;\n if (dump.taskInfo?.searchAreaUsage) {\n task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n // from bbox (plan hit)\n const elementFromBbox = ifPlanLocateParamIsBbox(param)\n ? matchElementFromPlan(param)\n : undefined;\n const isPlanHit = !!elementFromBbox;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanHit &&\n param.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [param.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? generateElementByRect(\n // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n transformLogicalRectToScreenshotRect(\n rectFromXpath,\n shrunkShotToLogicalRatio,\n ),\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt \|\| '',\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = param.prompt;\n const locateCacheRecord =\n await this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n param.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n let elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n let isCacheHit = !!elementFromCache;\n let isLegacyCache = false;\n const timing = taskContext.task.timing;\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n\n if (isCacheHit) {\n const cacheFeature = cacheEntry as ElementCacheFeature;\n const cachedCenter = cacheFeature?.cachedCenter as\n \| [number, number]\n \| undefined;\n\n isLegacyCache = !cachedCenter;\n\n const enableCoordCheck =\n process.env[MIDSCENE_CACHE_ENABLE_COORD_CHECK] === 'true';\n const enableVisualVerify =\n process.env[MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY] === 'true';\n\n const coordOffsetThreshold =\n Number.parseInt(\n process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] \|\| '16',\n 10,\n ) \|\| 16;\n\n if (enableCoordCheck \|\| enableVisualVerify) {\n const confidenceState = (cacheFeature?.confidenceState \|\|\n createInitialConfidenceState()) as CacheConfidenceState;\n const confidence = isLegacyCache\n ? 0.35\n : calculateConfidence(confidenceState);\n const level = determineVerificationLevel(confidence);\n const actions = getVerificationActions(level);\n\n debug('cache confidence assessment', {\n confidence: confidence.toFixed(3),\n level,\n actions,\n verificationCount: confidenceState.verificationCount,\n isLegacyCache,\n });\n\n if (actions.skipCache) {\n debug('cache confidence too low, skipping cache entirely', {\n confidence,\n level,\n });\n isCacheHit = false;\n }\n\n try {\n if (\n isCacheHit &&\n enableCoordCheck &&\n actions.coordCheck &&\n cachedCenter\n ) {\n const offset = Math.sqrt(\n (elementFromCache!.center[0] - cachedCenter[0]) * 2 +\n (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n );\n\n debug('cache coord offset check', {\n cachedCenter,\n currentCenter: elementFromCache!.center,\n offset: Math.round(offset),\n threshold: coordOffsetThreshold,\n });\n\n if (offset > coordOffsetThreshold) {\n debug(\n 'cache coord offset exceeded threshold, fallback to AI locate',\n { offset, threshold: coordOffsetThreshold },\n );\n isCacheHit = false;\n }\n }\n\n if (isCacheHit && enableVisualVerify && actions.visualVerify) {\n const verification = await this.service.verifyCachedElement(\n elementFromCache!.center,\n cachePrompt,\n modelConfigForDefaultIntent,\n uiContext,\n );\n if (!verification.pass) {\n debug(\n 'cache hit but visual verification failed, fallback to AI locate',\n {\n reason: verification.reason,\n description: verification.description,\n prompt: cachePrompt,\n },\n );\n isCacheHit = false;\n } else {\n debug('cache hit and visual verification passed', {\n description: verification.description,\n });\n }\n }\n } catch (verifyError) {\n debug(\n 'cache verification error, fallback to AI locate',\n verifyError,\n );\n isCacheHit = false;\n }\n\n if (isCacheHit) {\n const updatedState = updateConfidenceOnVerify(\n confidenceState,\n true,\n );\n cacheFeature.confidenceState = updatedState;\n\n if (isLegacyCache) {\n cacheFeature.cachedCenter = elementFromCache!.center;\n cacheFeature.progressiveRecord = createInitialProgressiveRecord(\n elementFromCache!.center,\n );\n debug('legacy cache upgraded with new fields', {\n cachedCenter: cacheFeature.cachedCenter,\n });\n }\n\n const progressiveRecord = cacheFeature.progressiveRecord as\n \| ProgressiveLocateRecord\n \| undefined;\n if (progressiveRecord) {\n const updated = updateProgressiveConvergence(\n progressiveRecord,\n elementFromCache!.center,\n updatedState.confidenceScore,\n );\n cacheFeature.progressiveRecord = updated;\n\n if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n debug(\n 'using converged center instead of single-result center',\n {\n convergedCenter: updated.convergedCenter.map((v) =>\n v.toFixed(1),\n ),\n singleCenter: elementFromCache!.center,\n convergenceRadius: updated.convergenceRadius.toFixed(1),\n sampleCount: updated.sampleCount,\n },\n );\n elementFromCache = {\n ...elementFromCache!,\n center: [\n Math.round(updated.convergedCenter[0]),\n Math.round(updated.convergedCenter[1]),\n ],\n };\n }\n }\n } else {\n const updatedState = updateConfidenceOnVerify(\n confidenceState,\n false,\n );\n cacheFeature.confidenceState = updatedState;\n }\n }\n }\n\n if (!isXpathHit && !isCacheHit && !isPlanHit) {\n const cacheFeature = cacheEntry as ElementCacheFeature;\n const semanticAnchor = cacheFeature?.semanticAnchor as\n \| SemanticAnchor\n \| undefined;\n\n if (\n semanticAnchor &&\n process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true'\n ) {\n try {\n const anchorResult = await this.service.locateBySemanticAnchor(\n semanticAnchor,\n modelConfigForDefaultIntent,\n this.interface,\n uiContext,\n );\n if (anchorResult) {\n elementFromAiLocate = anchorResult;\n debug(\n 'semantic anchor locate succeeded, skipping full AI locate',\n );\n }\n } catch (anchorError) {\n debug('semantic anchor locate failed:', anchorError);\n }\n }\n\n if (!elementFromAiLocate) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n param,\n {\n context: uiContext,\n },\n modelConfigForDefaultIntent,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n }\n\n const element =\n elementFromBbox \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read), OR cache hit but legacy cache needs upgrade\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n (!isCacheHit \|\| isLegacyCache) &&\n (!isPlanHit \|\| !locateCacheAlreadyExists) &&\n param?.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n let pointForCache: [number, number] = element.center;\n if (shrunkShotToLogicalRatio !== 1) {\n pointForCache = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n debug(\n 'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n element.center,\n pointForCache,\n );\n }\n\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt,\n modelConfig: modelConfigForDefaultIntent,\n },\n );\n if (hasNonEmptyCache(feature)) {\n feature.cachedCenter = pointForCache;\n feature.confidenceState = createInitialConfidenceState();\n feature.progressiveRecord =\n createInitialProgressiveRecord(pointForCache);\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n\n const enableSemanticAnchor =\n process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true';\n if (enableSemanticAnchor) {\n try {\n const anchor = await this.service.generateSemanticAnchor(\n pointForCache,\n modelConfigForDefaultIntent,\n uiContext,\n );\n if (anchor) {\n feature.semanticAnchor = anchor;\n debug(\n 'semantic anchor generated for prompt: %s',\n cachePrompt,\n );\n }\n } catch (anchorError) {\n debug('generateSemanticAnchor failed:', anchorError);\n }\n }\n\n currentCacheEntry = feature;\n await this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${param.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${param.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanHit) {\n hitBy = {\n from: 'Plan',\n context: {\n bbox: param.bbox,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: param.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n onResult?.(element);\n\n return {\n output: {\n element: {\n ...element,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","isLegacyCache","elementFromAiLocate","cacheFeature","cachedCenter","enableCoordCheck","process","MIDSCENE_CACHE_ENABLE_COORD_CHECK","enableVisualVerify","MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY","coordOffsetThreshold","Number","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","createInitialProgressiveRecord","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqDA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC3C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,MAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,iBAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,mBAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,wBAAwB5B,SAC5C4D,qBAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,oCACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,IAAIG,gBAAgB;gBACpB,MAAMzC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI2C;gBAEJ,IAAIF,YAAY;oBACd,MAAMG,eAAeR;oBACrB,MAAMS,eAAeD,cAAc;oBAInCF,gBAAgB,CAACG;oBAEjB,MAAMC,mBACJC,AAAmD,WAAnDA,QAAQ,GAAG,CAACC,kCAAkC;oBAChD,MAAMC,qBACJF,AAAqD,WAArDA,QAAQ,GAAG,CAACG,oCAAoC;oBAElD,MAAMC,uBACJC,OAAO,QAAQ,CACbL,QAAQ,GAAG,CAACM,sCAAsC,IAAI,MACtD,OACG;oBAEP,IAAIP,oBAAoBG,oBAAoB;wBAC1C,MAAMK,kBAAmBV,cAAc,mBACrCW;wBACF,MAAMC,aAAad,gBACf,OACAe,oBAAoBH;wBACxB,MAAMI,QAAQC,2BAA2BH;wBACzC,MAAMI,UAAUC,uBAAuBH;wBAEvCjG,MAAM,+BAA+B;4BACnC,YAAY+F,WAAW,OAAO,CAAC;4BAC/BE;4BACAE;4BACA,mBAAmBN,gBAAgB,iBAAiB;4BACpDZ;wBACF;wBAEA,IAAIkB,QAAQ,SAAS,EAAE;4BACrBnG,MAAM,qDAAqD;gCACzD+F;gCACAE;4BACF;4BACAjB,aAAa;wBACf;wBAEA,IAAI;4BACF,IACEA,cACAK,oBACAc,QAAQ,UAAU,IAClBf,cACA;gCACA,MAAMiB,SAASC,KAAK,IAAI,CACrBxB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM,IAChDN,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM;gCAGvDpF,MAAM,4BAA4B;oCAChCoF;oCACA,eAAeN,iBAAkB,MAAM;oCACvC,QAAQwB,KAAK,KAAK,CAACD;oCACnB,WAAWX;gCACb;gCAEA,IAAIW,SAASX,sBAAsB;oCACjC1F,MACE,gEACA;wCAAEqG;wCAAQ,WAAWX;oCAAqB;oCAE5CV,aAAa;gCACf;4BACF;4BAEA,IAAIA,cAAcQ,sBAAsBW,QAAQ,YAAY,EAAE;gCAC5D,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDzB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;gCAEF,IAAK8D,aAAa,IAAI,EAWpBvG,MAAM,4CAA4C;oCAChD,aAAauG,aAAa,WAAW;gCACvC;qCAbsB;oCACtBvG,MACE,mEACA;wCACE,QAAQuG,aAAa,MAAM;wCAC3B,aAAaA,aAAa,WAAW;wCACrC,QAAQ9B;oCACV;oCAEFO,aAAa;gCACf;4BAKF;wBACF,EAAE,OAAOwB,aAAa;4BACpBxG,MACE,mDACAwG;4BAEFxB,aAAa;wBACf;wBAEA,IAAIA,YAAY;4BACd,MAAMyB,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;4BAE/B,IAAIxB,eAAe;gCACjBE,aAAa,YAAY,GAAGL,iBAAkB,MAAM;gCACpDK,aAAa,iBAAiB,GAAGwB,+BAC/B7B,iBAAkB,MAAM;gCAE1B9E,MAAM,yCAAyC;oCAC7C,cAAcmF,aAAa,YAAY;gCACzC;4BACF;4BAEA,MAAMyB,oBAAoBzB,aAAa,iBAAiB;4BAGxD,IAAIyB,mBAAmB;gCACrB,MAAMC,UAAUC,6BACdF,mBACA9B,iBAAkB,MAAM,EACxB2B,aAAa,eAAe;gCAE9BtB,aAAa,iBAAiB,GAAG0B;gCAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;oCAC7D7G,MACE,0DACA;wCACE,iBAAiB6G,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;wCAEZ,cAAcjC,iBAAkB,MAAM;wCACtC,mBAAmB+B,QAAQ,iBAAiB,CAAC,OAAO,CAAC;wCACrD,aAAaA,QAAQ,WAAW;oCAClC;oCAEF/B,mBAAmB;wCACjB,GAAGA,gBAAgB;wCACnB,QAAQ;4CACNwB,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;4CACrCP,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;yCACtC;oCACH;gCACF;4BACF;wBACF,OAAO;4BACL,MAAMJ,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;wBACjC;oBACF;gBACF;gBAEA,IAAI,CAACjC,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMgB,eAAeR;oBACrB,MAAMqC,iBAAiB7B,cAAc;oBAIrC,IACE6B,kBACA1B,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACApG,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAIyE,cAAc;4BAChBhC,sBAAsBgC;4BACtBlH,MACE;wBAEJ;oBACF,EAAE,OAAOmH,aAAa;wBACpBnH,MAAM,kCAAkCmH;oBAC1C;oBAGF,IAAI,CAACjC,qBACH,IAAI;wBACFxC,mBAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BoB,sBAAsBpB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiBiE,cACnBrD,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,mBAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAM6E,UACJpD,mBACAI,oBACAS,oBACAI;gBAGF,MAAMoC,2BAA2BpH,iBAC/BwE,mBAAmB,cAAc;gBAGnC,IAAI6C;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACb,EAACrC,cAAcC,aAAY,KAC3B,EAACd,aAAa,CAACmD,wBAAuB,KACvChH,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIkH,gBAAkCH,QAAQ,MAAM;oBACpD,IAAIrE,AAA6B,MAA7BA,0BAAgC;wBAClCwE,gBAAgB;4BACdlB,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;4BAC/BsD,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;yBAChC;wBACDhD,MACE,8DACAqH,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOlH,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiBuH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG3B;wBAC1B2B,QAAQ,iBAAiB,GACvBd,+BAA+Ba;wBACjCxH,MACE,uCACAyE,aACAgD;wBAGF,MAAMC,uBACJpC,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC;wBACpD,IAAIS,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDH,eACA5G,6BACA6B;4BAEF,IAAIkF,QAAQ;gCACVF,QAAQ,cAAc,GAAGE;gCACzB3H,MACE,4CACAyE;4BAEJ;wBACF,EAAE,OAAO0C,aAAa;4BACpBnH,MAAM,kCAAkCmH;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQhD;4BACR,OAAOgD;wBACT,GACA/C;oBAEJ,OACE1E,MACE,yDACAyE;gBAGN,EAAE,OAAOtB,OAAO;oBACdnD,MAAM,mCAAmCmD;gBAC3C;qBAEAnD,MAAM;gBAIV,IAAI,CAACqH,SAAS;oBACZ,IAAIxD,YACF,MAAM,IAAIuD,aACR,CAAC,oBAAoB,EAAE9G,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIsH;gBAEJ,IAAIzD,WACFyD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMtH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACToD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOtH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACT4C,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPjD;wBACA,aAAa4C;oBACf;gBACF;gBAGF/D,WAAW6D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK5E,UAAU,aAAa;wBAC9B;oBACF;oBACAmF;gBACF;YACF;QACF;QAEA,OAAOhE;IACT;IAzwBA,YAAY,EACViE,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTtG,WAAW,EACXuG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGtG;QACnB,IAAI,CAAC,eAAe,GAAGuG;IACzB;AA8vBF"}
1	+ {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import {\n calculateConfidence,\n createInitialConfidenceState,\n createInitialProgressiveRecord,\n determineVerificationLevel,\n getVerificationActions,\n updateConfidenceOnVerify,\n updateProgressiveConvergence,\n} from '@/agent/cache-confidence';\nimport { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n CacheConfidenceState,\n CacheValidationOptions,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n PlanningLocateParam,\n ProgressiveLocateRecord,\n Rect,\n SemanticAnchor,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD,\n MIDSCENE_CACHE_ENABLE_COORD_CHECK,\n MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR,\n MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY,\n} from '@midscene/shared/env/constants';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n ifPlanLocateParamIsBbox,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n /\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = typeof param === 'string' ? { prompt: param } : param;\n const locatePlan: PlanningAction<PlanningLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n modelConfigForPlanning: IModelConfig;\n modelConfigForDefaultIntent: IModelConfig;\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n\n constructor({\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n }\n\n public async build(\n plans: PlanningAction[],\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n cacheable,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<PlanningLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<PlanningLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = actionSpace.find((item) => item.name === planType);\n const param = plan.param;\n\n if (!action) {\n throw new Error(`Action type '${planType}' not found`);\n }\n\n const locateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema)\n : [];\n\n const requiredLocateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema, true)\n : [];\n\n locateFields.forEach((field) => {\n if (param[field]) {\n // Always use createLocateTask for all locate params (including bbox)\n // This ensures cache writing happens even when bbox is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n executor: async (param, taskContext) => {\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug(\n `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.beforeInvokeAction(action.name, param);\n debug(\n `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n })(),\n sleep(200),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n\n if (action.paramSchema) {\n try {\n param = parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n }\n\n setTimingFieldOnce(timing, 'callActionStart');\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(param, taskContext);\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug(\n `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.afterInvokeAction(action.name, param);\n debug(\n `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<PlanningLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n context;\n\n let locateParam = detailedLocateParam;\n\n if (typeof locateParam === 'string') {\n locateParam = {\n prompt: locateParam,\n };\n }\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n\n assert(\n param?.prompt \|\| param?.bbox,\n `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n param,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n };\n task.usage = dump.taskInfo?.usage;\n if (dump.taskInfo?.searchAreaUsage) {\n task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n // from bbox (plan hit)\n const elementFromBbox = ifPlanLocateParamIsBbox(param)\n ? matchElementFromPlan(param)\n : undefined;\n const isPlanHit = !!elementFromBbox;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanHit &&\n param.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [param.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? generateElementByRect(\n // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n transformLogicalRectToScreenshotRect(\n rectFromXpath,\n shrunkShotToLogicalRatio,\n ),\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt \|\| '',\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = param.prompt;\n const locateCacheRecord =\n await this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n param.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n let elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n let isCacheHit = !!elementFromCache;\n let isLegacyCache = false;\n const timing = taskContext.task.timing;\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n\n if (isCacheHit) {\n const cacheFeature = cacheEntry as ElementCacheFeature;\n const cachedCenter = cacheFeature?.cachedCenter as\n \| [number, number]\n \| undefined;\n\n isLegacyCache = !cachedCenter;\n\n const enableCoordCheck =\n process.env[MIDSCENE_CACHE_ENABLE_COORD_CHECK] === 'true';\n const enableVisualVerify =\n process.env[MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY] === 'true';\n\n const coordOffsetThreshold =\n Number.parseInt(\n process.env[MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD] \|\| '16',\n 10,\n ) \|\| 16;\n\n if (enableCoordCheck \|\| enableVisualVerify) {\n const confidenceState = (cacheFeature?.confidenceState \|\|\n createInitialConfidenceState()) as CacheConfidenceState;\n const confidence = isLegacyCache\n ? 0.35\n : calculateConfidence(confidenceState);\n const level = determineVerificationLevel(confidence);\n const actions = getVerificationActions(level);\n\n debug('cache confidence assessment', {\n confidence: confidence.toFixed(3),\n level,\n actions,\n verificationCount: confidenceState.verificationCount,\n isLegacyCache,\n });\n\n if (actions.skipCache) {\n debug('cache confidence too low, skipping cache entirely', {\n confidence,\n level,\n });\n isCacheHit = false;\n elementFromCache = undefined;\n }\n\n try {\n if (\n isCacheHit &&\n enableCoordCheck &&\n actions.coordCheck &&\n cachedCenter\n ) {\n const offset = Math.sqrt(\n (elementFromCache!.center[0] - cachedCenter[0]) * 2 +\n (elementFromCache!.center[1] - cachedCenter[1]) ** 2,\n );\n\n debug('cache coord offset check', {\n cachedCenter,\n currentCenter: elementFromCache!.center,\n offset: Math.round(offset),\n threshold: coordOffsetThreshold,\n });\n\n if (offset > coordOffsetThreshold) {\n debug(\n 'cache coord offset exceeded threshold, fallback to AI locate',\n { offset, threshold: coordOffsetThreshold },\n );\n isCacheHit = false;\n elementFromCache = undefined;\n }\n }\n\n if (isCacheHit && enableVisualVerify && actions.visualVerify) {\n const verification = await this.service.verifyCachedElement(\n elementFromCache!.center,\n cachePrompt,\n modelConfigForDefaultIntent,\n uiContext,\n );\n if (!verification.pass) {\n debug(\n 'cache hit but visual verification failed, fallback to AI locate',\n {\n reason: verification.reason,\n description: verification.description,\n prompt: cachePrompt,\n },\n );\n isCacheHit = false;\n elementFromCache = undefined;\n } else {\n debug('cache hit and visual verification passed', {\n description: verification.description,\n });\n }\n }\n } catch (verifyError) {\n debug(\n 'cache verification error, fallback to AI locate',\n verifyError,\n );\n isCacheHit = false;\n elementFromCache = undefined;\n }\n\n if (isCacheHit) {\n const updatedState = updateConfidenceOnVerify(\n confidenceState,\n true,\n );\n cacheFeature.confidenceState = updatedState;\n\n if (isLegacyCache) {\n cacheFeature.cachedCenter = elementFromCache!.center;\n cacheFeature.progressiveRecord = createInitialProgressiveRecord(\n elementFromCache!.center,\n );\n debug('legacy cache upgraded with new fields', {\n cachedCenter: cacheFeature.cachedCenter,\n });\n }\n\n const progressiveRecord = cacheFeature.progressiveRecord as\n \| ProgressiveLocateRecord\n \| undefined;\n if (progressiveRecord) {\n const updated = updateProgressiveConvergence(\n progressiveRecord,\n elementFromCache!.center,\n updatedState.confidenceScore,\n );\n cacheFeature.progressiveRecord = updated;\n\n if (updated.convergenceRadius < 5 && updated.sampleCount >= 3) {\n debug(\n 'using converged center instead of single-result center',\n {\n convergedCenter: updated.convergedCenter.map((v) =>\n v.toFixed(1),\n ),\n singleCenter: elementFromCache!.center,\n convergenceRadius: updated.convergenceRadius.toFixed(1),\n sampleCount: updated.sampleCount,\n },\n );\n elementFromCache = {\n ...elementFromCache!,\n center: [\n Math.round(updated.convergedCenter[0]),\n Math.round(updated.convergedCenter[1]),\n ],\n };\n }\n }\n } else {\n const updatedState = updateConfidenceOnVerify(\n confidenceState,\n false,\n );\n cacheFeature.confidenceState = updatedState;\n elementFromCache = undefined;\n debug(\n 'cache verification failed, clearing cached element for AI fallback',\n {\n punishedScore: updatedState.confidenceScore.toFixed(3),\n verificationCount: updatedState.verificationCount,\n },\n );\n }\n }\n }\n\n if (!isXpathHit && !isCacheHit && !isPlanHit) {\n const cacheFeature = cacheEntry as ElementCacheFeature;\n const semanticAnchor = cacheFeature?.semanticAnchor as\n \| SemanticAnchor\n \| undefined;\n\n if (\n semanticAnchor &&\n process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true'\n ) {\n try {\n const anchorResult = await this.service.locateBySemanticAnchor(\n semanticAnchor,\n modelConfigForDefaultIntent,\n this.interface,\n uiContext,\n );\n if (anchorResult) {\n elementFromAiLocate = anchorResult;\n debug(\n 'semantic anchor locate succeeded, skipping full AI locate',\n );\n }\n } catch (anchorError) {\n debug('semantic anchor locate failed:', anchorError);\n }\n }\n\n if (!elementFromAiLocate) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n param,\n {\n context: uiContext,\n },\n modelConfigForDefaultIntent,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n }\n\n const element =\n elementFromBbox \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read), OR cache hit but legacy cache needs upgrade\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n (!isCacheHit \|\| isLegacyCache) &&\n (!isPlanHit \|\| !locateCacheAlreadyExists) &&\n param?.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n let pointForCache: [number, number] = element.center;\n if (shrunkShotToLogicalRatio !== 1) {\n pointForCache = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n debug(\n 'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n element.center,\n pointForCache,\n );\n }\n\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt,\n modelConfig: modelConfigForDefaultIntent,\n },\n );\n if (hasNonEmptyCache(feature)) {\n feature.cachedCenter = pointForCache;\n feature.confidenceState = createInitialConfidenceState();\n feature.progressiveRecord =\n createInitialProgressiveRecord(pointForCache);\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n\n const enableSemanticAnchor =\n process.env[MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR] === 'true';\n if (enableSemanticAnchor) {\n try {\n const anchor = await this.service.generateSemanticAnchor(\n pointForCache,\n modelConfigForDefaultIntent,\n uiContext,\n );\n if (anchor) {\n feature.semanticAnchor = anchor;\n debug(\n 'semantic anchor generated for prompt: %s',\n cachePrompt,\n );\n }\n } catch (anchorError) {\n debug('generateSemanticAnchor failed:', anchorError);\n }\n }\n\n currentCacheEntry = feature;\n await this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${param.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${param.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanHit) {\n hitBy = {\n from: 'Plan',\n context: {\n bbox: param.bbox,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: param.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n onResult?.(element);\n\n return {\n output: {\n element: {\n ...element,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","isLegacyCache","elementFromAiLocate","cacheFeature","cachedCenter","enableCoordCheck","process","MIDSCENE_CACHE_ENABLE_COORD_CHECK","enableVisualVerify","MIDSCENE_CACHE_ENABLE_VISUAL_VERIFY","coordOffsetThreshold","Number","MIDSCENE_CACHE_COORD_OFFSET_THRESHOLD","confidenceState","createInitialConfidenceState","confidence","calculateConfidence","level","determineVerificationLevel","actions","getVerificationActions","offset","Math","verification","verifyError","updatedState","updateConfidenceOnVerify","createInitialProgressiveRecord","progressiveRecord","updated","updateProgressiveConvergence","v","semanticAnchor","MIDSCENE_CACHE_ENABLE_SEMANTIC_ANCHOR","anchorResult","anchorError","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","feature","enableSemanticAnchor","anchor","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqDA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,IAAI;oBACF,MAAMG,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC3C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAkB,MAAM;qBACP;gBACH,EAAE,OAAOC,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAH,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEQ,wBAAwB,EAAE,GAAGP;gBACrC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,iBAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAElB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFT,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW2B;gBAE/CX,mBAAmBF,QAAQ;gBAE3B,MAAMc,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAH,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQa;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GACvE1C;QAEF,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEO,wBAAwB,EAAE,GAAGP;gBAErC,IAAIO,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIiC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb1B,KAAK,GAAG,GAAG;wBACT0B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1B,KAAK,KAAK,GAAG0B,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjB1B,KAAK,eAAe,GAAG0B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1B,KAAK,iBAAiB,GAAG0B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMC,kBAAkB/B,wBAAwB5B,SAC5C4D,qBAAqB5D,SACrB2C;gBACJ,MAAMkB,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD7D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF8D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC9D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAM+D,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACApB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAMuB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcnE,MAAM,MAAM;gBAChC,MAAMoE,oBACJ,MAAM,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBACzC,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAnE,MAAM,SAAS;gBAIvB,IAAIwE,mBAAmBF,yBACnBG,oCACEH,wBACA5B,4BAEFC;gBAEJ,IAAI+B,aAAa,CAAC,CAACF;gBACnB,IAAIG,gBAAgB;gBACpB,MAAMzC,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI2C;gBAEJ,IAAIF,YAAY;oBACd,MAAMG,eAAeR;oBACrB,MAAMS,eAAeD,cAAc;oBAInCF,gBAAgB,CAACG;oBAEjB,MAAMC,mBACJC,AAAmD,WAAnDA,QAAQ,GAAG,CAACC,kCAAkC;oBAChD,MAAMC,qBACJF,AAAqD,WAArDA,QAAQ,GAAG,CAACG,oCAAoC;oBAElD,MAAMC,uBACJC,OAAO,QAAQ,CACbL,QAAQ,GAAG,CAACM,sCAAsC,IAAI,MACtD,OACG;oBAEP,IAAIP,oBAAoBG,oBAAoB;wBAC1C,MAAMK,kBAAmBV,cAAc,mBACrCW;wBACF,MAAMC,aAAad,gBACf,OACAe,oBAAoBH;wBACxB,MAAMI,QAAQC,2BAA2BH;wBACzC,MAAMI,UAAUC,uBAAuBH;wBAEvCjG,MAAM,+BAA+B;4BACnC,YAAY+F,WAAW,OAAO,CAAC;4BAC/BE;4BACAE;4BACA,mBAAmBN,gBAAgB,iBAAiB;4BACpDZ;wBACF;wBAEA,IAAIkB,QAAQ,SAAS,EAAE;4BACrBnG,MAAM,qDAAqD;gCACzD+F;gCACAE;4BACF;4BACAjB,aAAa;4BACbF,mBAAmB7B;wBACrB;wBAEA,IAAI;4BACF,IACE+B,cACAK,oBACAc,QAAQ,UAAU,IAClBf,cACA;gCACA,MAAMiB,SAASC,KAAK,IAAI,CACrBxB,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM,IAChDN,AAAAA,CAAAA,iBAAkB,MAAM,CAAC,EAAE,GAAGM,YAAY,CAAC,EAAC,KAAM;gCAGvDpF,MAAM,4BAA4B;oCAChCoF;oCACA,eAAeN,iBAAkB,MAAM;oCACvC,QAAQwB,KAAK,KAAK,CAACD;oCACnB,WAAWX;gCACb;gCAEA,IAAIW,SAASX,sBAAsB;oCACjC1F,MACE,gEACA;wCAAEqG;wCAAQ,WAAWX;oCAAqB;oCAE5CV,aAAa;oCACbF,mBAAmB7B;gCACrB;4BACF;4BAEA,IAAI+B,cAAcQ,sBAAsBW,QAAQ,YAAY,EAAE;gCAC5D,MAAMI,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,mBAAmB,CACzDzB,iBAAkB,MAAM,EACxBL,aACA7D,6BACA6B;gCAEF,IAAK8D,aAAa,IAAI,EAYpBvG,MAAM,4CAA4C;oCAChD,aAAauG,aAAa,WAAW;gCACvC;qCAdsB;oCACtBvG,MACE,mEACA;wCACE,QAAQuG,aAAa,MAAM;wCAC3B,aAAaA,aAAa,WAAW;wCACrC,QAAQ9B;oCACV;oCAEFO,aAAa;oCACbF,mBAAmB7B;gCACrB;4BAKF;wBACF,EAAE,OAAOuD,aAAa;4BACpBxG,MACE,mDACAwG;4BAEFxB,aAAa;4BACbF,mBAAmB7B;wBACrB;wBAEA,IAAI+B,YAAY;4BACd,MAAMyB,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;4BAE/B,IAAIxB,eAAe;gCACjBE,aAAa,YAAY,GAAGL,iBAAkB,MAAM;gCACpDK,aAAa,iBAAiB,GAAGwB,+BAC/B7B,iBAAkB,MAAM;gCAE1B9E,MAAM,yCAAyC;oCAC7C,cAAcmF,aAAa,YAAY;gCACzC;4BACF;4BAEA,MAAMyB,oBAAoBzB,aAAa,iBAAiB;4BAGxD,IAAIyB,mBAAmB;gCACrB,MAAMC,UAAUC,6BACdF,mBACA9B,iBAAkB,MAAM,EACxB2B,aAAa,eAAe;gCAE9BtB,aAAa,iBAAiB,GAAG0B;gCAEjC,IAAIA,QAAQ,iBAAiB,GAAG,KAAKA,QAAQ,WAAW,IAAI,GAAG;oCAC7D7G,MACE,0DACA;wCACE,iBAAiB6G,QAAQ,eAAe,CAAC,GAAG,CAAC,CAACE,IAC5CA,EAAE,OAAO,CAAC;wCAEZ,cAAcjC,iBAAkB,MAAM;wCACtC,mBAAmB+B,QAAQ,iBAAiB,CAAC,OAAO,CAAC;wCACrD,aAAaA,QAAQ,WAAW;oCAClC;oCAEF/B,mBAAmB;wCACjB,GAAGA,gBAAgB;wCACnB,QAAQ;4CACNwB,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;4CACrCP,KAAK,KAAK,CAACO,QAAQ,eAAe,CAAC,EAAE;yCACtC;oCACH;gCACF;4BACF;wBACF,OAAO;4BACL,MAAMJ,eAAeC,yBACnBb,iBACA;4BAEFV,aAAa,eAAe,GAAGsB;4BAC/B3B,mBAAmB7B;4BACnBjD,MACE,sEACA;gCACE,eAAeyG,aAAa,eAAe,CAAC,OAAO,CAAC;gCACpD,mBAAmBA,aAAa,iBAAiB;4BACnD;wBAEJ;oBACF;gBACF;gBAEA,IAAI,CAACjC,cAAc,CAACQ,cAAc,CAACb,WAAW;oBAC5C,MAAMgB,eAAeR;oBACrB,MAAMqC,iBAAiB7B,cAAc;oBAIrC,IACE6B,kBACA1B,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC,EAElD,IAAI;wBACF,MAAMC,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAC5DF,gBACApG,6BACA,IAAI,CAAC,SAAS,EACd6B;wBAEF,IAAIyE,cAAc;4BAChBhC,sBAAsBgC;4BACtBlH,MACE;wBAEJ;oBACF,EAAE,OAAOmH,aAAa;wBACpBnH,MAAM,kCAAkCmH;oBAC1C;oBAGF,IAAI,CAACjC,qBACH,IAAI;wBACFxC,mBAAmBF,QAAQ;wBAC3BsB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCxD,OACA;4BACE,SAASmC;wBACX,GACA7B,6BACA8C;wBAEFK,UAAUD,aAAa,IAAI;wBAC3BoB,sBAAsBpB,aAAa,OAAO;oBAC5C,EAAE,OAAOX,OAAO;wBACd,IAAIA,iBAAiBiE,cACnBrD,UAAUZ,MAAM,IAAI;wBAEtB,MAAMA;oBACR,SAAU;wBACRT,mBAAmBF,QAAQ;oBAC7B;gBAEJ;gBAEA,MAAM6E,UACJpD,mBACAI,oBACAS,oBACAI;gBAGF,MAAMoC,2BAA2BpH,iBAC/BwE,mBAAmB,cAAc;gBAGnC,IAAI6C;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACb,EAACrC,cAAcC,aAAY,KAC3B,EAACd,aAAa,CAACmD,wBAAuB,KACvChH,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIkH,gBAAkCH,QAAQ,MAAM;oBACpD,IAAIrE,AAA6B,MAA7BA,0BAAgC;wBAClCwE,gBAAgB;4BACdlB,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;4BAC/BsD,KAAK,KAAK,CAACe,QAAQ,MAAM,CAAC,EAAE,GAAGrE;yBAChC;wBACDhD,MACE,8DACAqH,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAMC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDD,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOlH,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiBuH,UAAU;wBAC7BA,QAAQ,YAAY,GAAGD;wBACvBC,QAAQ,eAAe,GAAG3B;wBAC1B2B,QAAQ,iBAAiB,GACvBd,+BAA+Ba;wBACjCxH,MACE,uCACAyE,aACAgD;wBAGF,MAAMC,uBACJpC,AAAuD,WAAvDA,QAAQ,GAAG,CAAC2B,sCAAsC;wBACpD,IAAIS,sBACF,IAAI;4BACF,MAAMC,SAAS,MAAM,IAAI,CAAC,OAAO,CAAC,sBAAsB,CACtDH,eACA5G,6BACA6B;4BAEF,IAAIkF,QAAQ;gCACVF,QAAQ,cAAc,GAAGE;gCACzB3H,MACE,4CACAyE;4BAEJ;wBACF,EAAE,OAAO0C,aAAa;4BACpBnH,MAAM,kCAAkCmH;wBAC1C;wBAGFI,oBAAoBE;wBACpB,MAAM,IAAI,CAAC,SAAS,CAAC,yBAAyB,CAC5C;4BACE,MAAM;4BACN,QAAQhD;4BACR,OAAOgD;wBACT,GACA/C;oBAEJ,OACE1E,MACE,yDACAyE;gBAGN,EAAE,OAAOtB,OAAO;oBACdnD,MAAM,mCAAmCmD;gBAC3C;qBAEAnD,MAAM;gBAIV,IAAI,CAACqH,SAAS;oBACZ,IAAIxD,YACF,MAAM,IAAIuD,aACR,CAAC,oBAAoB,EAAE9G,MAAM,MAAM,EAAE,EACrCuD;oBAGJ,MAAM,IAAIjC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIsH;gBAEJ,IAAIzD,WACFyD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMtH,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIkE,YACToD,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOtH,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI0E,YACT4C,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPjD;wBACA,aAAa4C;oBACf;gBACF;gBAGF/D,WAAW6D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK5E,UAAU,aAAa;wBAC9B;oBACF;oBACAmF;gBACF;YACF;QACF;QAEA,OAAOhE;IACT;IArxBA,YAAY,EACViE,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTtG,WAAW,EACXuG,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGtG;QACnB,IAAI,CAAC,eAAe,GAAGuG;IACzB;AA0wBF"}

package/dist/es/agent/utils.mjs CHANGED Viewed

@@ -152,7 +152,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.6.10";
+const getMidsceneVersion = ()=>"1.6.11";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/types.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any /\n\nimport {\n existsSync,\n mkdirSync,\n readFileSync,\n rmSync,\n writeFileSync,\n} from 'node:fs';\nimport { join } from 'node:path';\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n BaseElement,\n LocateResultElement,\n Rect,\n Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport { restoreImageReferences } from './dump/image-restoration';\nimport { ScreenshotItem } from './screenshot-item';\nimport type {\n DetailedLocateParam,\n MidsceneYamlFlowItem,\n ServiceExtractOption,\n} from './yaml';\n\nexport type {\n ElementTreeNode,\n BaseElement,\n Rect,\n Size,\n Point,\n} from '@midscene/shared/types';\nexport from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n prompt_tokens: number \| undefined;\n completion_tokens: number \| undefined;\n total_tokens: number \| undefined;\n cached_input: number \| undefined;\n time_cost: number \| undefined;\n model_name: string \| undefined;\n model_description: string \| undefined;\n intent: string \| undefined;\n request_id: string \| undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n position?: {\n x: number;\n y: number;\n };\n bbox?: [number, number, number, number];\n reason: string;\n text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n bbox: [number, number, number, number];\n errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n data: DataDemand;\n errors?: string[];\n thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n bbox: [number, number, number, number];\n references_bbox?: [number, number, number, number][];\n error?: string;\n}\n\nexport interface AIAssertionResponse {\n pass: boolean;\n thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n description: string;\n error?: string;\n}\n\nexport interface LocatorValidatorOption {\n centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n pass: boolean;\n rect: Rect;\n center: [number, number];\n centerDistance?: number;\n}\n\nexport interface CacheVerificationResult {\n pass: boolean;\n description?: string;\n reason?: string;\n}\n\nexport interface CacheValidationOptions {\n enableCoordOffsetCheck?: boolean;\n coordOffsetThresholdPx?: number;\n enableVisualVerify?: boolean;\n enableSemanticAnchor?: boolean;\n}\n\nexport interface CacheConfidenceState {\n lastVerifiedAt: number;\n verificationCount: number;\n confidenceScore: number;\n}\n\nexport type VerificationLevel = 'minimal' \| 'standard' \| 'enhanced' \| 'full';\n\nexport interface ProgressiveLocateRecord {\n convergedCenter: [number, number];\n convergenceRadius: number;\n sampleCount: number;\n lastUpdatedAt: number;\n}\n\nexport interface Landmark {\n xpath: string;\n description: string;\n}\n\nexport interface SemanticAnchor {\n visualFingerprint: string;\n contextDescription: string;\n nearbyLandmarks: Landmark[];\n}\n\nexport interface AgentDescribeElementAtPointResult {\n prompt: string;\n deepLocate: boolean;\n verifyResult?: LocateValidatorResult;\n}\n\n/*\n context\n /\n\nexport abstract class UIContext {\n /\n screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n /\n abstract screenshot: ScreenshotItem;\n\n /\n screenshot size after shrinking\n /\n abstract shotSize: Size;\n\n /\n The ratio for converting shrunk screenshot coordinates to logical coordinates.\n \n Example:\n * - Physical screen width: 3000px, dpr=6\n * - Logical width: 500px\n * - User-defined screenshotShrinkFactor: 2\n * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n /\n abstract shrunkShotToLogicalRatio: number;\n\n abstract _isFrozen?: boolean;\n\n // @deprecated - backward compatibility for aiLocate\n abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' \| 'extract' \| 'assert' \| 'describe';\n\nexport type ServiceExtractParam = string \| Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n element: LocateResultElement \| null;\n rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' \| 'medium' \| 'high';\n\nexport type DeepThinkOption = 'unset' \| true \| false;\n\nexport interface ServiceTaskInfo {\n durationMs: number;\n formatResponse?: string;\n rawResponse?: string;\n usage?: AIUsageInfo;\n searchArea?: Rect;\n searchAreaRawResponse?: string;\n searchAreaUsage?: AIUsageInfo;\n reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n dumpString: string;\n attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n type: 'locate' \| 'extract' \| 'assert';\n logId: string;\n userQuery: {\n element?: TUserPrompt;\n dataDemand?: ServiceExtractParam;\n assertion?: TUserPrompt;\n };\n matchedElement: LocateResultElement[];\n matchedRect?: Rect;\n deepLocate?: boolean;\n data: any;\n assertionPass?: boolean;\n assertionThought?: string;\n assertionReason?: string;\n beforeScreenshot?: string;\n systemCheckResults?: SystemCheckResults;\n taskInfo: ServiceTaskInfo;\n error?: string;\n output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n ServiceDump,\n 'logTime' \| 'logId' \| 'model_name'\n>;\n\nexport interface ServiceResultBase {\n dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n data: T;\n thought?: string;\n usage?: AIUsageInfo;\n reasoning_content?: string;\n}\n\nexport class ServiceError extends Error {\n dump: ServiceDump;\n\n constructor(message: string, dump: ServiceDump) {\n super(message);\n this.name = 'ServiceError';\n this.dump = dump;\n }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n name: string;\n description: string;\n sectionCharacteristics: string;\n textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement \| null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n usage?: AIUsageInfo;\n};\n\n/\n agent\n /\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> \| void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n checkIntervalMs?: number;\n timeoutMs?: number;\n}\n\nexport interface ReferenceImage {\n name: string;\n url: string;\n}\n\nexport interface IgnoreRegion {\n x: number;\n y: number;\n width: number;\n height: number;\n}\n\nexport interface VideoInput {\n /* 视频帧列表（Base64 编码的图片），最多 150 帧（5秒 * 30fps） /\n frames: string[];\n /* 视频时长（秒），最大 5 秒 /\n duration?: number;\n /* 帧率，默认 30 /\n fps?: number;\n /* 视频格式，默认 'frames' /\n format?: 'frames' \| 'base64' \| 'url';\n /* 视频 URL（当 format 为 'url' 时使用） /\n url?: string;\n}\n\nexport interface VideoAssertionOptions {\n /* 是否验证动画流畅度，默认 true /\n checkSmoothness?: boolean;\n /* 流畅度阈值（0-100），默认 60 /\n smoothnessThreshold?: number;\n /* 是否验证动画时长，默认 false /\n checkDuration?: boolean;\n /* 预期动画时长范围（秒） /\n expectedDuration?: {\n min?: number;\n max?: number;\n };\n /* 是否验证动画完整性，默认 true /\n checkCompleteness?: boolean;\n /* 关键帧验证配置 /\n keyframes?: {\n /* 关键帧时间点（秒） /\n timestamps: number[];\n /* 关键帧描述 /\n descriptions?: string[];\n };\n /* 是否忽略背景变化，默认 false /\n ignoreBackgroundChanges?: boolean;\n /* 动画类型，用于优化分析 /\n animationType?: 'transition' \| 'loading' \| 'interaction' \| 'auto';\n}\n\nexport interface VideoDetails {\n /* 动画流畅度评分（0-100） /\n smoothnessScore: number;\n /* 动画时长（秒） /\n duration: number;\n /* 动画是否完整 /\n isComplete: boolean;\n /* 关键帧匹配结果 /\n keyframeMatches?: {\n timestamp: number;\n matched: boolean;\n description?: string;\n }[];\n /* 动画类型识别 /\n detectedAnimationType: string;\n /* 动画质量评估 /\n qualityAssessment: {\n /* 是否有卡顿 /\n hasStuttering: boolean;\n /* 是否有跳帧 /\n hasFrameDropping: boolean;\n /* 平均帧间隔（ms） /\n averageFrameInterval: number;\n /* 帧间隔标准差 /\n frameIntervalStdDev: number;\n };\n /* 可接受的差异列表 /\n acceptableDifferences: string[];\n /* 不可接受的差异列表 /\n unacceptableDifferences: string[];\n}\n\nexport interface RetryOptions {\n maxRetries: number;\n retryInterval: number;\n}\n\nexport interface DiffDetails {\n layoutMatch: boolean;\n colorMatch: boolean;\n elementPositionMatch: boolean;\n textContentMatch: boolean;\n resourceMatch: boolean;\n acceptableDifferences: string[];\n unacceptableDifferences: string[];\n}\n\nexport interface AssertSnapshot {\n beforeScreenshot: string;\n afterScreenshot: string;\n timestamp: number;\n assertion: string;\n}\n\nexport interface AgentAssertOpt {\n keepRawResponse?: boolean;\n /* 启用系统级校验（白屏、遮挡等），默认 false /\n enableSystemCheck?: boolean;\n /* 自定义系统校验规则 /\n customSystemCheckRules?: string;\n /* 业务知识上下文 /\n businessContext?: string;\n /* 手动传入操作前截图（Base64） /\n beforeScreenshot?: string;\n /* 手动传入操作后截图（Base64） /\n afterScreenshot?: string;\n /\n 截图模式\n * - 'auto': 自动判断（默认）\n * - 'lastAction': 使用最后一次 action 的前后截图\n * - 'flow': 使用整个流程的开始和结束截图\n * - 'manual': 使用手动传入的截图\n * - 'currentOnly': 仅使用当前截图（无操作前截图）\n * - 'diff': 与基准图片进行 Diff 对比\n * - 'video': 与基准视频进行动画对比\n /\n screenshotMode?:\n \| 'auto'\n \| 'lastAction'\n \| 'flow'\n \| 'manual'\n \| 'currentOnly'\n \| 'diff'\n \| 'video';\n /* 基准图片列表（仅 Diff 模式） /\n referenceImages?: ReferenceImage[];\n /* Diff 差异阈值（0-1），默认 0.1 /\n diffThreshold?: number;\n /* 忽略对比的区域 /\n ignoreRegions?: IgnoreRegion[];\n /* 是否忽略动态内容，默认 false /\n ignoreDynamicContent?: boolean;\n /* 严格模式，任何差异都视为失败，默认 false /\n strictMode?: boolean;\n /* 视频断言配置（仅 Video 模式） /\n videoOptions?: VideoAssertionOptions;\n /* 当前视频输入（仅 Video 模式），用户传入的视频帧或视频 URL /\n currentVideo?: VideoInput;\n /* 断言重试配置 /\n retryOptions?: RetryOptions;\n /* 是否保存快照，默认 false /\n saveSnapshot?: boolean;\n /* 快照保存路径 /\n snapshotPath?: string;\n}\n\nexport interface ActionScreenshotContext {\n beforeScreenshot: string;\n afterScreenshot: string;\n actionType: string;\n actionParam?: any;\n timestamp: number;\n}\n\nexport interface SystemCheckResults {\n whiteScreen?: boolean;\n layoutBlocked?: boolean;\n loadingContent?: boolean;\n errorPrompt?: boolean;\n backendError?: boolean;\n}\n\n/\n planning\n \n /\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n thought?: string;\n log?: string; // a brief preamble to the user explaining what you’re about to do\n type: string;\n param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' \| 'running' \| 'finished';\n\nexport interface SubGoal {\n index: number;\n status: SubGoalStatus;\n description: string;\n logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n action: PlanningAction;\n thought?: string;\n log: string;\n memory?: string;\n error?: string;\n finalizeMessage?: string;\n finalizeSuccess?: boolean;\n updateSubGoals?: SubGoal[];\n markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n extends Omit<RawResponsePlanningAIResponse, 'action'> {\n actions?: PlanningAction[];\n usage?: AIUsageInfo;\n rawResponse?: string;\n yamlFlow?: MidsceneYamlFlowItem[];\n yamlString?: string;\n error?: string;\n reasoning_content?: string;\n shouldContinuePlanning: boolean;\n output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n duration?: number;\n}\n\nexport interface PullParam {\n direction: 'up' \| 'down';\n distance?: number;\n duration?: number;\n}\n/*\n misc\n /\n\nexport interface Color {\n name: string;\n hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/\naction\n/\nexport interface ExecutionTaskProgressOptions {\n onTaskStart?: (task: ExecutionTask) => Promise<void> \| void;\n}\n\nexport interface ExecutionRecorderItem {\n type: 'screenshot';\n ts: number;\n screenshot?: ScreenshotItem;\n timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' \| 'Insight' \| 'Action Space' \| 'Log';\n\nexport interface ExecutorContext {\n task: ExecutionTask;\n element?: LocateResultElement \| null;\n uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n Type extends ExecutionTaskType = any,\n TaskParam = any,\n TaskOutput = any,\n TaskLog = any,\n> {\n type: Type;\n subType?: string;\n param?: TaskParam;\n thought?: string;\n uiContext?: UIContext;\n executor: (\n param: TaskParam,\n context: ExecutorContext,\n ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n \| Promise<ExecutionTaskReturn<TaskOutput, TaskLog> \| undefined \| void>\n \| undefined\n \| void;\n}\n\nexport interface ExecutionTaskHitBy {\n from: string;\n context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n output?: TaskOutput;\n log?: TaskLog;\n recorder?: ExecutionRecorderItem[];\n hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n any,\n any,\n any\n >,\n> = E &\n ExecutionTaskReturn<\n E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n ? TaskOutput\n : unknown,\n E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n ? TaskLog\n : unknown\n > & {\n taskId: string;\n status: 'pending' \| 'running' \| 'finished' \| 'failed' \| 'cancelled';\n error?: Error;\n errorMessage?: string;\n errorStack?: string;\n timing?: {\n start: number;\n getUiContextStart?: number;\n getUiContextEnd?: number;\n callAiStart?: number;\n callAiEnd?: number;\n beforeInvokeActionHookStart?: number;\n beforeInvokeActionHookEnd?: number;\n callActionStart?: number;\n callActionEnd?: number;\n afterInvokeActionHookStart?: number;\n afterInvokeActionHookEnd?: number;\n captureAfterCallingSnapshotStart?: number;\n captureAfterCallingSnapshotEnd?: number;\n end?: number;\n cost?: number;\n };\n usage?: AIUsageInfo;\n searchAreaUsage?: AIUsageInfo;\n reasoning_content?: string;\n };\n\nexport interface IExecutionDump extends DumpMeta {\n name: string;\n description?: string;\n tasks: ExecutionTask[];\n aiActContext?: string;\n}\n\n/\n Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem\n /\nfunction replacerForDumpSerialization(_key: string, value: any): any {\n if (value && value.constructor?.name === 'Page') {\n return '[Page object]';\n }\n if (value && value.constructor?.name === 'Browser') {\n return '[Browser object]';\n }\n // Handle ScreenshotItem serialization\n if (value && typeof value.toSerializable === 'function') {\n return value.toSerializable();\n }\n return value;\n}\n\n/\n Reviver function for JSON deserialization that handles ScreenshotItem formats.\n \n BEHAVIOR:\n * - For { $screenshot: \"id\" } format: Left as-is (plain object)\n * Consumer must use imageMap to restore base64 data\n * - For { base64: \"...\" } format: Creates ScreenshotItem from base64 data\n \n @param key - JSON key being processed\n * @param value - JSON value being processed\n * @returns Restored value\n /\nfunction reviverForDumpDeserialization(key: string, value: any): any {\n // Only process screenshot fields\n if (key !== 'screenshot' \|\| typeof value !== 'object' \|\| value === null) {\n return value;\n }\n\n // Handle serialized format: { $screenshot: \"id\" }\n // Leave as plain object — consumer uses imageMap to restore\n if (ScreenshotItem.isSerialized(value)) {\n return value;\n }\n\n // Handle inline base64 format: { base64: \"...\" }\n if ('base64' in value && typeof value.base64 === 'string') {\n return value;\n }\n\n return value;\n}\n\n/\n ExecutionDump class for serializing and deserializing execution dumps\n /\nexport class ExecutionDump implements IExecutionDump {\n logTime: number;\n name: string;\n description?: string;\n tasks: ExecutionTask[];\n aiActContext?: string;\n\n constructor(data: IExecutionDump) {\n this.logTime = data.logTime;\n this.name = data.name;\n this.description = data.description;\n this.tasks = data.tasks;\n this.aiActContext = data.aiActContext;\n }\n\n /\n Serialize the ExecutionDump to a JSON string\n /\n serialize(indents?: number): string {\n return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n }\n\n /\n Convert to a plain object for JSON serialization\n /\n toJSON(): IExecutionDump {\n return {\n logTime: this.logTime,\n name: this.name,\n description: this.description,\n tasks: this.tasks.map((task) => ({\n ...task,\n recorder: task.recorder \|\| [],\n })),\n aiActContext: this.aiActContext,\n };\n }\n\n /\n Create an ExecutionDump instance from a serialized JSON string\n /\n static fromSerializedString(serialized: string): ExecutionDump {\n const parsed = JSON.parse(\n serialized,\n reviverForDumpDeserialization,\n ) as IExecutionDump;\n return new ExecutionDump(parsed);\n }\n\n /\n Create an ExecutionDump instance from a plain object\n /\n static fromJSON(data: IExecutionDump): ExecutionDump {\n return new ExecutionDump(data);\n }\n\n /\n Collect all ScreenshotItem instances from tasks.\n * Scans through uiContext and recorder items to find screenshots.\n \n @returns Array of ScreenshotItem instances\n /\n collectScreenshots(): ScreenshotItem[] {\n const screenshots: ScreenshotItem[] = [];\n\n for (const task of this.tasks) {\n // Collect uiContext.screenshot if present\n if (task.uiContext?.screenshot instanceof ScreenshotItem) {\n screenshots.push(task.uiContext.screenshot);\n }\n\n // Collect recorder screenshots\n if (task.recorder) {\n for (const record of task.recorder) {\n if (record.screenshot instanceof ScreenshotItem) {\n screenshots.push(record.screenshot);\n }\n }\n }\n }\n\n return screenshots;\n }\n}\n\n/\ntask - service-locate\n/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n element: LocateResultElement \| null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightLocateParam,\n ExecutionTaskInsightLocateOutput,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/\ntask - service-query\n/\nexport interface ExecutionTaskInsightQueryParam {\n dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightQueryParam,\n any,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/\ntask - assertion\n/\nexport interface ExecutionTaskInsightAssertionParam {\n assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightAssertionParam,\n ServiceAssertionResponse,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/\ntask - action (i.e. interact) \n/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n 'Action Space',\n ActionParam,\n void,\n void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/\ntask - Log\n/\n\nexport type ExecutionTaskLogApply<\n LogParam = {\n content: string;\n },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/\ntask - planning\n/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n 'Planning',\n {\n userInstruction: string;\n aiActContext?: string;\n },\n PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/\ntask - planning-locate\n/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n element: LocateResultElement \| null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n 'Planning',\n ExecutionTaskPlanningLocateParam,\n ExecutionTaskPlanningLocateOutput,\n ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/\nGrouped dump\n/\nexport interface IGroupedActionDump {\n sdkVersion: string;\n groupName: string;\n groupDescription?: string;\n modelBriefs: string[];\n executions: IExecutionDump[];\n deviceType?: string;\n}\n\n/\n GroupedActionDump class for serializing and deserializing grouped action dumps\n /\nexport class GroupedActionDump implements IGroupedActionDump {\n sdkVersion: string;\n groupName: string;\n groupDescription?: string;\n modelBriefs: string[];\n executions: ExecutionDump[];\n deviceType?: string;\n\n constructor(data: IGroupedActionDump) {\n this.sdkVersion = data.sdkVersion;\n this.groupName = data.groupName;\n this.groupDescription = data.groupDescription;\n this.modelBriefs = data.modelBriefs;\n this.executions = data.executions.map((exec) =>\n exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec),\n );\n this.deviceType = data.deviceType;\n }\n\n /\n Serialize the GroupedActionDump to a JSON string\n * Uses compact { $screenshot: id } format\n /\n serialize(indents?: number): string {\n return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n }\n\n /\n Serialize the GroupedActionDump with inline screenshots to a JSON string.\n * Each ScreenshotItem is replaced with { base64: \"...\", capturedAt }.\n /\n serializeWithInlineScreenshots(indents?: number): string {\n const processValue = (obj: unknown): unknown => {\n if (obj instanceof ScreenshotItem) {\n return { base64: obj.base64, capturedAt: obj.capturedAt };\n }\n if (Array.isArray(obj)) {\n return obj.map(processValue);\n }\n if (obj && typeof obj === 'object') {\n const entries = Object.entries(obj).map(([key, value]) => [\n key,\n processValue(value),\n ]);\n return Object.fromEntries(entries);\n }\n return obj;\n };\n\n const data = processValue(this.toJSON());\n return JSON.stringify(data, null, indents);\n }\n\n /\n Convert to a plain object for JSON serialization\n /\n toJSON(): IGroupedActionDump {\n return {\n sdkVersion: this.sdkVersion,\n groupName: this.groupName,\n groupDescription: this.groupDescription,\n modelBriefs: this.modelBriefs,\n executions: this.executions.map((exec) => exec.toJSON()),\n deviceType: this.deviceType,\n };\n }\n\n /\n Create a GroupedActionDump instance from a serialized JSON string\n /\n static fromSerializedString(serialized: string): GroupedActionDump {\n const parsed = JSON.parse(\n serialized,\n reviverForDumpDeserialization,\n ) as IGroupedActionDump;\n return new GroupedActionDump(parsed);\n }\n\n /\n Create a GroupedActionDump instance from a plain object\n /\n static fromJSON(data: IGroupedActionDump): GroupedActionDump {\n return new GroupedActionDump(data);\n }\n\n /\n Collect all ScreenshotItem instances from all executions.\n \n @returns Array of all ScreenshotItem instances across all executions\n /\n collectAllScreenshots(): ScreenshotItem[] {\n const screenshots: ScreenshotItem[] = [];\n for (const execution of this.executions) {\n screenshots.push(...execution.collectScreenshots());\n }\n return screenshots;\n }\n\n /\n Serialize the dump to files with screenshots as separate PNG files.\n * Creates:\n * - {basePath} - dump JSON with { $screenshot: id } references\n * - {basePath}.screenshots/ - PNG files\n * - {basePath}.screenshots.json - ID to path mapping\n \n @param basePath - Base path for the dump file\n /\n serializeToFiles(basePath: string): void {\n const screenshotsDir = `${basePath}.screenshots`;\n if (!existsSync(screenshotsDir)) {\n mkdirSync(screenshotsDir, { recursive: true });\n }\n\n // Write screenshots to separate files\n const screenshotMap: Record<string, string> = {};\n const screenshots = this.collectAllScreenshots();\n\n for (const screenshot of screenshots) {\n if (screenshot.hasBase64()) {\n const imagePath = join(\n screenshotsDir,\n `${screenshot.id}.${screenshot.extension}`,\n );\n const rawBase64 = screenshot.rawBase64;\n writeFileSync(imagePath, Buffer.from(rawBase64, 'base64'));\n screenshotMap[screenshot.id] = imagePath;\n }\n }\n\n // Write screenshot map file\n writeFileSync(\n `${basePath}.screenshots.json`,\n JSON.stringify(screenshotMap),\n 'utf-8',\n );\n\n // Write dump JSON with references\n writeFileSync(basePath, this.serialize(), 'utf-8');\n }\n\n /\n Read dump from files and return JSON string with inline screenshots.\n * Reads the dump JSON and screenshot files, then inlines the base64 data.\n \n @param basePath - Base path for the dump file\n * @returns JSON string with inline screenshots ({ base64: \"...\" } format)\n /\n static fromFilesAsInlineJson(basePath: string): string {\n const dumpString = readFileSync(basePath, 'utf-8');\n const screenshotsMapPath = `${basePath}.screenshots.json`;\n\n if (!existsSync(screenshotsMapPath)) {\n return dumpString;\n }\n\n // Read screenshot map and build imageMap from files\n const screenshotMap: Record<string, string> = JSON.parse(\n readFileSync(screenshotsMapPath, 'utf-8'),\n );\n\n const imageMap: Record<string, string> = {};\n for (const [id, filePath] of Object.entries(screenshotMap)) {\n if (existsSync(filePath)) {\n const data = readFileSync(filePath);\n const mime =\n filePath.endsWith('.jpeg') \|\| filePath.endsWith('.jpg')\n ? 'jpeg'\n : 'png';\n imageMap[id] = `data:image/${mime};base64,${data.toString('base64')}`;\n }\n }\n\n // Restore image references\n const dumpData = JSON.parse(dumpString);\n const processedData = restoreImageReferences(\n dumpData,\n (id) => imageMap[id] ?? '',\n );\n return JSON.stringify(processedData);\n }\n\n /\n Clean up all files associated with a serialized dump.\n \n @param basePath - Base path for the dump file\n /\n static cleanupFiles(basePath: string): void {\n const filesToClean = [\n basePath,\n `${basePath}.screenshots.json`,\n `${basePath}.screenshots`,\n ];\n\n for (const filePath of filesToClean) {\n try {\n rmSync(filePath, { force: true, recursive: true });\n } catch {\n // Ignore errors - file may already be deleted\n }\n }\n }\n\n /\n Get all file paths associated with a serialized dump.\n \n @param basePath - Base path for the dump file\n * @returns Array of all associated file paths\n /\n static getFilePaths(basePath: string): string[] {\n return [\n basePath,\n `${basePath}.screenshots.json`,\n `${basePath}.screenshots`,\n ];\n }\n}\n\nexport type InterfaceType =\n \| 'puppeteer'\n \| 'playwright'\n \| 'static'\n \| 'chrome-extension-proxy'\n \| 'android'\n \| string;\n\nexport interface StreamingCodeGenerationOptions {\n /* Whether to enable streaming output /\n stream?: boolean;\n /* Callback function to handle streaming chunks /\n onChunk?: StreamingCallback;\n /* Callback function to handle streaming completion /\n onComplete?: (finalCode: string) => void;\n /* Callback function to handle streaming errors /\n onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n /* The incremental content chunk /\n content: string;\n /* The reasoning content /\n reasoning_content: string;\n /* The accumulated content so far /\n accumulated: string;\n /* Whether this is the final chunk /\n isComplete: boolean;\n /* Token usage information if available /\n usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n /* The final accumulated content /\n content: string;\n /* Token usage information /\n usage?: AIUsageInfo;\n /* Whether the response was streamed /\n isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n name: string;\n description?: string;\n interfaceAlias?: string;\n paramSchema?: z.ZodType<TParam>;\n call: (param: TParam, context: ExecutorContext) => Promise<TReturn> \| TReturn;\n delayAfterRunner?: number;\n /\n An example param object for this action.\n * Locate fields with { prompt } will automatically get bbox injected when needed.\n /\n sample?: { [K in keyof TParam]?: any };\n}\n\n/\n Type utilities for extracting types from DeviceAction definitions\n /\n\n/\n Extract parameter type from a DeviceAction\n /\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n Action extends DeviceAction<infer P, any> ? P : never;\n\n/\n Extract return type from a DeviceAction\n /\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n Action extends DeviceAction<any, infer R> ? R : never;\n\n/\n Web-specific types\n /\nexport interface WebElementInfo extends BaseElement {\n id: string;\n attributes: {\n nodeType: NodeType;\n [key: string]: string;\n };\n}\n\n/\n Agent\n /\n\nexport type CacheConfig = {\n strategy?: 'read-only' \| 'read-write' \| 'write-only';\n id: string;\n cacheAdapter?: import('./agent/cache-adapter').CacheAdapter;\n};\n\nexport type Cache =\n \| false // No read, no write\n \| true // Will throw error at runtime - deprecated\n \| CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n testId?: string;\n // @deprecated\n cacheId?: string; // Keep backward compatibility, but marked as deprecated\n groupName?: string;\n groupDescription?: string;\n / if auto generate report, default true /\n generateReport?: boolean;\n / if auto print report msg, default true /\n autoPrintReportMsg?: boolean;\n\n /\n Use directory-based report format with separate image files.\n \n When enabled:\n * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n * - Report is generated as `index.html` with relative image paths\n * - Reduces memory usage and report file size\n \n IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n * work due to browser CORS restrictions.\n \n @default 'single-html'\n /\n outputFormat?: 'single-html' \| 'html-and-external-assets';\n\n onTaskStartTip?: OnTaskStartTip;\n aiActContext?: string;\n aiActionContext?: string;\n / custom report file name /\n reportFileName?: string;\n modelConfig?: TModelConfig;\n cache?: Cache;\n /\n Maximum number of replanning cycles for aiAct.\n * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n /\n replanningCycleLimit?: number;\n\n /\n Wait time in milliseconds after each action execution.\n * This allows the UI to settle and stabilize before the next action.\n * Defaults to 300ms when not provided.\n /\n waitAfterAction?: number;\n\n /\n When set to true, Midscene will use the target device's time (Android/iOS)\n * instead of the system time. Useful when the device time differs from the\n * host machine. Default: false\n /\n useDeviceTimestamp?: boolean;\n\n /\n Custom screenshot shrink factor to reduce AI token usage.\n * When set, the screenshot will be scaled down by this factor from the physical resolution.\n \n Example:\n * - Physical screen width: 3000px, dpr=6\n * - Logical width: 500px\n * - screenshotShrinkFactor: 2\n * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n * - AI analyzes the 1500px screenshot\n * - Coordinates are transformed back to logical (500px) before actions execute\n \n Benefits:\n * - Reduces token usage for high-resolution screenshots\n * - Maintains accuracy by scaling coordinates appropriately\n \n Must be >= 1 (shrinking only, enlarging is not supported).\n \n @default 1 (no shrinking, uses original physical screenshot)\n /\n screenshotShrinkFactor?: number;\n\n /\n Custom OpenAI client factory function\n \n If provided, this function will be called to create OpenAI client instances\n * for each AI call, allowing you to:\n * - Wrap clients with observability tools (langsmith, langfuse)\n * - Use custom OpenAI-compatible clients\n * - Apply different configurations based on intent\n \n @param config - Resolved model configuration\n * @returns OpenAI client instance (original or wrapped)\n \n @example\n * ```typescript\n * createOpenAIClient: async (openai, opts) => {\n * // Wrap with langsmith for planning tasks\n * if (opts.baseURL?.includes('planning')) {\n * return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n * }\n \n return openai;\n * }\n * ```\n */\n createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n \| 'passed'\n \| 'failed'\n \| 'timedOut'\n \| 'skipped'\n \| 'interrupted';\n\nexport interface ReportFileWithAttributes {\n reportFilePath: string;\n reportAttributes: {\n testDuration: number;\n testStatus: TestStatus;\n testTitle: string;\n testId: string;\n testDescription: string;\n };\n}\n"],"names":["UIContext","ServiceError","Error","message","dump","replacerForDumpSerialization","_key","value","reviverForDumpDeserialization","key","ScreenshotItem","ExecutionDump","indents","JSON","task","serialized","parsed","data","screenshots","record","GroupedActionDump","processValue","obj","Array","entries","Object","exec","execution","basePath","screenshotsDir","existsSync","mkdirSync","screenshotMap","screenshot","imagePath","join","rawBase64","writeFileSync","Buffer","dumpString","readFileSync","screenshotsMapPath","imageMap","id","filePath","mime","dumpData","processedData","restoreImageReferences","filesToClean","rmSync"],"mappings":";;;;;AAAqD;;;;;;;;;;AAsJ9C,MAAeA;AA4BtB;AA+EO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF;AA0YA,SAASC,6BAA6BC,IAAY,EAAEC,KAAU;IAC5D,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,QACvC,OAAO;IAET,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,WACvC,OAAO;IAGT,IAAIA,SAAS,AAAgC,cAAhC,OAAOA,MAAM,cAAc,EACtC,OAAOA,MAAM,cAAc;IAE7B,OAAOA;AACT;AAcA,SAASC,8BAA8BC,GAAW,EAAEF,KAAU;IAE5D,IAAIE,AAAQ,iBAARA,OAAwB,AAAiB,YAAjB,OAAOF,SAAsBA,AAAU,SAAVA,OACvD,OAAOA;IAKT,IAAIG,eAAe,YAAY,CAACH,QAC9B,OAAOA;IAIL,YAAYA,SAAgBA,MAAM,MAAM;IAI5C,OAAOA;AACT;AAKO,MAAMI;IAkBX,UAAUC,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAAyB;QACvB,OAAO;YACL,SAAS,IAAI,CAAC,OAAO;YACrB,MAAM,IAAI,CAAC,IAAI;YACf,aAAa,IAAI,CAAC,WAAW;YAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAACE,OAAU;oBAC/B,GAAGA,IAAI;oBACP,UAAUA,KAAK,QAAQ,IAAI,EAAE;gBAC/B;YACA,cAAc,IAAI,CAAC,YAAY;QACjC;IACF;IAKA,OAAO,qBAAqBC,UAAkB,EAAiB;QAC7D,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIG,cAAcK;IAC3B;IAKA,OAAO,SAASC,IAAoB,EAAiB;QACnD,OAAO,IAAIN,cAAcM;IAC3B;IAQA,qBAAuC;QACrC,MAAMC,cAAgC,EAAE;QAExC,KAAK,MAAMJ,QAAQ,IAAI,CAAC,KAAK,CAAE;YAE7B,IAAIA,KAAK,SAAS,EAAE,sBAAsBJ,gBACxCQ,YAAY,IAAI,CAACJ,KAAK,SAAS,CAAC,UAAU;YAI5C,IAAIA,KAAK,QAAQ,EACf;gBAAA,KAAK,MAAMK,UAAUL,KAAK,QAAQ,CAChC,IAAIK,OAAO,UAAU,YAAYT,gBAC/BQ,YAAY,IAAI,CAACC,OAAO,UAAU;YAEtC;QAEJ;QAEA,OAAOD;IACT;IA3EA,YAAYD,IAAoB,CAAE;QANlC;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,OAAO,GAAGA,KAAK,OAAO;QAC3B,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI;QACrB,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,YAAY,GAAGA,KAAK,YAAY;IACvC;AAsEF;AAwIO,MAAMG;IAuBX,UAAUR,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAMA,+BAA+BA,OAAgB,EAAU;QACvD,MAAMS,eAAe,CAACC;YACpB,IAAIA,eAAeZ,gBACjB,OAAO;gBAAE,QAAQY,IAAI,MAAM;gBAAE,YAAYA,IAAI,UAAU;YAAC;YAE1D,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAACD;YAEjB,IAAIC,OAAO,AAAe,YAAf,OAAOA,KAAkB;gBAClC,MAAME,UAAUC,OAAO,OAAO,CAACH,KAAK,GAAG,CAAC,CAAC,CAACb,KAAKF,MAAM,GAAK;wBACxDE;wBACAY,aAAad;qBACd;gBACD,OAAOkB,OAAO,WAAW,CAACD;YAC5B;YACA,OAAOF;QACT;QAEA,MAAML,OAAOI,aAAa,IAAI,CAAC,MAAM;QACrC,OAAOR,KAAK,SAAS,CAACI,MAAM,MAAML;IACpC;IAKA,SAA6B;QAC3B,OAAO;YACL,YAAY,IAAI,CAAC,UAAU;YAC3B,WAAW,IAAI,CAAC,SAAS;YACzB,kBAAkB,IAAI,CAAC,gBAAgB;YACvC,aAAa,IAAI,CAAC,WAAW;YAC7B,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAACc,OAASA,KAAK,MAAM;YACrD,YAAY,IAAI,CAAC,UAAU;QAC7B;IACF;IAKA,OAAO,qBAAqBX,UAAkB,EAAqB;QACjE,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIY,kBAAkBJ;IAC/B;IAKA,OAAO,SAASC,IAAwB,EAAqB;QAC3D,OAAO,IAAIG,kBAAkBH;IAC/B;IAOA,wBAA0C;QACxC,MAAMC,cAAgC,EAAE;QACxC,KAAK,MAAMS,aAAa,IAAI,CAAC,UAAU,CACrCT,YAAY,IAAI,IAAIS,UAAU,kBAAkB;QAElD,OAAOT;IACT;IAWA,iBAAiBU,QAAgB,EAAQ;QACvC,MAAMC,iBAAiB,GAAGD,SAAS,YAAY,CAAC;QAChD,IAAI,CAACE,WAAWD,iBACdE,UAAUF,gBAAgB;YAAE,WAAW;QAAK;QAI9C,MAAMG,gBAAwC,CAAC;QAC/C,MAAMd,cAAc,IAAI,CAAC,qBAAqB;QAE9C,KAAK,MAAMe,cAAcf,YACvB,IAAIe,WAAW,SAAS,IAAI;YAC1B,MAAMC,YAAYC,KAChBN,gBACA,GAAGI,WAAW,EAAE,CAAC,CAAC,EAAEA,WAAW,SAAS,EAAE;YAE5C,MAAMG,YAAYH,WAAW,SAAS;YACtCI,cAAcH,WAAWI,OAAO,IAAI,CAACF,WAAW;YAChDJ,aAAa,CAACC,WAAW,EAAE,CAAC,GAAGC;QACjC;QAIFG,cACE,GAAGT,SAAS,iBAAiB,CAAC,EAC9Bf,KAAK,SAAS,CAACmB,gBACf;QAIFK,cAAcT,UAAU,IAAI,CAAC,SAAS,IAAI;IAC5C;IASA,OAAO,sBAAsBA,QAAgB,EAAU;QACrD,MAAMW,aAAaC,aAAaZ,UAAU;QAC1C,MAAMa,qBAAqB,GAAGb,SAAS,iBAAiB,CAAC;QAEzD,IAAI,CAACE,WAAWW,qBACd,OAAOF;QAIT,MAAMP,gBAAwCnB,KAAK,KAAK,CACtD2B,aAAaC,oBAAoB;QAGnC,MAAMC,WAAmC,CAAC;QAC1C,KAAK,MAAM,CAACC,IAAIC,SAAS,IAAInB,OAAO,OAAO,CAACO,eAC1C,IAAIF,WAAWc,WAAW;YACxB,MAAM3B,OAAOuB,aAAaI;YAC1B,MAAMC,OACJD,SAAS,QAAQ,CAAC,YAAYA,SAAS,QAAQ,CAAC,UAC5C,SACA;YACNF,QAAQ,CAACC,GAAG,GAAG,CAAC,WAAW,EAAEE,KAAK,QAAQ,EAAE5B,KAAK,QAAQ,CAAC,WAAW;QACvE;QAIF,MAAM6B,WAAWjC,KAAK,KAAK,CAAC0B;QAC5B,MAAMQ,gBAAgBC,uBACpBF,UACA,CAACH,KAAOD,QAAQ,CAACC,GAAG,IAAI;QAE1B,OAAO9B,KAAK,SAAS,CAACkC;IACxB;IAOA,OAAO,aAAanB,QAAgB,EAAQ;QAC1C,MAAMqB,eAAe;YACnBrB;YACA,GAAGA,SAAS,iBAAiB,CAAC;YAC9B,GAAGA,SAAS,YAAY,CAAC;SAC1B;QAED,KAAK,MAAMgB,YAAYK,aACrB,IAAI;YACFC,OAAON,UAAU;gBAAE,OAAO;gBAAM,WAAW;YAAK;QAClD,EAAE,OAAM,CAER;IAEJ;IAQA,OAAO,aAAahB,QAAgB,EAAY;QAC9C,OAAO;YACLA;YACA,GAAGA,SAAS,iBAAiB,CAAC;YAC9B,GAAGA,SAAS,YAAY,CAAC;SAC1B;IACH;IA9MA,YAAYX,IAAwB,CAAE;QAPtC;QACA;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU;QACjC,IAAI,CAAC,SAAS,GAAGA,KAAK,SAAS;QAC/B,IAAI,CAAC,gBAAgB,GAAGA,KAAK,gBAAgB;QAC7C,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU,CAAC,GAAG,CAAC,CAACS,OACrCA,gBAAgBf,gBAAgBe,OAAOf,cAAc,QAAQ,CAACe;QAEhE,IAAI,CAAC,UAAU,GAAGT,KAAK,UAAU;IACnC;AAsMF"}
1	+ {"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any /\n\nimport {\n existsSync,\n mkdirSync,\n readFileSync,\n rmSync,\n writeFileSync,\n} from 'node:fs';\nimport { join } from 'node:path';\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n BaseElement,\n LocateResultElement,\n Rect,\n Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport { restoreImageReferences } from './dump/image-restoration';\nimport { ScreenshotItem } from './screenshot-item';\nimport type {\n DetailedLocateParam,\n MidsceneYamlFlowItem,\n ServiceExtractOption,\n} from './yaml';\n\nexport type {\n ElementTreeNode,\n BaseElement,\n Rect,\n Size,\n Point,\n} from '@midscene/shared/types';\nexport from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n prompt_tokens: number \| undefined;\n completion_tokens: number \| undefined;\n total_tokens: number \| undefined;\n cached_input: number \| undefined;\n time_cost: number \| undefined;\n model_name: string \| undefined;\n model_description: string \| undefined;\n intent: string \| undefined;\n request_id: string \| undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n position?: {\n x: number;\n y: number;\n };\n bbox?: [number, number, number, number];\n reason: string;\n text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n bbox: [number, number, number, number];\n errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n data: DataDemand;\n errors?: string[];\n thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n bbox: [number, number, number, number];\n references_bbox?: [number, number, number, number][];\n error?: string;\n}\n\nexport interface AIAssertionResponse {\n pass: boolean;\n thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n description: string;\n error?: string;\n}\n\nexport interface LocatorValidatorOption {\n centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n pass: boolean;\n rect: Rect;\n center: [number, number];\n centerDistance?: number;\n}\n\nexport interface CacheVerificationResult {\n pass: boolean;\n description?: string;\n reason?: string;\n}\n\nexport interface CacheValidationOptions {\n enableCoordOffsetCheck?: boolean;\n coordOffsetThresholdPx?: number;\n enableVisualVerify?: boolean;\n enableSemanticAnchor?: boolean;\n}\n\nexport interface CacheConfidenceState {\n lastVerifiedAt: number;\n verificationCount: number;\n confidenceScore: number;\n}\n\nexport type VerificationLevel = 'minimal' \| 'standard' \| 'enhanced' \| 'full';\n\nexport interface ProgressiveLocateRecord {\n convergedCenter: [number, number];\n convergenceRadius: number;\n sampleCount: number;\n lastUpdatedAt: number;\n}\n\nexport interface Landmark {\n xpath: string;\n description: string;\n}\n\nexport interface SemanticAnchor {\n visualFingerprint: string;\n contextDescription: string;\n nearbyLandmarks: Landmark[];\n}\n\nexport interface AgentDescribeElementAtPointResult {\n prompt: string;\n deepLocate: boolean;\n verifyResult?: LocateValidatorResult;\n}\n\n/*\n context\n /\n\nexport abstract class UIContext {\n /\n screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n /\n abstract screenshot: ScreenshotItem;\n\n /\n screenshot size after shrinking\n /\n abstract shotSize: Size;\n\n /\n The ratio for converting shrunk screenshot coordinates to logical coordinates.\n \n Example:\n * - Physical screen width: 3000px, dpr=6\n * - Logical width: 500px\n * - User-defined screenshotShrinkFactor: 2\n * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n /\n abstract shrunkShotToLogicalRatio: number;\n\n abstract _isFrozen?: boolean;\n\n // @deprecated - backward compatibility for aiLocate\n abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' \| 'extract' \| 'assert' \| 'describe';\n\nexport type ServiceExtractParam = string \| Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n element: LocateResultElement \| null;\n rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' \| 'medium' \| 'high';\n\nexport type DeepThinkOption = 'unset' \| true \| false;\n\nexport interface ServiceTaskInfo {\n durationMs: number;\n formatResponse?: string;\n rawResponse?: string;\n usage?: AIUsageInfo;\n searchArea?: Rect;\n searchAreaRawResponse?: string;\n searchAreaUsage?: AIUsageInfo;\n reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n dumpString: string;\n attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n type: 'locate' \| 'extract' \| 'assert';\n logId: string;\n userQuery: {\n element?: TUserPrompt;\n dataDemand?: ServiceExtractParam;\n assertion?: TUserPrompt;\n };\n matchedElement: LocateResultElement[];\n matchedRect?: Rect;\n deepLocate?: boolean;\n data: any;\n assertionPass?: boolean;\n assertionThought?: string;\n assertionReason?: string;\n beforeScreenshot?: string;\n systemCheckResults?: SystemCheckResults;\n taskInfo: ServiceTaskInfo;\n error?: string;\n output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n ServiceDump,\n 'logTime' \| 'logId' \| 'model_name'\n>;\n\nexport interface ServiceResultBase {\n dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n data: T;\n thought?: string;\n usage?: AIUsageInfo;\n reasoning_content?: string;\n}\n\nexport class ServiceError extends Error {\n dump: ServiceDump;\n\n constructor(message: string, dump: ServiceDump) {\n super(message);\n this.name = 'ServiceError';\n this.dump = dump;\n }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n name: string;\n description: string;\n sectionCharacteristics: string;\n textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement \| null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n usage?: AIUsageInfo;\n};\n\n/\n agent\n /\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> \| void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n checkIntervalMs?: number;\n timeoutMs?: number;\n}\n\nexport interface ReferenceImage {\n name: string;\n url: string;\n}\n\nexport interface IgnoreRegion {\n x: number;\n y: number;\n width: number;\n height: number;\n}\n\nexport interface VideoInput {\n /* 视频帧列表（Base64 编码的图片），最多 150 帧（5秒 * 30fps） /\n frames: string[];\n /* 视频时长（秒），最大 5 秒 /\n duration?: number;\n /* 帧率，默认 30 /\n fps?: number;\n /* 视频格式，默认 'frames' /\n format?: 'frames' \| 'base64' \| 'url';\n /* 视频 URL（当 format 为 'url' 时使用） /\n url?: string;\n}\n\nexport interface VideoAssertionOptions {\n /* 是否验证动画流畅度，默认 true /\n checkSmoothness?: boolean;\n /* 流畅度阈值（0-100），默认 60 /\n smoothnessThreshold?: number;\n /* 是否验证动画时长，默认 false /\n checkDuration?: boolean;\n /* 预期动画时长范围（秒） /\n expectedDuration?: {\n min?: number;\n max?: number;\n };\n /* 是否验证动画完整性，默认 true /\n checkCompleteness?: boolean;\n /* 关键帧验证配置 /\n keyframes?: {\n /* 关键帧时间点（秒） /\n timestamps: number[];\n /* 关键帧描述 /\n descriptions?: string[];\n };\n /* 是否忽略背景变化，默认 false /\n ignoreBackgroundChanges?: boolean;\n /* 动画类型，用于优化分析 /\n animationType?: 'transition' \| 'loading' \| 'interaction' \| 'auto';\n}\n\nexport interface VideoDetails {\n /* 动画流畅度评分（0-100） /\n smoothnessScore: number;\n /* 动画时长（秒） /\n duration: number;\n /* 动画是否完整 /\n isComplete: boolean;\n /* 关键帧匹配结果 /\n keyframeMatches?: {\n timestamp: number;\n matched: boolean;\n description?: string;\n }[];\n /* 动画类型识别 /\n detectedAnimationType: string;\n /* 动画质量评估 /\n qualityAssessment: {\n /* 是否有卡顿 /\n hasStuttering: boolean;\n /* 是否有跳帧 /\n hasFrameDropping: boolean;\n /* 平均帧间隔（ms） /\n averageFrameInterval: number;\n /* 帧间隔标准差 /\n frameIntervalStdDev: number;\n };\n /* 可接受的差异列表 /\n acceptableDifferences: string[];\n /* 不可接受的差异列表 /\n unacceptableDifferences: string[];\n}\n\nexport interface RetryOptions {\n maxRetries: number;\n retryInterval: number;\n}\n\nexport interface DiffDetails {\n layoutMatch: boolean;\n colorMatch: boolean;\n elementPositionMatch: boolean;\n textContentMatch: boolean;\n resourceMatch: boolean;\n acceptableDifferences: string[];\n unacceptableDifferences: string[];\n}\n\nexport interface AssertSnapshot {\n beforeScreenshot: string;\n afterScreenshot: string;\n timestamp: number;\n assertion: string;\n}\n\nexport interface AgentAssertOpt {\n keepRawResponse?: boolean;\n /* 启用系统级校验（白屏、遮挡等），默认 false /\n enableSystemCheck?: boolean;\n /* 自定义系统校验规则 /\n customSystemCheckRules?: string;\n /* 业务知识上下文 /\n businessContext?: string;\n /* 手动传入操作前截图（Base64） /\n beforeScreenshot?: string;\n /* 手动传入操作后截图（Base64） /\n afterScreenshot?: string;\n /\n 截图模式\n * - 'currentOnly': 仅使用当前截图（默认）\n * - 'auto': 自动判断\n * - 'lastAction': 使用最后一次 action 的前后截图\n * - 'flow': 使用整个流程的开始和结束截图\n * - 'manual': 使用手动传入的截图\n * - 'diff': 与基准图片进行 Diff 对比\n * - 'video': 与基准视频进行动画对比\n /\n screenshotMode?:\n \| 'auto'\n \| 'lastAction'\n \| 'flow'\n \| 'manual'\n \| 'currentOnly'\n \| 'diff'\n \| 'video';\n /* 基准图片列表（仅 Diff 模式） /\n referenceImages?: ReferenceImage[];\n /* Diff 差异阈值（0-1），默认 0.1 /\n diffThreshold?: number;\n /* 忽略对比的区域 /\n ignoreRegions?: IgnoreRegion[];\n /* 是否忽略动态内容，默认 false /\n ignoreDynamicContent?: boolean;\n /* 严格模式，任何差异都视为失败，默认 false /\n strictMode?: boolean;\n /* 视频断言配置（仅 Video 模式） /\n videoOptions?: VideoAssertionOptions;\n /* 当前视频输入（仅 Video 模式），用户传入的视频帧或视频 URL /\n currentVideo?: VideoInput;\n /* 断言重试配置 /\n retryOptions?: RetryOptions;\n /* 是否保存快照，默认 false /\n saveSnapshot?: boolean;\n /* 快照保存路径 /\n snapshotPath?: string;\n}\n\nexport interface ActionScreenshotContext {\n beforeScreenshot: string;\n afterScreenshot: string;\n actionType: string;\n actionParam?: any;\n timestamp: number;\n}\n\nexport interface SystemCheckResults {\n whiteScreen?: boolean;\n layoutBlocked?: boolean;\n loadingContent?: boolean;\n errorPrompt?: boolean;\n backendError?: boolean;\n}\n\n/\n planning\n \n /\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n thought?: string;\n log?: string; // a brief preamble to the user explaining what you’re about to do\n type: string;\n param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' \| 'running' \| 'finished';\n\nexport interface SubGoal {\n index: number;\n status: SubGoalStatus;\n description: string;\n logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n action: PlanningAction;\n thought?: string;\n log: string;\n memory?: string;\n error?: string;\n finalizeMessage?: string;\n finalizeSuccess?: boolean;\n updateSubGoals?: SubGoal[];\n markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n extends Omit<RawResponsePlanningAIResponse, 'action'> {\n actions?: PlanningAction[];\n usage?: AIUsageInfo;\n rawResponse?: string;\n yamlFlow?: MidsceneYamlFlowItem[];\n yamlString?: string;\n error?: string;\n reasoning_content?: string;\n shouldContinuePlanning: boolean;\n output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n duration?: number;\n}\n\nexport interface PullParam {\n direction: 'up' \| 'down';\n distance?: number;\n duration?: number;\n}\n/*\n misc\n /\n\nexport interface Color {\n name: string;\n hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/\naction\n/\nexport interface ExecutionTaskProgressOptions {\n onTaskStart?: (task: ExecutionTask) => Promise<void> \| void;\n}\n\nexport interface ExecutionRecorderItem {\n type: 'screenshot';\n ts: number;\n screenshot?: ScreenshotItem;\n timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' \| 'Insight' \| 'Action Space' \| 'Log';\n\nexport interface ExecutorContext {\n task: ExecutionTask;\n element?: LocateResultElement \| null;\n uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n Type extends ExecutionTaskType = any,\n TaskParam = any,\n TaskOutput = any,\n TaskLog = any,\n> {\n type: Type;\n subType?: string;\n param?: TaskParam;\n thought?: string;\n uiContext?: UIContext;\n executor: (\n param: TaskParam,\n context: ExecutorContext,\n ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n \| Promise<ExecutionTaskReturn<TaskOutput, TaskLog> \| undefined \| void>\n \| undefined\n \| void;\n}\n\nexport interface ExecutionTaskHitBy {\n from: string;\n context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n output?: TaskOutput;\n log?: TaskLog;\n recorder?: ExecutionRecorderItem[];\n hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n any,\n any,\n any\n >,\n> = E &\n ExecutionTaskReturn<\n E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n ? TaskOutput\n : unknown,\n E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n ? TaskLog\n : unknown\n > & {\n taskId: string;\n status: 'pending' \| 'running' \| 'finished' \| 'failed' \| 'cancelled';\n error?: Error;\n errorMessage?: string;\n errorStack?: string;\n timing?: {\n start: number;\n getUiContextStart?: number;\n getUiContextEnd?: number;\n callAiStart?: number;\n callAiEnd?: number;\n beforeInvokeActionHookStart?: number;\n beforeInvokeActionHookEnd?: number;\n callActionStart?: number;\n callActionEnd?: number;\n afterInvokeActionHookStart?: number;\n afterInvokeActionHookEnd?: number;\n captureAfterCallingSnapshotStart?: number;\n captureAfterCallingSnapshotEnd?: number;\n end?: number;\n cost?: number;\n };\n usage?: AIUsageInfo;\n searchAreaUsage?: AIUsageInfo;\n reasoning_content?: string;\n };\n\nexport interface IExecutionDump extends DumpMeta {\n name: string;\n description?: string;\n tasks: ExecutionTask[];\n aiActContext?: string;\n}\n\n/\n Replacer function for JSON serialization that handles Page, Browser objects and ScreenshotItem\n /\nfunction replacerForDumpSerialization(_key: string, value: any): any {\n if (value && value.constructor?.name === 'Page') {\n return '[Page object]';\n }\n if (value && value.constructor?.name === 'Browser') {\n return '[Browser object]';\n }\n // Handle ScreenshotItem serialization\n if (value && typeof value.toSerializable === 'function') {\n return value.toSerializable();\n }\n return value;\n}\n\n/\n Reviver function for JSON deserialization that handles ScreenshotItem formats.\n \n BEHAVIOR:\n * - For { $screenshot: \"id\" } format: Left as-is (plain object)\n * Consumer must use imageMap to restore base64 data\n * - For { base64: \"...\" } format: Creates ScreenshotItem from base64 data\n \n @param key - JSON key being processed\n * @param value - JSON value being processed\n * @returns Restored value\n /\nfunction reviverForDumpDeserialization(key: string, value: any): any {\n // Only process screenshot fields\n if (key !== 'screenshot' \|\| typeof value !== 'object' \|\| value === null) {\n return value;\n }\n\n // Handle serialized format: { $screenshot: \"id\" }\n // Leave as plain object — consumer uses imageMap to restore\n if (ScreenshotItem.isSerialized(value)) {\n return value;\n }\n\n // Handle inline base64 format: { base64: \"...\" }\n if ('base64' in value && typeof value.base64 === 'string') {\n return value;\n }\n\n return value;\n}\n\n/\n ExecutionDump class for serializing and deserializing execution dumps\n /\nexport class ExecutionDump implements IExecutionDump {\n logTime: number;\n name: string;\n description?: string;\n tasks: ExecutionTask[];\n aiActContext?: string;\n\n constructor(data: IExecutionDump) {\n this.logTime = data.logTime;\n this.name = data.name;\n this.description = data.description;\n this.tasks = data.tasks;\n this.aiActContext = data.aiActContext;\n }\n\n /\n Serialize the ExecutionDump to a JSON string\n /\n serialize(indents?: number): string {\n return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n }\n\n /\n Convert to a plain object for JSON serialization\n /\n toJSON(): IExecutionDump {\n return {\n logTime: this.logTime,\n name: this.name,\n description: this.description,\n tasks: this.tasks.map((task) => ({\n ...task,\n recorder: task.recorder \|\| [],\n })),\n aiActContext: this.aiActContext,\n };\n }\n\n /\n Create an ExecutionDump instance from a serialized JSON string\n /\n static fromSerializedString(serialized: string): ExecutionDump {\n const parsed = JSON.parse(\n serialized,\n reviverForDumpDeserialization,\n ) as IExecutionDump;\n return new ExecutionDump(parsed);\n }\n\n /\n Create an ExecutionDump instance from a plain object\n /\n static fromJSON(data: IExecutionDump): ExecutionDump {\n return new ExecutionDump(data);\n }\n\n /\n Collect all ScreenshotItem instances from tasks.\n * Scans through uiContext and recorder items to find screenshots.\n \n @returns Array of ScreenshotItem instances\n /\n collectScreenshots(): ScreenshotItem[] {\n const screenshots: ScreenshotItem[] = [];\n\n for (const task of this.tasks) {\n // Collect uiContext.screenshot if present\n if (task.uiContext?.screenshot instanceof ScreenshotItem) {\n screenshots.push(task.uiContext.screenshot);\n }\n\n // Collect recorder screenshots\n if (task.recorder) {\n for (const record of task.recorder) {\n if (record.screenshot instanceof ScreenshotItem) {\n screenshots.push(record.screenshot);\n }\n }\n }\n }\n\n return screenshots;\n }\n}\n\n/\ntask - service-locate\n/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n element: LocateResultElement \| null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightLocateParam,\n ExecutionTaskInsightLocateOutput,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/\ntask - service-query\n/\nexport interface ExecutionTaskInsightQueryParam {\n dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightQueryParam,\n any,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/\ntask - assertion\n/\nexport interface ExecutionTaskInsightAssertionParam {\n assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightAssertionParam,\n ServiceAssertionResponse,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/\ntask - action (i.e. interact) \n/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n 'Action Space',\n ActionParam,\n void,\n void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/\ntask - Log\n/\n\nexport type ExecutionTaskLogApply<\n LogParam = {\n content: string;\n },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/\ntask - planning\n/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n 'Planning',\n {\n userInstruction: string;\n aiActContext?: string;\n },\n PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/\ntask - planning-locate\n/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n element: LocateResultElement \| null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n 'Planning',\n ExecutionTaskPlanningLocateParam,\n ExecutionTaskPlanningLocateOutput,\n ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/\nGrouped dump\n/\nexport interface IGroupedActionDump {\n sdkVersion: string;\n groupName: string;\n groupDescription?: string;\n modelBriefs: string[];\n executions: IExecutionDump[];\n deviceType?: string;\n}\n\n/\n GroupedActionDump class for serializing and deserializing grouped action dumps\n /\nexport class GroupedActionDump implements IGroupedActionDump {\n sdkVersion: string;\n groupName: string;\n groupDescription?: string;\n modelBriefs: string[];\n executions: ExecutionDump[];\n deviceType?: string;\n\n constructor(data: IGroupedActionDump) {\n this.sdkVersion = data.sdkVersion;\n this.groupName = data.groupName;\n this.groupDescription = data.groupDescription;\n this.modelBriefs = data.modelBriefs;\n this.executions = data.executions.map((exec) =>\n exec instanceof ExecutionDump ? exec : ExecutionDump.fromJSON(exec),\n );\n this.deviceType = data.deviceType;\n }\n\n /\n Serialize the GroupedActionDump to a JSON string\n * Uses compact { $screenshot: id } format\n /\n serialize(indents?: number): string {\n return JSON.stringify(this.toJSON(), replacerForDumpSerialization, indents);\n }\n\n /\n Serialize the GroupedActionDump with inline screenshots to a JSON string.\n * Each ScreenshotItem is replaced with { base64: \"...\", capturedAt }.\n /\n serializeWithInlineScreenshots(indents?: number): string {\n const processValue = (obj: unknown): unknown => {\n if (obj instanceof ScreenshotItem) {\n return { base64: obj.base64, capturedAt: obj.capturedAt };\n }\n if (Array.isArray(obj)) {\n return obj.map(processValue);\n }\n if (obj && typeof obj === 'object') {\n const entries = Object.entries(obj).map(([key, value]) => [\n key,\n processValue(value),\n ]);\n return Object.fromEntries(entries);\n }\n return obj;\n };\n\n const data = processValue(this.toJSON());\n return JSON.stringify(data, null, indents);\n }\n\n /\n Convert to a plain object for JSON serialization\n /\n toJSON(): IGroupedActionDump {\n return {\n sdkVersion: this.sdkVersion,\n groupName: this.groupName,\n groupDescription: this.groupDescription,\n modelBriefs: this.modelBriefs,\n executions: this.executions.map((exec) => exec.toJSON()),\n deviceType: this.deviceType,\n };\n }\n\n /\n Create a GroupedActionDump instance from a serialized JSON string\n /\n static fromSerializedString(serialized: string): GroupedActionDump {\n const parsed = JSON.parse(\n serialized,\n reviverForDumpDeserialization,\n ) as IGroupedActionDump;\n return new GroupedActionDump(parsed);\n }\n\n /\n Create a GroupedActionDump instance from a plain object\n /\n static fromJSON(data: IGroupedActionDump): GroupedActionDump {\n return new GroupedActionDump(data);\n }\n\n /\n Collect all ScreenshotItem instances from all executions.\n \n @returns Array of all ScreenshotItem instances across all executions\n /\n collectAllScreenshots(): ScreenshotItem[] {\n const screenshots: ScreenshotItem[] = [];\n for (const execution of this.executions) {\n screenshots.push(...execution.collectScreenshots());\n }\n return screenshots;\n }\n\n /\n Serialize the dump to files with screenshots as separate PNG files.\n * Creates:\n * - {basePath} - dump JSON with { $screenshot: id } references\n * - {basePath}.screenshots/ - PNG files\n * - {basePath}.screenshots.json - ID to path mapping\n \n @param basePath - Base path for the dump file\n /\n serializeToFiles(basePath: string): void {\n const screenshotsDir = `${basePath}.screenshots`;\n if (!existsSync(screenshotsDir)) {\n mkdirSync(screenshotsDir, { recursive: true });\n }\n\n // Write screenshots to separate files\n const screenshotMap: Record<string, string> = {};\n const screenshots = this.collectAllScreenshots();\n\n for (const screenshot of screenshots) {\n if (screenshot.hasBase64()) {\n const imagePath = join(\n screenshotsDir,\n `${screenshot.id}.${screenshot.extension}`,\n );\n const rawBase64 = screenshot.rawBase64;\n writeFileSync(imagePath, Buffer.from(rawBase64, 'base64'));\n screenshotMap[screenshot.id] = imagePath;\n }\n }\n\n // Write screenshot map file\n writeFileSync(\n `${basePath}.screenshots.json`,\n JSON.stringify(screenshotMap),\n 'utf-8',\n );\n\n // Write dump JSON with references\n writeFileSync(basePath, this.serialize(), 'utf-8');\n }\n\n /\n Read dump from files and return JSON string with inline screenshots.\n * Reads the dump JSON and screenshot files, then inlines the base64 data.\n \n @param basePath - Base path for the dump file\n * @returns JSON string with inline screenshots ({ base64: \"...\" } format)\n /\n static fromFilesAsInlineJson(basePath: string): string {\n const dumpString = readFileSync(basePath, 'utf-8');\n const screenshotsMapPath = `${basePath}.screenshots.json`;\n\n if (!existsSync(screenshotsMapPath)) {\n return dumpString;\n }\n\n // Read screenshot map and build imageMap from files\n const screenshotMap: Record<string, string> = JSON.parse(\n readFileSync(screenshotsMapPath, 'utf-8'),\n );\n\n const imageMap: Record<string, string> = {};\n for (const [id, filePath] of Object.entries(screenshotMap)) {\n if (existsSync(filePath)) {\n const data = readFileSync(filePath);\n const mime =\n filePath.endsWith('.jpeg') \|\| filePath.endsWith('.jpg')\n ? 'jpeg'\n : 'png';\n imageMap[id] = `data:image/${mime};base64,${data.toString('base64')}`;\n }\n }\n\n // Restore image references\n const dumpData = JSON.parse(dumpString);\n const processedData = restoreImageReferences(\n dumpData,\n (id) => imageMap[id] ?? '',\n );\n return JSON.stringify(processedData);\n }\n\n /\n Clean up all files associated with a serialized dump.\n \n @param basePath - Base path for the dump file\n /\n static cleanupFiles(basePath: string): void {\n const filesToClean = [\n basePath,\n `${basePath}.screenshots.json`,\n `${basePath}.screenshots`,\n ];\n\n for (const filePath of filesToClean) {\n try {\n rmSync(filePath, { force: true, recursive: true });\n } catch {\n // Ignore errors - file may already be deleted\n }\n }\n }\n\n /\n Get all file paths associated with a serialized dump.\n \n @param basePath - Base path for the dump file\n * @returns Array of all associated file paths\n /\n static getFilePaths(basePath: string): string[] {\n return [\n basePath,\n `${basePath}.screenshots.json`,\n `${basePath}.screenshots`,\n ];\n }\n}\n\nexport type InterfaceType =\n \| 'puppeteer'\n \| 'playwright'\n \| 'static'\n \| 'chrome-extension-proxy'\n \| 'android'\n \| string;\n\nexport interface StreamingCodeGenerationOptions {\n /* Whether to enable streaming output /\n stream?: boolean;\n /* Callback function to handle streaming chunks /\n onChunk?: StreamingCallback;\n /* Callback function to handle streaming completion /\n onComplete?: (finalCode: string) => void;\n /* Callback function to handle streaming errors /\n onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n /* The incremental content chunk /\n content: string;\n /* The reasoning content /\n reasoning_content: string;\n /* The accumulated content so far /\n accumulated: string;\n /* Whether this is the final chunk /\n isComplete: boolean;\n /* Token usage information if available /\n usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n /* The final accumulated content /\n content: string;\n /* Token usage information /\n usage?: AIUsageInfo;\n /* Whether the response was streamed /\n isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n name: string;\n description?: string;\n interfaceAlias?: string;\n paramSchema?: z.ZodType<TParam>;\n call: (param: TParam, context: ExecutorContext) => Promise<TReturn> \| TReturn;\n delayAfterRunner?: number;\n /\n An example param object for this action.\n * Locate fields with { prompt } will automatically get bbox injected when needed.\n /\n sample?: { [K in keyof TParam]?: any };\n}\n\n/\n Type utilities for extracting types from DeviceAction definitions\n /\n\n/\n Extract parameter type from a DeviceAction\n /\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n Action extends DeviceAction<infer P, any> ? P : never;\n\n/\n Extract return type from a DeviceAction\n /\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n Action extends DeviceAction<any, infer R> ? R : never;\n\n/\n Web-specific types\n /\nexport interface WebElementInfo extends BaseElement {\n id: string;\n attributes: {\n nodeType: NodeType;\n [key: string]: string;\n };\n}\n\n/\n Agent\n /\n\nexport type CacheConfig = {\n strategy?: 'read-only' \| 'read-write' \| 'write-only';\n id: string;\n cacheAdapter?: import('./agent/cache-adapter').CacheAdapter;\n};\n\nexport type Cache =\n \| false // No read, no write\n \| true // Will throw error at runtime - deprecated\n \| CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n testId?: string;\n // @deprecated\n cacheId?: string; // Keep backward compatibility, but marked as deprecated\n groupName?: string;\n groupDescription?: string;\n / if auto generate report, default true /\n generateReport?: boolean;\n / if auto print report msg, default true /\n autoPrintReportMsg?: boolean;\n\n /\n Use directory-based report format with separate image files.\n \n When enabled:\n * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n * - Report is generated as `index.html` with relative image paths\n * - Reduces memory usage and report file size\n \n IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n * work due to browser CORS restrictions.\n \n @default 'single-html'\n /\n outputFormat?: 'single-html' \| 'html-and-external-assets';\n\n onTaskStartTip?: OnTaskStartTip;\n aiActContext?: string;\n aiActionContext?: string;\n / custom report file name /\n reportFileName?: string;\n modelConfig?: TModelConfig;\n cache?: Cache;\n /\n Maximum number of replanning cycles for aiAct.\n * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n /\n replanningCycleLimit?: number;\n\n /\n Wait time in milliseconds after each action execution.\n * This allows the UI to settle and stabilize before the next action.\n * Defaults to 300ms when not provided.\n /\n waitAfterAction?: number;\n\n /\n When set to true, Midscene will use the target device's time (Android/iOS)\n * instead of the system time. Useful when the device time differs from the\n * host machine. Default: false\n /\n useDeviceTimestamp?: boolean;\n\n /\n Custom screenshot shrink factor to reduce AI token usage.\n * When set, the screenshot will be scaled down by this factor from the physical resolution.\n \n Example:\n * - Physical screen width: 3000px, dpr=6\n * - Logical width: 500px\n * - screenshotShrinkFactor: 2\n * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n * - AI analyzes the 1500px screenshot\n * - Coordinates are transformed back to logical (500px) before actions execute\n \n Benefits:\n * - Reduces token usage for high-resolution screenshots\n * - Maintains accuracy by scaling coordinates appropriately\n \n Must be >= 1 (shrinking only, enlarging is not supported).\n \n @default 1 (no shrinking, uses original physical screenshot)\n /\n screenshotShrinkFactor?: number;\n\n /\n Custom OpenAI client factory function\n \n If provided, this function will be called to create OpenAI client instances\n * for each AI call, allowing you to:\n * - Wrap clients with observability tools (langsmith, langfuse)\n * - Use custom OpenAI-compatible clients\n * - Apply different configurations based on intent\n \n @param config - Resolved model configuration\n * @returns OpenAI client instance (original or wrapped)\n \n @example\n * ```typescript\n * createOpenAIClient: async (openai, opts) => {\n * // Wrap with langsmith for planning tasks\n * if (opts.baseURL?.includes('planning')) {\n * return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n * }\n \n return openai;\n * }\n * ```\n */\n createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n \| 'passed'\n \| 'failed'\n \| 'timedOut'\n \| 'skipped'\n \| 'interrupted';\n\nexport interface ReportFileWithAttributes {\n reportFilePath: string;\n reportAttributes: {\n testDuration: number;\n testStatus: TestStatus;\n testTitle: string;\n testId: string;\n testDescription: string;\n };\n}\n"],"names":["UIContext","ServiceError","Error","message","dump","replacerForDumpSerialization","_key","value","reviverForDumpDeserialization","key","ScreenshotItem","ExecutionDump","indents","JSON","task","serialized","parsed","data","screenshots","record","GroupedActionDump","processValue","obj","Array","entries","Object","exec","execution","basePath","screenshotsDir","existsSync","mkdirSync","screenshotMap","screenshot","imagePath","join","rawBase64","writeFileSync","Buffer","dumpString","readFileSync","screenshotsMapPath","imageMap","id","filePath","mime","dumpData","processedData","restoreImageReferences","filesToClean","rmSync"],"mappings":";;;;;AAAqD;;;;;;;;;;AAsJ9C,MAAeA;AA4BtB;AA+EO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF;AA0YA,SAASC,6BAA6BC,IAAY,EAAEC,KAAU;IAC5D,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,QACvC,OAAO;IAET,IAAIA,SAASA,MAAM,WAAW,EAAE,SAAS,WACvC,OAAO;IAGT,IAAIA,SAAS,AAAgC,cAAhC,OAAOA,MAAM,cAAc,EACtC,OAAOA,MAAM,cAAc;IAE7B,OAAOA;AACT;AAcA,SAASC,8BAA8BC,GAAW,EAAEF,KAAU;IAE5D,IAAIE,AAAQ,iBAARA,OAAwB,AAAiB,YAAjB,OAAOF,SAAsBA,AAAU,SAAVA,OACvD,OAAOA;IAKT,IAAIG,eAAe,YAAY,CAACH,QAC9B,OAAOA;IAIL,YAAYA,SAAgBA,MAAM,MAAM;IAI5C,OAAOA;AACT;AAKO,MAAMI;IAkBX,UAAUC,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAKA,SAAyB;QACvB,OAAO;YACL,SAAS,IAAI,CAAC,OAAO;YACrB,MAAM,IAAI,CAAC,IAAI;YACf,aAAa,IAAI,CAAC,WAAW;YAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAACE,OAAU;oBAC/B,GAAGA,IAAI;oBACP,UAAUA,KAAK,QAAQ,IAAI,EAAE;gBAC/B;YACA,cAAc,IAAI,CAAC,YAAY;QACjC;IACF;IAKA,OAAO,qBAAqBC,UAAkB,EAAiB;QAC7D,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIG,cAAcK;IAC3B;IAKA,OAAO,SAASC,IAAoB,EAAiB;QACnD,OAAO,IAAIN,cAAcM;IAC3B;IAQA,qBAAuC;QACrC,MAAMC,cAAgC,EAAE;QAExC,KAAK,MAAMJ,QAAQ,IAAI,CAAC,KAAK,CAAE;YAE7B,IAAIA,KAAK,SAAS,EAAE,sBAAsBJ,gBACxCQ,YAAY,IAAI,CAACJ,KAAK,SAAS,CAAC,UAAU;YAI5C,IAAIA,KAAK,QAAQ,EACf;gBAAA,KAAK,MAAMK,UAAUL,KAAK,QAAQ,CAChC,IAAIK,OAAO,UAAU,YAAYT,gBAC/BQ,YAAY,IAAI,CAACC,OAAO,UAAU;YAEtC;QAEJ;QAEA,OAAOD;IACT;IA3EA,YAAYD,IAAoB,CAAE;QANlC;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,OAAO,GAAGA,KAAK,OAAO;QAC3B,IAAI,CAAC,IAAI,GAAGA,KAAK,IAAI;QACrB,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,YAAY,GAAGA,KAAK,YAAY;IACvC;AAsEF;AAwIO,MAAMG;IAuBX,UAAUR,OAAgB,EAAU;QAClC,OAAOC,KAAK,SAAS,CAAC,IAAI,CAAC,MAAM,IAAIR,8BAA8BO;IACrE;IAMA,+BAA+BA,OAAgB,EAAU;QACvD,MAAMS,eAAe,CAACC;YACpB,IAAIA,eAAeZ,gBACjB,OAAO;gBAAE,QAAQY,IAAI,MAAM;gBAAE,YAAYA,IAAI,UAAU;YAAC;YAE1D,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAACD;YAEjB,IAAIC,OAAO,AAAe,YAAf,OAAOA,KAAkB;gBAClC,MAAME,UAAUC,OAAO,OAAO,CAACH,KAAK,GAAG,CAAC,CAAC,CAACb,KAAKF,MAAM,GAAK;wBACxDE;wBACAY,aAAad;qBACd;gBACD,OAAOkB,OAAO,WAAW,CAACD;YAC5B;YACA,OAAOF;QACT;QAEA,MAAML,OAAOI,aAAa,IAAI,CAAC,MAAM;QACrC,OAAOR,KAAK,SAAS,CAACI,MAAM,MAAML;IACpC;IAKA,SAA6B;QAC3B,OAAO;YACL,YAAY,IAAI,CAAC,UAAU;YAC3B,WAAW,IAAI,CAAC,SAAS;YACzB,kBAAkB,IAAI,CAAC,gBAAgB;YACvC,aAAa,IAAI,CAAC,WAAW;YAC7B,YAAY,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAACc,OAASA,KAAK,MAAM;YACrD,YAAY,IAAI,CAAC,UAAU;QAC7B;IACF;IAKA,OAAO,qBAAqBX,UAAkB,EAAqB;QACjE,MAAMC,SAASH,KAAK,KAAK,CACvBE,YACAP;QAEF,OAAO,IAAIY,kBAAkBJ;IAC/B;IAKA,OAAO,SAASC,IAAwB,EAAqB;QAC3D,OAAO,IAAIG,kBAAkBH;IAC/B;IAOA,wBAA0C;QACxC,MAAMC,cAAgC,EAAE;QACxC,KAAK,MAAMS,aAAa,IAAI,CAAC,UAAU,CACrCT,YAAY,IAAI,IAAIS,UAAU,kBAAkB;QAElD,OAAOT;IACT;IAWA,iBAAiBU,QAAgB,EAAQ;QACvC,MAAMC,iBAAiB,GAAGD,SAAS,YAAY,CAAC;QAChD,IAAI,CAACE,WAAWD,iBACdE,UAAUF,gBAAgB;YAAE,WAAW;QAAK;QAI9C,MAAMG,gBAAwC,CAAC;QAC/C,MAAMd,cAAc,IAAI,CAAC,qBAAqB;QAE9C,KAAK,MAAMe,cAAcf,YACvB,IAAIe,WAAW,SAAS,IAAI;YAC1B,MAAMC,YAAYC,KAChBN,gBACA,GAAGI,WAAW,EAAE,CAAC,CAAC,EAAEA,WAAW,SAAS,EAAE;YAE5C,MAAMG,YAAYH,WAAW,SAAS;YACtCI,cAAcH,WAAWI,OAAO,IAAI,CAACF,WAAW;YAChDJ,aAAa,CAACC,WAAW,EAAE,CAAC,GAAGC;QACjC;QAIFG,cACE,GAAGT,SAAS,iBAAiB,CAAC,EAC9Bf,KAAK,SAAS,CAACmB,gBACf;QAIFK,cAAcT,UAAU,IAAI,CAAC,SAAS,IAAI;IAC5C;IASA,OAAO,sBAAsBA,QAAgB,EAAU;QACrD,MAAMW,aAAaC,aAAaZ,UAAU;QAC1C,MAAMa,qBAAqB,GAAGb,SAAS,iBAAiB,CAAC;QAEzD,IAAI,CAACE,WAAWW,qBACd,OAAOF;QAIT,MAAMP,gBAAwCnB,KAAK,KAAK,CACtD2B,aAAaC,oBAAoB;QAGnC,MAAMC,WAAmC,CAAC;QAC1C,KAAK,MAAM,CAACC,IAAIC,SAAS,IAAInB,OAAO,OAAO,CAACO,eAC1C,IAAIF,WAAWc,WAAW;YACxB,MAAM3B,OAAOuB,aAAaI;YAC1B,MAAMC,OACJD,SAAS,QAAQ,CAAC,YAAYA,SAAS,QAAQ,CAAC,UAC5C,SACA;YACNF,QAAQ,CAACC,GAAG,GAAG,CAAC,WAAW,EAAEE,KAAK,QAAQ,EAAE5B,KAAK,QAAQ,CAAC,WAAW;QACvE;QAIF,MAAM6B,WAAWjC,KAAK,KAAK,CAAC0B;QAC5B,MAAMQ,gBAAgBC,uBACpBF,UACA,CAACH,KAAOD,QAAQ,CAACC,GAAG,IAAI;QAE1B,OAAO9B,KAAK,SAAS,CAACkC;IACxB;IAOA,OAAO,aAAanB,QAAgB,EAAQ;QAC1C,MAAMqB,eAAe;YACnBrB;YACA,GAAGA,SAAS,iBAAiB,CAAC;YAC9B,GAAGA,SAAS,YAAY,CAAC;SAC1B;QAED,KAAK,MAAMgB,YAAYK,aACrB,IAAI;YACFC,OAAON,UAAU;gBAAE,OAAO;gBAAM,WAAW;YAAK;QAClD,EAAE,OAAM,CAER;IAEJ;IAQA,OAAO,aAAahB,QAAgB,EAAY;QAC9C,OAAO;YACLA;YACA,GAAGA,SAAS,iBAAiB,CAAC;YAC9B,GAAGA,SAAS,YAAY,CAAC;SAC1B;IACH;IA9MA,YAAYX,IAAwB,CAAE;QAPtC;QACA;QACA;QACA;QACA;QACA;QAGE,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU;QACjC,IAAI,CAAC,SAAS,GAAGA,KAAK,SAAS;QAC/B,IAAI,CAAC,gBAAgB,GAAGA,KAAK,gBAAgB;QAC7C,IAAI,CAAC,WAAW,GAAGA,KAAK,WAAW;QACnC,IAAI,CAAC,UAAU,GAAGA,KAAK,UAAU,CAAC,GAAG,CAAC,CAACS,OACrCA,gBAAgBf,gBAAgBe,OAAOf,cAAc,QAAQ,CAACe;QAEhE,IAAI,CAAC,UAAU,GAAGT,KAAK,UAAU;IACnC;AAsMF"}

package/dist/es/utils.mjs CHANGED Viewed

@@ -159,7 +159,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "1.6.10";
+    return "1.6.11";
 }
 function debugLog(...message) {
     const debugMode = process.env[MIDSCENE_DEBUG_MODE];

package/dist/lib/agent/agent.js CHANGED Viewed

@@ -483,7 +483,7 @@ class Agent {
         const assertionText = 'string' == typeof assertion ? assertion : assertion.prompt;
         let beforeScreenshot;
         let afterScreenshot;
-        const mode = opt?.screenshotMode || 'auto';
+        const mode = opt?.screenshotMode || 'currentOnly';
         switch(mode){
             case 'manual':
                 beforeScreenshot = opt?.beforeScreenshot;