npm - @midscene/core - Versions diffs - 1.12.5-beta-20260908034641.0 → 1.12.5 - Mend

@midscene/core 1.12.5-beta-20260908034641.0 → 1.12.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (165) hide show

package/dist/es/agent/task-builder.mjs CHANGED Viewed

@@ -3,11 +3,10 @@ import { findActionInActionSpaceOrThrow } from "../common.mjs";
 import { setTimingFieldOnce } from "../task-timing.mjs";
 import { ServiceError } from "../types.mjs";
 import { sleep } from "../utils.mjs";
-import { generateElementByRect } from "@midscene/shared/extractor";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { withUsageIntent } from "./usage-intent.mjs";
-import { ifLocateParamHasLocatedPixelBbox, matchElementFromCache, matchElementFromPlan, transformLogicalElementToScreenshot, transformLogicalRectToScreenshotRect } from "./utils.mjs";
+import { ifLocateParamHasLocatedPixelResult, matchElementFromCache, matchElementFromPlan, transformLogicalElementToScreenshot } from "./utils.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -25,14 +24,9 @@ function hasNonEmptyCache(cache) {
 function invalidLocateElementReason(element) {
     const values = [
         element.center?.[0],
-        element.center?.[1],
-        element.rect?.left,
-        element.rect?.top,
-        element.rect?.width,
-        element.rect?.height
+        element.center?.[1]
     ];
-    if (values.some((value)=>'number' != typeof value || !Number.isFinite(value))) return `Invalid locate result coordinates: ${JSON.stringify(element)}`;
-    if (element.rect.width <= 0 || element.rect.height <= 0) return `Invalid locate result rect size: ${JSON.stringify(element)}`;
+    if (element.center?.length !== 2 || values.some((value)=>'number' != typeof value || !Number.isFinite(value))) return `Invalid locate result coordinates: ${JSON.stringify(element)}`;
 }
 function normalizeLocateParam(param) {
     if ('string' == typeof param) return {
@@ -109,7 +103,7 @@ class TaskBuilder {
         locateFields.forEach((field)=>{
             if (param[field]) {
                 const locatePlan = locatePlanForLocate(param[field]);
-                debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`, `hasLocatedPixelBbox=${ifLocateParamHasLocatedPixelBbox(param[field])}`);
+                debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`, `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`);
                 const locateTask = this.createLocateTask(locatePlan, param[field], context, (result)=>{
                     param[field] = result;
                 });
@@ -214,8 +208,8 @@ class TaskBuilder {
             executor: async (taskContext)=>{
                 const { task } = taskContext;
                 let { uiContext } = taskContext;
-                const paramWithLocatedPixelBbox = ifLocateParamHasLocatedPixelBbox(locateParam) ? locateParam : void 0;
-                assert(locateParam?.prompt || paramWithLocatedPixelBbox, `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(locateParam)}`);
+                const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(locateParam) ? locateParam : void 0;
+                assert(locateParam?.prompt || paramWithLocatedPixelResult, `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(locateParam)}`);
                 if (!uiContext) uiContext = await this.service.contextRetrieverFn();
                 assert(uiContext, 'uiContext is required for Service task');
                 const { shrunkShotToLogicalRatio } = uiContext;
@@ -236,7 +230,7 @@ class TaskBuilder {
                     if (dump.taskInfo?.searchAreaUsage) task.searchAreaUsage = withUsageIntent(dump.taskInfo.searchAreaUsage, 'default');
                     if (dump.taskInfo?.reasoning_content) task.reasoning_content = dump.taskInfo.reasoning_content;
                 };
-                const planLocatedElement = paramWithLocatedPixelBbox ? matchElementFromPlan(paramWithLocatedPixelBbox) : void 0;
+                const planLocatedElement = paramWithLocatedPixelResult ? matchElementFromPlan(paramWithLocatedPixelResult) : void 0;
                 const elementFromPlan = locateParam.deepLocate ? void 0 : planLocatedElement;
                 const isPlanDirectHit = !!elementFromPlan;
                 let rectFromXpath;
@@ -247,7 +241,14 @@ class TaskBuilder {
                         ]
                     });
                 } catch  {}
-                const elementFromXpath = rectFromXpath ? generateElementByRect(transformLogicalRectToScreenshotRect(rectFromXpath, shrunkShotToLogicalRatio), 'string' == typeof locateParam.prompt ? locateParam.prompt : locateParam.prompt?.prompt || '') : void 0;
+                const elementFromXpath = rectFromXpath ? transformLogicalElementToScreenshot({
+                    center: [
+                        rectFromXpath.left + rectFromXpath.width / 2,
+                        rectFromXpath.top + rectFromXpath.height / 2
+                    ],
+                    description: 'string' == typeof locateParam.prompt ? locateParam.prompt : locateParam.prompt?.prompt || '',
+                    rect: rectFromXpath
+                }, shrunkShotToLogicalRatio) : void 0;
                 const isXpathHit = !!elementFromXpath;
                 const cachePrompt = locateParam.prompt;
                 const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
@@ -285,14 +286,10 @@ class TaskBuilder {
                 const locateCacheAlreadyExists = hasNonEmptyCache(locateCacheRecord?.cacheContent?.cache);
                 let currentCacheEntry;
                 if (element && this.taskCache && !isCacheHit && (!isPlanDirectHit || !locateCacheAlreadyExists) && false !== locateParam.cacheable) if (this.interface.cacheFeatureForPoint) try {
-                    let pointForCache = element.center;
-                    if (1 !== shrunkShotToLogicalRatio) {
-                        pointForCache = [
-                            Math.round(element.center[0] / shrunkShotToLogicalRatio),
-                            Math.round(element.center[1] / shrunkShotToLogicalRatio)
-                        ];
-                        debug('Transformed coordinates for cacheFeatureForPoint: %o -> %o', element.center, pointForCache);
-                    }
+                    const pointForCache = [
+                        Math.round(element.center[0] / shrunkShotToLogicalRatio),
+                        Math.round(element.center[1] / shrunkShotToLogicalRatio)
+                    ];
                     const feature = await this.interface.cacheFeatureForPoint(pointForCache, {
                         targetDescription: 'string' == typeof locateParam.prompt ? locateParam.prompt : locateParam.prompt?.prompt,
                         modelRuntime: defaultModel
@@ -315,10 +312,10 @@ class TaskBuilder {
                     throw new Error(`Element not found: ${locateParam.prompt}`);
                 }
                 let hitBy;
-                if (isPlanDirectHit && paramWithLocatedPixelBbox) hitBy = {
+                if (isPlanDirectHit && paramWithLocatedPixelResult) hitBy = {
                     from: 'Plan',
                     context: {
-                        locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox
+                        locatedPixelResult: paramWithLocatedPixelResult.locatedPixelResult
                     }
                 };
                 else if (isXpathHit) hitBy = {

package/dist/es/agent/task-builder.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { findActionInActionSpaceOrThrow } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n AIUsageInfo,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n ifLocateParamHasLocatedPixelBbox,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nfunction invalidLocateElementReason(\n element: LocateResultElement,\n): string \| undefined {\n const values = [\n element.center?.[0],\n element.center?.[1],\n element.rect?.left,\n element.rect?.top,\n element.rect?.width,\n element.rect?.height,\n ];\n if (\n values.some((value) => typeof value !== 'number' \|\| !Number.isFinite(value))\n ) {\n return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n }\n if (element.rect.width <= 0 \|\| element.rect.height <= 0) {\n return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n }\n return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n param: string \| DetailedLocateParam,\n): DetailedLocateParam {\n if (typeof param === 'string') {\n return { prompt: param };\n }\n\n const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n const deepLocate = rest.deepLocate ?? deepThink;\n\n return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = normalizeLocateParam(param);\n const locatePlan: PlanningAction<DetailedLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n planningModel: ModelRuntime;\n defaultModel: ModelRuntime;\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n\n constructor({\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n }\n\n public async build(\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n planningModel,\n defaultModel,\n cacheable,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<DetailedLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<DetailedLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n const param = plan.param;\n\n const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n const requiredLocateFields = findAllMidsceneLocatorField(\n action.paramSchema,\n true,\n );\n\n locateFields.forEach((field) => {\n if (param[field]) {\n // Always use createLocateTask for all locate params.\n // This ensures cache writing happens even when locatedPixelBbox is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasLocatedPixelBbox=${ifLocateParamHasLocatedPixelBbox(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n executor: async (taskContext) => {\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug(\n `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.beforeInvokeAction(action.name, param);\n debug(\n `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n })(),\n delayBeforeRunner > 0\n ? sleep(delayBeforeRunner)\n : Promise.resolve(),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n\n const parsedParam = (() => {\n if (!action.paramSchema) {\n return param;\n }\n\n try {\n return parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n })();\n\n setTimingFieldOnce(timing, 'callActionStart');\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(parsedParam, taskContext);\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug(\n `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.afterInvokeAction(action.name, parsedParam);\n debug(\n `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<DetailedLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n let locateParam = normalizeLocateParam(detailedLocateParam);\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n const paramWithLocatedPixelBbox = ifLocateParamHasLocatedPixelBbox(\n locateParam,\n )\n ? locateParam\n : undefined;\n\n assert(\n locateParam?.prompt \|\| paramWithLocatedPixelBbox,\n `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n locateParam,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n searchAreaRawChoiceMessage:\n dump.taskInfo?.searchAreaRawChoiceMessage,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n task.searchArea = dump.taskInfo?.searchArea;\n if (dump.taskInfo?.searchAreaUsage) {\n // The deepLocate search-area call belongs to the same locate task,\n // so it shares the companion usage intent.\n task.searchAreaUsage = withUsageIntent(\n dump.taskInfo.searchAreaUsage,\n 'default',\n );\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n const planLocatedElement = paramWithLocatedPixelBbox\n ? matchElementFromPlan(paramWithLocatedPixelBbox)\n : undefined;\n\n // from locatedPixelBbox (direct plan hit)\n // when deepLocate is enabled, locatedPixelBbox should be used as search\n // area hint, not as a final direct hit\n const elementFromPlan = locateParam.deepLocate\n ? undefined\n : planLocatedElement;\n const isPlanDirectHit = !!elementFromPlan;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanDirectHit &&\n locateParam.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [locateParam.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? generateElementByRect(\n // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n transformLogicalRectToScreenshotRect(\n rectFromXpath,\n shrunkShotToLogicalRatio,\n ),\n typeof locateParam.prompt === 'string'\n ? locateParam.prompt\n : locateParam.prompt?.prompt \|\| '',\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = locateParam.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanDirectHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n locateParam.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n const elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isCacheHit = !!elementFromCache;\n\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n const timing = taskContext.task.timing;\n if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n locateParam,\n {\n context: uiContext,\n planLocatedElement,\n },\n defaultModel,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n\n const element =\n elementFromPlan \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n if (element) {\n const invalidElementReason = invalidLocateElementReason(element);\n if (invalidElementReason) {\n if (locateDump) {\n throw new ServiceError(invalidElementReason, locateDump);\n }\n throw new Error(invalidElementReason);\n }\n }\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read)\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n !isCacheHit &&\n (!isPlanDirectHit \|\| !locateCacheAlreadyExists) &&\n locateParam.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n let pointForCache: [number, number] = element.center;\n if (shrunkShotToLogicalRatio !== 1) {\n pointForCache = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n debug(\n 'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n element.center,\n pointForCache,\n );\n }\n\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof locateParam.prompt === 'string'\n ? locateParam.prompt\n : locateParam.prompt?.prompt,\n modelRuntime: defaultModel,\n },\n );\n if (hasNonEmptyCache(feature)) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${locateParam.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${locateParam.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n hitBy = {\n from: 'Plan',\n context: {\n locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: locateParam.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n const promptDisplay = locateParam.promptDisplay;\n const elementForAction = promptDisplay\n ? {\n ...element,\n description: promptDisplay,\n }\n : element;\n\n if (promptDisplay && locateDump?.matchedElement) {\n locateDump.matchedElement = locateDump.matchedElement.map(\n (matchedElement) => ({\n ...matchedElement,\n description: promptDisplay,\n }),\n );\n }\n\n onResult?.(elementForAction);\n\n return {\n output: {\n element: {\n ...elementForAction,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","Error","shrunkShotToLogicalRatio","parsedParam","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","promptDisplay","elementForAction","matchedElement","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAoCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnE,MAAMI,uBAAuBD,4BAC3BH,OAAO,WAAW,EAClB;QAGFE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EAAE;gBAGhB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,oBAAoB,EAAEwB,iCAAiC/B,KAAK,CAAC8B,MAAM,GAAG;gBAEzE,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEP,UAAU;gBAE3EnC,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;YACjE;QACF;QAEA,MAAMY,OAKF;YACF,MAAM;YACN,SAASZ;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOkB;gBACf,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtChD,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEoC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACElC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEP,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAgB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBf,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMgB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCrD,MACE,CAAC,8DAA8D,EAAEqC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDZ,MACE,CAAC,2DAA2D,EAAEqC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAe,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,wCAAwC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEU,wBAAwB,EAAE,GAAGT;gBACrC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,MAAME,cAAe,AAAC;oBACpB,IAAI,CAACvB,OAAO,WAAW,EACrB,OAAOzB;oBAGT,IAAI;wBACF,OAAOiD,iBAAiBjD,OAAOyB,OAAO,WAAW,EAAE;4BACjDsB;wBACF;oBACF,EAAE,OAAOG,OAAY;wBACnB,MAAM,IAAIJ,MACR,CAAC,8BAA8B,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAEpD,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAOkD;wBAAM;oBAEnB;gBACF;gBAEAX,mBAAmBF,QAAQ;gBAE3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAASH,aAAaZ;gBACjDG,mBAAmBF,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW2B;gBAE/Cb,mBAAmBF,QAAQ;gBAE3B,MAAMgB,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMX,MAAMW;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCjE,MACE,CAAC,6DAA6D,EAAEqC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEuB;wBACpD5D,MACE,CAAC,0DAA0D,EAAEqC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOkB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,uCAAuC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQe;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACoB;IACrB;IAEQ,iBACNjB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEqD,WAAW,EAAE,GAAGzC;QAE7D,IAAI0C,cAAc1D,qBAAqBuD;QAEvC,IAAIxC,AAAcV,WAAdU,WACF2C,cAAc;YACZ,GAAGA,WAAW;YACd3C;QACF;QAGF,IAAIX,cAAc,CAACsD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASvC,KAAK,OAAO;YACrB,UAAU,OAAOkB;gBACf,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMuB,4BAA4B5B,iCAChC0B,eAEEA,cACArD;gBAEJ8B,OACEuB,aAAa,UAAUE,2BACvB,CAAC,iEAAiE,EAAE7D,KAAK,SAAS,CAChF2D,cACC;gBAGL,IAAI,CAACnB,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAES,wBAAwB,EAAE,GAAGT;gBAErC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIc;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb5B,KAAK,GAAG,GAAG;wBACT4B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA5B,KAAK,KAAK,GAAG6B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD5B,KAAK,UAAU,GAAG4B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjB5B,KAAK,eAAe,GAAG6B,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjB5B,KAAK,iBAAiB,GAAG4B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,qBAAqBP,6BACrBvD;gBAKJ,MAAM+D,kBAAkBV,YAAY,UAAU,GAC1CrD,SACA6D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDX,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACZ,YAAY,KAAK;yBAAC;oBAC7B;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMa,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACAtB,2BAEF,AAA8B,YAA9B,OAAOU,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU,MAEpCrD;gBAEJ,MAAMqE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAcjB,YAAY,MAAM;gBACtC,MAAMkB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAjB,YAAY,SAAS;gBAI7B,MAAMsB,mBAAmBF,yBACrBG,oCACEH,wBACA9B,4BAEF3C;gBAEJ,MAAM6E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM7C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACqC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF7B,mBAAmBF,QAAQ;oBAC3BwB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCJ,aACA;wBACE,SAASnB;wBACT2B;oBACF,GACAtD,cACA6C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,cACnBrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRX,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJyE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIxF,SAAS;oBACX,MAAM0F,uBAAuB3F,2BAA2BC;oBACxD,IAAI0F,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,aAAaC,sBAAsBxB;wBAE/C,MAAM,IAAId,MAAMsC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B/F,iBAC/BqF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACE5F,WACA,IAAI,CAAC,SAAS,IACd,CAACuF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7C5B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAI8B,gBAAkC7F,QAAQ,MAAM;oBACpD,IAAIqD,AAA6B,MAA7BA,0BAAgC;wBAClCwC,gBAAgB;4BACdC,KAAK,KAAK,CAAC9F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;4BAC/ByC,KAAK,KAAK,CAAC9F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;yBAChC;wBACD3D,MACE,8DACAM,QAAQ,MAAM,EACd6F;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO9B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAc9C;oBAChB;oBAEF,IAAIrB,iBAAiBmG,UAAU;wBAC7BrG,MACE,uCACAsF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACEvF,MACE,yDACAsF;gBAGN,EAAE,OAAOxB,OAAO;oBACd9D,MAAM,mCAAmC8D;gBAC3C;qBAEA9D,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIkE,YACF,MAAM,IAAIuB,aACR,CAAC,oBAAoB,EAAE1B,YAAY,MAAM,EAAE,EAC3CG;oBAGJ,MAAM,IAAId,MAAM,CAAC,mBAAmB,EAAEW,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAIiC;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOjC,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIwB,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF,MAAMK,gBAAgBlC,YAAY,aAAa;gBAC/C,MAAMmC,mBAAmBD,gBACrB;oBACE,GAAGjG,OAAO;oBACV,aAAaiG;gBACf,IACAjG;gBAEJ,IAAIiG,iBAAiB/B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAACiC,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJpC,WAAWqC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKtD,UAAU,aAAa;wBAC9B;oBACF;oBACAoD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IAzlBA,YAAY,EACVoC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTxE,WAAW,EACXyE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGxE;QACnB,IAAI,CAAC,eAAe,GAAGyE;IACzB;AA8kBF"}
1	+ {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { findActionInActionSpaceOrThrow } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n AIUsageInfo,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n ifLocateParamHasLocatedPixelResult,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nfunction invalidLocateElementReason(\n element: LocateResultElement,\n): string \| undefined {\n const values = [element.center?.[0], element.center?.[1]];\n if (\n element.center?.length !== 2 \|\|\n values.some((value) => typeof value !== 'number' \|\| !Number.isFinite(value))\n ) {\n return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n }\n return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n param: string \| DetailedLocateParam,\n): DetailedLocateParam {\n if (typeof param === 'string') {\n return { prompt: param };\n }\n\n const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n const deepLocate = rest.deepLocate ?? deepThink;\n\n return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = normalizeLocateParam(param);\n const locatePlan: PlanningAction<DetailedLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n planningModel: ModelRuntime;\n defaultModel: ModelRuntime;\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n\n constructor({\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n }\n\n public async build(\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n planningModel,\n defaultModel,\n cacheable,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<DetailedLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<DetailedLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n const param = plan.param;\n\n const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n const requiredLocateFields = findAllMidsceneLocatorField(\n action.paramSchema,\n true,\n );\n\n locateFields.forEach((field) => {\n if (param[field]) {\n // Always use createLocateTask for all locate params.\n // This ensures cache writing happens even when locatedPixelResult is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasLocatedPixelResult=${ifLocateParamHasLocatedPixelResult(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n executor: async (taskContext) => {\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug(\n `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.beforeInvokeAction(action.name, param);\n debug(\n `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n })(),\n delayBeforeRunner > 0\n ? sleep(delayBeforeRunner)\n : Promise.resolve(),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n\n const parsedParam = (() => {\n if (!action.paramSchema) {\n return param;\n }\n\n try {\n return parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n })();\n\n setTimingFieldOnce(timing, 'callActionStart');\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(parsedParam, taskContext);\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug(\n `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.afterInvokeAction(action.name, parsedParam);\n debug(\n `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<DetailedLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n let locateParam = normalizeLocateParam(detailedLocateParam);\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n const paramWithLocatedPixelResult = ifLocateParamHasLocatedPixelResult(\n locateParam,\n )\n ? locateParam\n : undefined;\n\n assert(\n locateParam?.prompt \|\| paramWithLocatedPixelResult,\n `No prompt or id or position or locatedPixelResult to locate, param=${JSON.stringify(\n locateParam,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n searchAreaRawChoiceMessage:\n dump.taskInfo?.searchAreaRawChoiceMessage,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n task.searchArea = dump.taskInfo?.searchArea;\n if (dump.taskInfo?.searchAreaUsage) {\n // The deepLocate search-area call belongs to the same locate task,\n // so it shares the companion usage intent.\n task.searchAreaUsage = withUsageIntent(\n dump.taskInfo.searchAreaUsage,\n 'default',\n );\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n const planLocatedElement = paramWithLocatedPixelResult\n ? matchElementFromPlan(paramWithLocatedPixelResult)\n : undefined;\n\n // from locatedPixelResult (direct plan hit)\n // when deepLocate is enabled, locatedPixelResult should be used as search\n // area hint, not as a final direct hit\n const elementFromPlan = locateParam.deepLocate\n ? undefined\n : planLocatedElement;\n const isPlanDirectHit = !!elementFromPlan;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanDirectHit &&\n locateParam.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [locateParam.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? transformLogicalElementToScreenshot(\n {\n center: [\n rectFromXpath.left + rectFromXpath.width / 2,\n rectFromXpath.top + rectFromXpath.height / 2,\n ],\n description:\n typeof locateParam.prompt === 'string'\n ? locateParam.prompt\n : locateParam.prompt?.prompt \|\| '',\n rect: rectFromXpath,\n },\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = locateParam.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanDirectHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n locateParam.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n const elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isCacheHit = !!elementFromCache;\n\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n const timing = taskContext.task.timing;\n if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n locateParam,\n {\n context: uiContext,\n planLocatedElement,\n },\n defaultModel,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n\n const element =\n elementFromPlan \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n if (element) {\n const invalidElementReason = invalidLocateElementReason(element);\n if (invalidElementReason) {\n if (locateDump) {\n throw new ServiceError(invalidElementReason, locateDump);\n }\n throw new Error(invalidElementReason);\n }\n }\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read)\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n !isCacheHit &&\n (!isPlanDirectHit \|\| !locateCacheAlreadyExists) &&\n locateParam.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n const pointForCache: [number, number] = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof locateParam.prompt === 'string'\n ? locateParam.prompt\n : locateParam.prompt?.prompt,\n modelRuntime: defaultModel,\n },\n );\n if (hasNonEmptyCache(feature)) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${locateParam.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${locateParam.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanDirectHit && paramWithLocatedPixelResult) {\n hitBy = {\n from: 'Plan',\n context: {\n locatedPixelResult:\n paramWithLocatedPixelResult.locatedPixelResult,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: locateParam.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n const promptDisplay = locateParam.promptDisplay;\n const elementForAction = promptDisplay\n ? {\n ...element,\n description: promptDisplay,\n }\n : element;\n\n if (promptDisplay && locateDump?.matchedElement) {\n locateDump.matchedElement = locateDump.matchedElement.map(\n (matchedElement) => ({\n ...matchedElement,\n description: promptDisplay,\n }),\n );\n }\n\n onResult?.(elementForAction);\n\n return {\n output: {\n element: {\n ...elementForAction,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifLocateParamHasLocatedPixelResult","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","Error","shrunkShotToLogicalRatio","parsedParam","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelResult","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","transformLogicalElementToScreenshot","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","promptDisplay","elementForAction","matchedElement","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;AAkCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QAACD,QAAQ,MAAM,EAAE,CAAC,EAAE;QAAEA,QAAQ,MAAM,EAAE,CAAC,EAAE;KAAC;IACzD,IACEA,QAAQ,MAAM,EAAE,WAAW,KAC3BC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;AAG1E;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnE,MAAMI,uBAAuBD,4BAC3BH,OAAO,WAAW,EAClB;QAGFE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EAAE;gBAGhB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,sBAAsB,EAAEwB,mCAAmC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEP,UAAU;gBAE3EnC,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;YACjE;QACF;QAEA,MAAMY,OAKF;YACF,MAAM;YACN,SAASZ;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOkB;gBACf,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtChD,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEoC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACElC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEP,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAgB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBf,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMgB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCrD,MACE,CAAC,8DAA8D,EAAEqC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDZ,MACE,CAAC,2DAA2D,EAAEqC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAe,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,wCAAwC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEU,wBAAwB,EAAE,GAAGT;gBACrC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,MAAME,cAAe,AAAC;oBACpB,IAAI,CAACvB,OAAO,WAAW,EACrB,OAAOzB;oBAGT,IAAI;wBACF,OAAOiD,iBAAiBjD,OAAOyB,OAAO,WAAW,EAAE;4BACjDsB;wBACF;oBACF,EAAE,OAAOG,OAAY;wBACnB,MAAM,IAAIJ,MACR,CAAC,8BAA8B,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAEpD,KAAK,SAAS,CAACE,QAAQ,EACtG;4BAAE,OAAOkD;wBAAM;oBAEnB;gBACF;gBAEAX,mBAAmBF,QAAQ;gBAE3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAASH,aAAaZ;gBACjDG,mBAAmBF,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW2B;gBAE/Cb,mBAAmBF,QAAQ;gBAE3B,MAAMgB,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMX,MAAMW;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCjE,MACE,CAAC,6DAA6D,EAAEqC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEuB;wBACpD5D,MACE,CAAC,0DAA0D,EAAEqC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOkB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,uCAAuC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQe;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACoB;IACrB;IAEQ,iBACNjB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEqD,WAAW,EAAE,GAAGzC;QAE7D,IAAI0C,cAAc1D,qBAAqBuD;QAEvC,IAAIxC,AAAcV,WAAdU,WACF2C,cAAc;YACZ,GAAGA,WAAW;YACd3C;QACF;QAGF,IAAIX,cAAc,CAACsD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASvC,KAAK,OAAO;YACrB,UAAU,OAAOkB;gBACf,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMuB,8BAA8B5B,mCAClC0B,eAEEA,cACArD;gBAEJ8B,OACEuB,aAAa,UAAUE,6BACvB,CAAC,mEAAmE,EAAE7D,KAAK,SAAS,CAClF2D,cACC;gBAGL,IAAI,CAACnB,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAES,wBAAwB,EAAE,GAAGT;gBAErC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIc;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb5B,KAAK,GAAG,GAAG;wBACT4B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA5B,KAAK,KAAK,GAAG6B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD5B,KAAK,UAAU,GAAG4B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjB5B,KAAK,eAAe,GAAG6B,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjB5B,KAAK,iBAAiB,GAAG4B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,8BACvBO,qBAAqBP,+BACrBvD;gBAKJ,MAAM+D,kBAAkBV,YAAY,UAAU,GAC1CrD,SACA6D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDX,YAAY,KAAK,IACjB,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACZ,YAAY,KAAK;yBAAC;oBAC7B;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMa,mBAAmBD,gBACrBE,oCACE;oBACE,QAAQ;wBACNF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;wBAC3CA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;qBAC5C;oBACD,aACE,AAA8B,YAA9B,OAAOZ,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE,UAAU;oBACpC,MAAMY;gBACR,GACAtB,4BAEF3C;gBAEJ,MAAMoE,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAchB,YAAY,MAAM;gBACtC,MAAMiB,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJR,mBAAmBI,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAhB,YAAY,SAAS;gBAI7B,MAAMqB,mBAAmBF,yBACrBL,oCACEK,wBACA7B,4BAEF3C;gBAEJ,MAAM2E,aAAa,CAAC,CAACD;gBAErB,IAAIE;gBACJ,MAAM3C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACO,cAAc,CAACX,iBACjC,IAAI;oBACF7B,mBAAmBF,QAAQ;oBAC3BwB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCJ,aACA;wBACE,SAASnB;wBACT2B;oBACF,GACAtD,cACA6C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BmB,sBAAsBnB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiB+B,cACnBnB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRX,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJyE,mBACAG,oBACAQ,oBACAE;gBAEF,IAAItF,SAAS;oBACX,MAAMwF,uBAAuBzF,2BAA2BC;oBACxD,IAAIwF,sBAAsB;wBACxB,IAAItB,YACF,MAAM,IAAIqB,aAAaC,sBAAsBtB;wBAE/C,MAAM,IAAId,MAAMoC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B7F,iBAC/BoF,mBAAmB,cAAc;gBAGnC,IAAIU;gBAOJ,IACE1F,WACA,IAAI,CAAC,SAAS,IACd,CAACqF,cACA,EAACX,mBAAmB,CAACe,wBAAuB,KAC7C1B,AAA0B,UAA1BA,YAAY,SAAS,EAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,MAAM4B,gBAAkC;wBACtCC,KAAK,KAAK,CAAC5F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;wBAC/BuC,KAAK,KAAK,CAAC5F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;qBAChC;oBAED,MAAMwC,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAA8B,YAA9B,OAAO5B,YAAY,MAAM,GACrBA,YAAY,MAAM,GAClBA,YAAY,MAAM,EAAE;wBAC1B,cAAc9C;oBAChB;oBAEF,IAAIrB,iBAAiBiG,UAAU;wBAC7BnG,MACE,uCACAqF,aACAc;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQd;4BACR,OAAOc;wBACT,GACAb;oBAEJ,OACEtF,MACE,yDACAqF;gBAGN,EAAE,OAAOvB,OAAO;oBACd9D,MAAM,mCAAmC8D;gBAC3C;qBAEA9D,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIkE,YACF,MAAM,IAAIqB,aACR,CAAC,oBAAoB,EAAExB,YAAY,MAAM,EAAE,EAC3CG;oBAGJ,MAAM,IAAId,MAAM,CAAC,mBAAmB,EAAEW,YAAY,MAAM,EAAE;gBAC5D;gBAEA,IAAI+B;gBAEJ,IAAIpB,mBAAmBT,6BACrB6B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,oBACE7B,4BAA4B,kBAAkB;oBAClD;gBACF;qBACK,IAAIa,YACTgB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO/B,YAAY,KAAK;oBAC1B;gBACF;qBACK,IAAIsB,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPb;wBACA,aAAaS;oBACf;gBACF;gBAGF,MAAMK,gBAAgBhC,YAAY,aAAa;gBAC/C,MAAMiC,mBAAmBD,gBACrB;oBACE,GAAG/F,OAAO;oBACV,aAAa+F;gBACf,IACA/F;gBAEJ,IAAI+F,iBAAiB7B,YAAY,gBAC/BA,WAAW,cAAc,GAAGA,WAAW,cAAc,CAAC,GAAG,CACvD,CAAC+B,iBAAoB;wBACnB,GAAGA,cAAc;wBACjB,aAAaF;oBACf;gBAIJlC,WAAWmC;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,gBAAgB;4BAEnB,KAAKpD,UAAU,aAAa;wBAC9B;oBACF;oBACAkD;gBACF;YACF;QACF;QAEA,OAAO9B;IACT;IAtlBA,YAAY,EACVkC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTtE,WAAW,EACXuE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGtE;QACnB,IAAI,CAAC,eAAe,GAAGuE;IACzB;AA2kBF"}

package/dist/es/agent/utils.mjs CHANGED Viewed

@@ -1,10 +1,8 @@
 import { existsSync } from "node:fs";
 import { resolve } from "node:path";
-import { pixelBboxToRect } from "../ai-model/workflows/grounding/locate-result-rect.mjs";
 import { ScreenshotItem } from "../screenshot-item.mjs";
 import { uploadTestInfoToServer } from "../utils.mjs";
 import { MIDSCENE_REPORT_QUIET, MIDSCENE_REPORT_TAG_NAME, globalConfigManager } from "@midscene/shared/env";
-import { generateElementByRect } from "@midscene/shared/extractor";
 import { createImgBase64ByFormat, imageInfoOfBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert, ifInBrowser, logMsg, uuid } from "@midscene/shared/utils";
@@ -120,17 +118,24 @@ function normalizeFilePaths(files, options = {}) {
         return `\\\\wsl$\\${wslDistroName}${absolutePath.replace(/\//g, '\\')}`;
     });
 }
-function isPixelBbox(value) {
-    return Array.isArray(value) && 4 === value.length && value.every((item)=>'number' == typeof item && Number.isFinite(item));
-}
-function ifLocateParamHasLocatedPixelBbox(locateParam) {
-    return isPixelBbox(locateParam.locatedPixelBbox);
+function ifLocateParamHasLocatedPixelResult(planLocateParam) {
+    const result = planLocateParam.locatedPixelResult;
+    if (!result || 'object' != typeof result || !('center' in result)) return false;
+    const point = result.center;
+    return Array.isArray(point) && 2 === point.length && point.every((v)=>'number' == typeof v && Number.isFinite(v));
 }
 function matchElementFromPlan(planLocateParam) {
     if (!planLocateParam) return;
-    const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);
-    const element = generateElementByRect(rect, 'string' == typeof planLocateParam.prompt ? planLocateParam.prompt : planLocateParam.prompt?.prompt || '');
-    return element;
+    const { center, rect } = planLocateParam.locatedPixelResult;
+    return {
+        center: [
+            ...center
+        ],
+        description: 'string' == typeof planLocateParam.prompt ? planLocateParam.prompt : planLocateParam.prompt?.prompt || '',
+        ...rect ? {
+            rect
+        } : {}
+    };
 }
 async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable) {
     if (!cacheEntry) return;
@@ -141,8 +146,8 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         const rect = await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);
         const element = {
             center: [
-                Math.round(rect.left + rect.width / 2),
-                Math.round(rect.top + rect.height / 2)
+                rect.left + rect.width / 2,
+                rect.top + rect.height / 2
             ],
             rect,
             description: 'string' == typeof cachePrompt ? cachePrompt : cachePrompt.prompt || ''
@@ -154,7 +159,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.12.5-beta-20260908034641.0";
+const getMidsceneVersion = ()=>"1.12.5";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,
@@ -173,16 +178,12 @@ const transformLogicalElementToScreenshot = (element, shrunkShotToLogicalRatio)=
     return {
         ...element,
         center: [
-            Math.round(element.center[0] * shrunkShotToLogicalRatio),
-            Math.round(element.center[1] * shrunkShotToLogicalRatio)
+            element.center[0] * shrunkShotToLogicalRatio,
+            element.center[1] * shrunkShotToLogicalRatio
         ],
-        rect: {
-            ...element.rect,
-            left: Math.round(element.rect.left * shrunkShotToLogicalRatio),
-            top: Math.round(element.rect.top * shrunkShotToLogicalRatio),
-            width: Math.round(element.rect.width * shrunkShotToLogicalRatio),
-            height: Math.round(element.rect.height * shrunkShotToLogicalRatio)
-        }
+        ...element.rect ? {
+            rect: transformLogicalRectToScreenshotRect(element.rect, shrunkShotToLogicalRatio)
+        } : {}
     };
 };
 const transformLogicalRectToScreenshotRect = (rect, shrunkShotToLogicalRatio)=>{
@@ -195,6 +196,6 @@ const transformLogicalRectToScreenshotRect = (rect, shrunkShotToLogicalRatio)=>{
         height: Math.round(rect.height * shrunkShotToLogicalRatio)
     };
 };
-export { commonContextParser, createScreenshotBoundUIContext, getMidsceneVersion, getReportFileName, ifLocateParamHasLocatedPixelBbox, isPixelBbox, matchElementFromCache, matchElementFromPlan, normalizeFilePaths, normalizeScrollType, parsePrompt, printReportMsg, transformLogicalElementToScreenshot, transformLogicalRectToScreenshotRect };
+export { commonContextParser, createScreenshotBoundUIContext, getMidsceneVersion, getReportFileName, ifLocateParamHasLocatedPixelResult, matchElementFromCache, matchElementFromPlan, normalizeFilePaths, normalizeScrollType, parsePrompt, printReportMsg, transformLogicalElementToScreenshot, transformLogicalRectToScreenshotRect };
 //# sourceMappingURL=utils.mjs.map

package/dist/es/agent/utils.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport { pixelBboxToRect } from '@/ai-model/workflows/grounding/locate-result-rect';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n DetailedLocateParam,\n ElementCacheFeature,\n LocateResultElement,\n PixelBbox,\n Rect,\n ScrollParam,\n Size,\n UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n MIDSCENE_REPORT_QUIET,\n MIDSCENE_REPORT_TAG_NAME,\n globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport {\n createImgBase64ByFormat,\n imageInfoOfBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport { prepareRawScreenshot } from './screenshot-preparation';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst agentDebug = getDebug('agent');\nconst screenshotDataUrlPattern = /^data:image\\/[a-zA-Z0-9.+-]+;base64,/i;\n\nconst inferBase64ImageFormat = (base64Body: string) => {\n if (base64Body.startsWith('iVBORw0KGgo')) {\n return 'png';\n }\n return 'jpeg';\n};\n\nconst normalizeScreenshotBase64 = (screenshotBase64: string) => {\n const trimmedBase64 = screenshotBase64.trim();\n if (screenshotDataUrlPattern.test(trimmedBase64)) {\n return trimmedBase64;\n }\n\n const base64Body = trimmedBase64.replace(/\\s/g, '');\n assert(base64Body, 'screenshotBase64 must include image data');\n return createImgBase64ByFormat(\n inferBase64ImageFormat(base64Body),\n base64Body,\n );\n};\n\nconst legacyScrollTypeMap = {\n once: 'singleAction',\n untilBottom: 'scrollToBottom',\n untilTop: 'scrollToTop',\n untilRight: 'scrollToRight',\n untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n scrollType: string \| undefined,\n): ScrollParam['scrollType'] \| undefined => {\n if (!scrollType) {\n return undefined;\n }\n\n if (scrollType in legacyScrollTypeMap) {\n return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n }\n\n return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n interfaceInstance: AbstractInterface,\n _opt: {\n uploadServerUrl?: string;\n screenshotShrinkFactor?: number;\n },\n): Promise<UIContext> {\n const debug = getDebug('commonContextParser');\n\n assert(interfaceInstance, 'interfaceInstance is required');\n\n debug('Getting interface description');\n const description = interfaceInstance.describe?.() \|\| '';\n debug('Interface description end');\n\n debug('Uploading test info to server');\n uploadTestInfoToServer({\n testUrl: description,\n serverUrl: _opt.uploadServerUrl,\n });\n debug('UploadTestInfoToServer end');\n\n debug('will get size');\n const interfaceSize = await interfaceInstance.size();\n const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n if ((interfaceSize as unknown as { dpr: number }).dpr) {\n console.warn(\n 'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n );\n }\n\n if (!Number.isFinite(logicalWidth) \|\| !Number.isFinite(logicalHeight)) {\n throw new Error(\n `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n );\n }\n\n if (logicalWidth <= 0 \|\| logicalHeight <= 0) {\n throw new Error(\n `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n );\n }\n\n debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n const screenshotBase64 = await interfaceInstance.screenshotBase64();\n const screenshotCapturedAt = Date.now();\n assert(screenshotBase64!, 'screenshotBase64 is required');\n const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n debug('will get screenshot dimensions');\n const preparedScreenshot = await prepareRawScreenshot(screenshotBase64, {\n shrinkFactor: userShrinkFactor,\n });\n const { width: imgWidth, height: imgHeight } =\n preparedScreenshot.originalSize;\n debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n // Detect orientation mismatch between logical size and screenshot.\n // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n // size() to return portrait dimensions even when the device is landscape.\n // We detect this by comparing aspect ratios and swap if they disagree.\n const logicalIsPortrait = logicalWidth < logicalHeight;\n const screenshotIsPortrait = imgWidth < imgHeight;\n let finalLogicalWidth = logicalWidth;\n let finalLogicalHeight = logicalHeight;\n if (logicalIsPortrait !== screenshotIsPortrait) {\n debug(\n `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n );\n finalLogicalWidth = logicalHeight;\n finalLogicalHeight = logicalWidth;\n }\n\n const dpr = imgWidth / finalLogicalWidth;\n\n debug('calculated dpr:', dpr);\n\n const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n if (userShrinkFactor > 1) {\n debug(\n `Applied screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${preparedScreenshot.shotSize.width}x${preparedScreenshot.shotSize.height})`,\n );\n }\n\n return {\n shotSize: preparedScreenshot.shotSize,\n deprecatedDpr: dpr,\n screenshot: ScreenshotItem.create(\n preparedScreenshot.base64,\n screenshotCapturedAt,\n ),\n shrunkShotToLogicalRatio,\n };\n}\n\nexport async function createScreenshotBoundUIContext(\n screenshotBase64: string,\n opt: {\n screenshotSize?: Size;\n },\n): Promise<UIContext> {\n const normalizedScreenshotBase64 =\n normalizeScreenshotBase64(screenshotBase64);\n const actualScreenshotSize = await imageInfoOfBase64(\n normalizedScreenshotBase64,\n );\n if (\n opt.screenshotSize &&\n (opt.screenshotSize.width !== actualScreenshotSize.width \|\|\n opt.screenshotSize.height !== actualScreenshotSize.height)\n ) {\n agentDebug(\n 'describeElementAtPoint screenshotSize mismatch, use actual size',\n {\n provided: opt.screenshotSize,\n actual: actualScreenshotSize,\n },\n );\n }\n\n return {\n screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n shotSize: actualScreenshotSize,\n shrunkShotToLogicalRatio: 1,\n _isFrozen: true,\n };\n}\n\nexport function getReportFileName(tag = 'web') {\n const reportTagName = globalConfigManager.getEnvConfigValue(\n MIDSCENE_REPORT_TAG_NAME,\n );\n const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n // ensure uniqueness at the same time\n const uniqueId = uuid().substring(0, 8);\n return `${reportTagName \|\| tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n return;\n }\n logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n fileExists?: (path: string) => boolean;\n isInBrowser?: boolean;\n resolvePath?: (path: string) => string;\n wslDistroName?: string;\n cwd?: string;\n};\n\nexport function normalizeFilePaths(\n files: string[],\n options: NormalizeFilePathsOptions = {},\n): string[] {\n const {\n fileExists = existsSync,\n isInBrowser = ifInBrowser,\n resolvePath = resolve,\n wslDistroName = process.env.WSL_DISTRO_NAME,\n cwd = process.cwd(),\n } = options;\n\n if (isInBrowser) {\n throw new Error('File chooser is not supported in browser environment');\n }\n\n return files.map((file) => {\n const absolutePath = resolvePath(file);\n if (!fileExists(absolutePath)) {\n throw new Error(\n `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n );\n }\n\n if (!wslDistroName) {\n return absolutePath;\n }\n\n const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n if (wslMount) {\n return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n }\n\n return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n });\n}\n\nexport function isPixelBbox(value: unknown): value is PixelBbox {\n return (\n Array.isArray(value) &&\n value.length === 4 &&\n value.every((item) => typeof item === 'number' && Number.isFinite(item))\n );\n}\n\ntype LocateParamWithMaybeLocatedPixelBbox = DetailedLocateParam & {\n locatedPixelBbox?: unknown;\n};\n\ntype LocateParamWithLocatedPixelBbox = DetailedLocateParam & {\n /** Pixel bbox of the located element in screenshot coordinates. /\n locatedPixelBbox: PixelBbox;\n};\n\nexport function ifLocateParamHasLocatedPixelBbox(\n locateParam: LocateParamWithMaybeLocatedPixelBbox,\n): locateParam is LocateParamWithLocatedPixelBbox {\n return isPixelBbox(locateParam.locatedPixelBbox);\n}\n\nexport function matchElementFromPlan(\n planLocateParam: LocateParamWithLocatedPixelBbox,\n): LocateResultElement \| undefined {\n if (!planLocateParam) {\n return undefined;\n }\n\n const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);\n\n const element = generateElementByRect(\n rect,\n typeof planLocateParam.prompt === 'string'\n ? planLocateParam.prompt\n : planLocateParam.prompt?.prompt \|\| '',\n );\n return element;\n}\n\nexport async function matchElementFromCache(\n context: {\n taskCache?: TaskCache;\n interfaceInstance: AbstractInterface;\n },\n cacheEntry: ElementCacheFeature \| undefined,\n cachePrompt: TUserPrompt,\n cacheable: boolean \| undefined,\n): Promise<LocateResultElement \| undefined> {\n if (!cacheEntry) {\n return undefined;\n }\n\n if (cacheable === false) {\n cacheDebug('cache disabled for prompt: %s', cachePrompt);\n return undefined;\n }\n\n if (!context.taskCache?.isCacheResultUsed) {\n return undefined;\n }\n\n if (!context.interfaceInstance.rectMatchesCacheFeature) {\n cacheDebug(\n 'interface does not implement rectMatchesCacheFeature, skip cache',\n );\n return undefined;\n }\n\n try {\n const rect =\n await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n const element: LocateResultElement = {\n center: [\n Math.round(rect.left + rect.width / 2),\n Math.round(rect.top + rect.height / 2),\n ],\n rect,\n description:\n typeof cachePrompt === 'string'\n ? cachePrompt\n : cachePrompt.prompt \|\| '',\n };\n\n cacheDebug('cache hit, prompt: %s', cachePrompt);\n return element;\n } catch (error) {\n cacheDebug('rectMatchesCacheFeature error: %s', error);\n return undefined;\n }\n}\n\ndeclare const __VERSION__: string \| undefined;\n\nexport const getMidsceneVersion = (): string => {\n if (typeof __VERSION__ !== 'undefined') {\n return __VERSION__;\n } else if (\n process.env.__VERSION__ &&\n process.env.__VERSION__ !== 'undefined'\n ) {\n return process.env.__VERSION__;\n }\n throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n prompt: TUserPrompt,\n): {\n textPrompt: string;\n multimodalPrompt?: TMultimodalPrompt;\n} => {\n if (typeof prompt === 'string') {\n return {\n textPrompt: prompt,\n multimodalPrompt: undefined,\n };\n }\n return {\n textPrompt: prompt.prompt,\n multimodalPrompt: prompt.images\n ? {\n images: prompt.images,\n convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n }\n : undefined,\n };\n};\n\nexport const transformLogicalElementToScreenshot = (\n element: LocateResultElement,\n shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n if (shrunkShotToLogicalRatio === 1) {\n return element;\n }\n\n return {\n ...element,\n center: [\n Math.round(element.center[0] shrunkShotToLogicalRatio),\n Math.round(element.center[1] * shrunkShotToLogicalRatio),\n ],\n rect: {\n ...element.rect,\n left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n },\n };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n rect: Rect,\n shrunkShotToLogicalRatio: number,\n): Rect => {\n if (shrunkShotToLogicalRatio === 1) {\n return rect;\n }\n\n return {\n ...rect,\n left: Math.round(rect.left * shrunkShotToLogicalRatio),\n top: Math.round(rect.top * shrunkShotToLogicalRatio),\n width: Math.round(rect.width * shrunkShotToLogicalRatio),\n height: Math.round(rect.height * shrunkShotToLogicalRatio),\n };\n};\n"],"names":["agentDebug","getDebug","screenshotDataUrlPattern","inferBase64ImageFormat","base64Body","normalizeScreenshotBase64","screenshotBase64","trimmedBase64","assert","createImgBase64ByFormat","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotCapturedAt","Date","userShrinkFactor","preparedScreenshot","prepareRawScreenshot","imgWidth","imgHeight","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","dpr","shrunkShotToLogicalRatio","ScreenshotItem","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","actualScreenshotSize","imageInfoOfBase64","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","isPixelBbox","value","Array","item","ifLocateParamHasLocatedPixelBbox","locateParam","matchElementFromPlan","planLocateParam","rect","pixelBboxToRect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","Math","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;;;;;AAmCA,MAAMA,aAAaC,SAAS;AAC5B,MAAMC,2BAA2B;AAEjC,MAAMC,yBAAyB,CAACC;IAC9B,IAAIA,WAAW,UAAU,CAAC,gBACxB,OAAO;IAET,OAAO;AACT;AAEA,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,gBAAgBD,iBAAiB,IAAI;IAC3C,IAAIJ,yBAAyB,IAAI,CAACK,gBAChC,OAAOA;IAGT,MAAMH,aAAaG,cAAc,OAAO,CAAC,OAAO;IAChDC,OAAOJ,YAAY;IACnB,OAAOK,wBACLN,uBAAuBC,aACvBA;AAEJ;AAEA,MAAMM,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQf,SAAS;IAEvBO,OAAOM,mBAAmB;IAE1BE,MAAM;IACN,MAAMC,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNE,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMG,gBAAgB,MAAML,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOM,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIL,MAAM,CAAC,MAAM,EAAEI,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMf,mBAAmB,MAAMQ,kBAAkB,gBAAgB;IACjE,MAAMW,uBAAuBC,KAAK,GAAG;IACrClB,OAAOF,kBAAmB;IAC1B,MAAMqB,mBAAmBZ,KAAK,sBAAsB,IAAI;IAExDC,MAAM;IACN,MAAMY,qBAAqB,MAAMC,qBAAqBvB,kBAAkB;QACtE,cAAcqB;IAChB;IACA,MAAM,EAAE,OAAOG,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1CH,mBAAmB,YAAY;IACjCZ,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAMC,oBAAoBZ,eAAeC;IACzC,MAAMY,uBAAuBH,WAAWC;IACxC,IAAIG,oBAAoBd;IAExB,IAAIY,sBAAsBC,sBAAsB;QAC9CjB,MACE,CAAC,4CAA4C,EAAEI,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEW,oBAAoB,aAAa,YAAY,gBAAgB,EAAEF,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEE,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBb;IAEtB;IAEA,MAAMc,MAAML,WAAWI;IAEvBlB,MAAM,mBAAmBmB;IAEzB,MAAMC,2BAA2BD,MAAMR;IAEvCX,MAAM,4BAA4BoB;IAElC,IAAIT,mBAAmB,GACrBX,MACE,CAAC,kCAAkC,EAAEW,iBAAiB,YAAY,EAAEG,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEH,mBAAmB,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAEA,mBAAmB,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAItL,OAAO;QACL,UAAUA,mBAAmB,QAAQ;QACrC,eAAeO;QACf,YAAYE,eAAe,MAAM,CAC/BT,mBAAmB,MAAM,EACzBH;QAEFW;IACF;AACF;AAEO,eAAeE,+BACpBhC,gBAAwB,EACxBiC,GAEC;IAED,MAAMC,6BACJnC,0BAA0BC;IAC5B,MAAMmC,uBAAuB,MAAMC,kBACjCF;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKE,qBAAqB,KAAK,IACtDF,IAAI,cAAc,CAAC,MAAM,KAAKE,qBAAqB,MAAK,GAE1DzC,WACE,mEACA;QACE,UAAUuC,IAAI,cAAc;QAC5B,QAAQE;IACV;IAIJ,OAAO;QACL,YAAYJ,eAAe,MAAM,CAACG,4BAA4Bd,KAAK,GAAG;QACtE,UAAUe;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASE,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAUO,SAASG,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAIrC,MAAM;IAGlB,OAAOiC,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAI7C,MACR,CAAC,gBAAgB,EAAE4C,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAEO,SAASE,YAAYC,KAAc;IACxC,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,OAAS,AAAgB,YAAhB,OAAOA,QAAqBnD,OAAO,QAAQ,CAACmD;AAEtE;AAWO,SAASC,iCACdC,WAAiD;IAEjD,OAAOL,YAAYK,YAAY,gBAAgB;AACjD;AAEO,SAASC,qBACdC,eAAgD;IAEhD,IAAI,CAACA,iBACH;IAGF,MAAMC,OAAOC,gBAAgBF,gBAAgB,gBAAgB;IAE7D,MAAMG,UAAUC,sBACdH,MACA,AAAkC,YAAlC,OAAOD,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;IAExC,OAAOG;AACT;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMT,OACJ,MAAMK,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNQ,KAAK,KAAK,CAACV,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCU,KAAK,KAAK,CAACV,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOO,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOS,OAAO;QACdF,8BAAW,qCAAqCE;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDf,SACA7C;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO6C;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNQ,KAAK,KAAK,CAACR,QAAQ,MAAM,CAAC,EAAE,GAAG7C;YAC/BqD,KAAK,KAAK,CAACR,QAAQ,MAAM,CAAC,EAAE,GAAG7C;SAChC;QACD,MAAM;YACJ,GAAG6C,QAAQ,IAAI;YACf,MAAMQ,KAAK,KAAK,CAACR,QAAQ,IAAI,CAAC,IAAI,GAAG7C;YACrC,KAAKqD,KAAK,KAAK,CAACR,QAAQ,IAAI,CAAC,GAAG,GAAG7C;YACnC,OAAOqD,KAAK,KAAK,CAACR,QAAQ,IAAI,CAAC,KAAK,GAAG7C;YACvC,QAAQqD,KAAK,KAAK,CAACR,QAAQ,IAAI,CAAC,MAAM,GAAG7C;QAC3C;IACF;AACF;AAEO,MAAM6D,uCAAuC,CAClDlB,MACA3C;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO2C;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMU,KAAK,KAAK,CAACV,KAAK,IAAI,GAAG3C;QAC7B,KAAKqD,KAAK,KAAK,CAACV,KAAK,GAAG,GAAG3C;QAC3B,OAAOqD,KAAK,KAAK,CAACV,KAAK,KAAK,GAAG3C;QAC/B,QAAQqD,KAAK,KAAK,CAACV,KAAK,MAAM,GAAG3C;IACnC;AACF"}
1	+ {"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport type { PixelLocateResult } from '@/ai-model/shared/model-locate-result';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n DetailedLocateParam,\n ElementCacheFeature,\n LocateResultElement,\n Rect,\n ScrollParam,\n Size,\n UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n MIDSCENE_REPORT_QUIET,\n MIDSCENE_REPORT_TAG_NAME,\n globalConfigManager,\n} from '@midscene/shared/env';\nimport {\n createImgBase64ByFormat,\n imageInfoOfBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport { prepareRawScreenshot } from './screenshot-preparation';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst agentDebug = getDebug('agent');\nconst screenshotDataUrlPattern = /^data:image\\/[a-zA-Z0-9.+-]+;base64,/i;\n\nconst inferBase64ImageFormat = (base64Body: string) => {\n if (base64Body.startsWith('iVBORw0KGgo')) {\n return 'png';\n }\n return 'jpeg';\n};\n\nconst normalizeScreenshotBase64 = (screenshotBase64: string) => {\n const trimmedBase64 = screenshotBase64.trim();\n if (screenshotDataUrlPattern.test(trimmedBase64)) {\n return trimmedBase64;\n }\n\n const base64Body = trimmedBase64.replace(/\\s/g, '');\n assert(base64Body, 'screenshotBase64 must include image data');\n return createImgBase64ByFormat(\n inferBase64ImageFormat(base64Body),\n base64Body,\n );\n};\n\nconst legacyScrollTypeMap = {\n once: 'singleAction',\n untilBottom: 'scrollToBottom',\n untilTop: 'scrollToTop',\n untilRight: 'scrollToRight',\n untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n scrollType: string \| undefined,\n): ScrollParam['scrollType'] \| undefined => {\n if (!scrollType) {\n return undefined;\n }\n\n if (scrollType in legacyScrollTypeMap) {\n return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n }\n\n return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n interfaceInstance: AbstractInterface,\n _opt: {\n uploadServerUrl?: string;\n screenshotShrinkFactor?: number;\n },\n): Promise<UIContext> {\n const debug = getDebug('commonContextParser');\n\n assert(interfaceInstance, 'interfaceInstance is required');\n\n debug('Getting interface description');\n const description = interfaceInstance.describe?.() \|\| '';\n debug('Interface description end');\n\n debug('Uploading test info to server');\n uploadTestInfoToServer({\n testUrl: description,\n serverUrl: _opt.uploadServerUrl,\n });\n debug('UploadTestInfoToServer end');\n\n debug('will get size');\n const interfaceSize = await interfaceInstance.size();\n const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n if ((interfaceSize as unknown as { dpr: number }).dpr) {\n console.warn(\n 'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n );\n }\n\n if (!Number.isFinite(logicalWidth) \|\| !Number.isFinite(logicalHeight)) {\n throw new Error(\n `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n );\n }\n\n if (logicalWidth <= 0 \|\| logicalHeight <= 0) {\n throw new Error(\n `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n );\n }\n\n debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n const screenshotBase64 = await interfaceInstance.screenshotBase64();\n const screenshotCapturedAt = Date.now();\n assert(screenshotBase64!, 'screenshotBase64 is required');\n const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n debug('will get screenshot dimensions');\n const preparedScreenshot = await prepareRawScreenshot(screenshotBase64, {\n shrinkFactor: userShrinkFactor,\n });\n const { width: imgWidth, height: imgHeight } =\n preparedScreenshot.originalSize;\n debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n // Detect orientation mismatch between logical size and screenshot.\n // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n // size() to return portrait dimensions even when the device is landscape.\n // We detect this by comparing aspect ratios and swap if they disagree.\n const logicalIsPortrait = logicalWidth < logicalHeight;\n const screenshotIsPortrait = imgWidth < imgHeight;\n let finalLogicalWidth = logicalWidth;\n let finalLogicalHeight = logicalHeight;\n if (logicalIsPortrait !== screenshotIsPortrait) {\n debug(\n `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n );\n finalLogicalWidth = logicalHeight;\n finalLogicalHeight = logicalWidth;\n }\n\n const dpr = imgWidth / finalLogicalWidth;\n\n debug('calculated dpr:', dpr);\n\n const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n if (userShrinkFactor > 1) {\n debug(\n `Applied screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${preparedScreenshot.shotSize.width}x${preparedScreenshot.shotSize.height})`,\n );\n }\n\n return {\n shotSize: preparedScreenshot.shotSize,\n deprecatedDpr: dpr,\n screenshot: ScreenshotItem.create(\n preparedScreenshot.base64,\n screenshotCapturedAt,\n ),\n shrunkShotToLogicalRatio,\n };\n}\n\nexport async function createScreenshotBoundUIContext(\n screenshotBase64: string,\n opt: {\n screenshotSize?: Size;\n },\n): Promise<UIContext> {\n const normalizedScreenshotBase64 =\n normalizeScreenshotBase64(screenshotBase64);\n const actualScreenshotSize = await imageInfoOfBase64(\n normalizedScreenshotBase64,\n );\n if (\n opt.screenshotSize &&\n (opt.screenshotSize.width !== actualScreenshotSize.width \|\|\n opt.screenshotSize.height !== actualScreenshotSize.height)\n ) {\n agentDebug(\n 'describeElementAtPoint screenshotSize mismatch, use actual size',\n {\n provided: opt.screenshotSize,\n actual: actualScreenshotSize,\n },\n );\n }\n\n return {\n screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n shotSize: actualScreenshotSize,\n shrunkShotToLogicalRatio: 1,\n _isFrozen: true,\n };\n}\n\nexport function getReportFileName(tag = 'web') {\n const reportTagName = globalConfigManager.getEnvConfigValue(\n MIDSCENE_REPORT_TAG_NAME,\n );\n const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n // ensure uniqueness at the same time\n const uniqueId = uuid().substring(0, 8);\n return `${reportTagName \|\| tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n return;\n }\n logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n fileExists?: (path: string) => boolean;\n isInBrowser?: boolean;\n resolvePath?: (path: string) => string;\n wslDistroName?: string;\n cwd?: string;\n};\n\nexport function normalizeFilePaths(\n files: string[],\n options: NormalizeFilePathsOptions = {},\n): string[] {\n const {\n fileExists = existsSync,\n isInBrowser = ifInBrowser,\n resolvePath = resolve,\n wslDistroName = process.env.WSL_DISTRO_NAME,\n cwd = process.cwd(),\n } = options;\n\n if (isInBrowser) {\n throw new Error('File chooser is not supported in browser environment');\n }\n\n return files.map((file) => {\n const absolutePath = resolvePath(file);\n if (!fileExists(absolutePath)) {\n throw new Error(\n `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n );\n }\n\n if (!wslDistroName) {\n return absolutePath;\n }\n\n const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n if (wslMount) {\n return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n }\n\n return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n });\n}\n\ntype LocateParamWithMaybeLocatedPixelResult = DetailedLocateParam & {\n locatedPixelResult?: unknown;\n};\n\ntype LocateParamWithLocatedPixelResult = DetailedLocateParam & {\n locatedPixelResult: PixelLocateResult;\n};\n\nexport function ifLocateParamHasLocatedPixelResult(\n planLocateParam: LocateParamWithMaybeLocatedPixelResult,\n): planLocateParam is LocateParamWithLocatedPixelResult {\n const result = planLocateParam.locatedPixelResult;\n if (!result \|\| typeof result !== 'object' \|\| !('center' in result)) {\n return false;\n }\n const point = result.center;\n return (\n Array.isArray(point) &&\n point.length === 2 &&\n point.every((v) => typeof v === 'number' && Number.isFinite(v))\n );\n}\n\nexport function matchElementFromPlan(\n planLocateParam: LocateParamWithLocatedPixelResult,\n): LocateResultElement \| undefined {\n if (!planLocateParam) {\n return undefined;\n }\n\n const { center, rect } = planLocateParam.locatedPixelResult;\n return {\n center: [...center],\n description:\n typeof planLocateParam.prompt === 'string'\n ? planLocateParam.prompt\n : planLocateParam.prompt?.prompt \|\| '',\n ...(rect ? { rect } : {}),\n };\n}\n\nexport async function matchElementFromCache(\n context: {\n taskCache?: TaskCache;\n interfaceInstance: AbstractInterface;\n },\n cacheEntry: ElementCacheFeature \| undefined,\n cachePrompt: TUserPrompt,\n cacheable: boolean \| undefined,\n): Promise<LocateResultElement \| undefined> {\n if (!cacheEntry) {\n return undefined;\n }\n\n if (cacheable === false) {\n cacheDebug('cache disabled for prompt: %s', cachePrompt);\n return undefined;\n }\n\n if (!context.taskCache?.isCacheResultUsed) {\n return undefined;\n }\n\n if (!context.interfaceInstance.rectMatchesCacheFeature) {\n cacheDebug(\n 'interface does not implement rectMatchesCacheFeature, skip cache',\n );\n return undefined;\n }\n\n try {\n const rect =\n await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n const element: LocateResultElement = {\n center: [rect.left + rect.width / 2, rect.top + rect.height / 2],\n rect,\n description:\n typeof cachePrompt === 'string'\n ? cachePrompt\n : cachePrompt.prompt \|\| '',\n };\n\n cacheDebug('cache hit, prompt: %s', cachePrompt);\n return element;\n } catch (error) {\n cacheDebug('rectMatchesCacheFeature error: %s', error);\n return undefined;\n }\n}\n\ndeclare const __VERSION__: string \| undefined;\n\nexport const getMidsceneVersion = (): string => {\n if (typeof __VERSION__ !== 'undefined') {\n return __VERSION__;\n } else if (\n process.env.__VERSION__ &&\n process.env.__VERSION__ !== 'undefined'\n ) {\n return process.env.__VERSION__;\n }\n throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n prompt: TUserPrompt,\n): {\n textPrompt: string;\n multimodalPrompt?: TMultimodalPrompt;\n} => {\n if (typeof prompt === 'string') {\n return {\n textPrompt: prompt,\n multimodalPrompt: undefined,\n };\n }\n return {\n textPrompt: prompt.prompt,\n multimodalPrompt: prompt.images\n ? {\n images: prompt.images,\n convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n }\n : undefined,\n };\n};\n\nexport const transformLogicalElementToScreenshot = (\n element: LocateResultElement,\n shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n if (shrunkShotToLogicalRatio === 1) {\n return element;\n }\n\n return {\n ...element,\n center: [\n element.center[0] * shrunkShotToLogicalRatio,\n element.center[1] * shrunkShotToLogicalRatio,\n ],\n ...(element.rect\n ? {\n rect: transformLogicalRectToScreenshotRect(\n element.rect,\n shrunkShotToLogicalRatio,\n ),\n }\n : {}),\n };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n rect: Rect,\n shrunkShotToLogicalRatio: number,\n): Rect => {\n if (shrunkShotToLogicalRatio === 1) {\n return rect;\n }\n\n return {\n ...rect,\n left: Math.round(rect.left * shrunkShotToLogicalRatio),\n top: Math.round(rect.top * shrunkShotToLogicalRatio),\n width: Math.round(rect.width * shrunkShotToLogicalRatio),\n height: Math.round(rect.height * shrunkShotToLogicalRatio),\n };\n};\n"],"names":["agentDebug","getDebug","screenshotDataUrlPattern","inferBase64ImageFormat","base64Body","normalizeScreenshotBase64","screenshotBase64","trimmedBase64","assert","createImgBase64ByFormat","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotCapturedAt","Date","userShrinkFactor","preparedScreenshot","prepareRawScreenshot","imgWidth","imgHeight","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","dpr","shrunkShotToLogicalRatio","ScreenshotItem","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","actualScreenshotSize","imageInfoOfBase64","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","ifLocateParamHasLocatedPixelResult","planLocateParam","result","point","Array","v","matchElementFromPlan","center","rect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","element","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect","Math"],"mappings":";;;;;;;;;;;AAiCA,MAAMA,aAAaC,SAAS;AAC5B,MAAMC,2BAA2B;AAEjC,MAAMC,yBAAyB,CAACC;IAC9B,IAAIA,WAAW,UAAU,CAAC,gBACxB,OAAO;IAET,OAAO;AACT;AAEA,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,gBAAgBD,iBAAiB,IAAI;IAC3C,IAAIJ,yBAAyB,IAAI,CAACK,gBAChC,OAAOA;IAGT,MAAMH,aAAaG,cAAc,OAAO,CAAC,OAAO;IAChDC,OAAOJ,YAAY;IACnB,OAAOK,wBACLN,uBAAuBC,aACvBA;AAEJ;AAEA,MAAMM,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQf,SAAS;IAEvBO,OAAOM,mBAAmB;IAE1BE,MAAM;IACN,MAAMC,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNE,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMG,gBAAgB,MAAML,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOM,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIL,MAAM,CAAC,MAAM,EAAEI,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMf,mBAAmB,MAAMQ,kBAAkB,gBAAgB;IACjE,MAAMW,uBAAuBC,KAAK,GAAG;IACrClB,OAAOF,kBAAmB;IAC1B,MAAMqB,mBAAmBZ,KAAK,sBAAsB,IAAI;IAExDC,MAAM;IACN,MAAMY,qBAAqB,MAAMC,qBAAqBvB,kBAAkB;QACtE,cAAcqB;IAChB;IACA,MAAM,EAAE,OAAOG,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1CH,mBAAmB,YAAY;IACjCZ,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAMC,oBAAoBZ,eAAeC;IACzC,MAAMY,uBAAuBH,WAAWC;IACxC,IAAIG,oBAAoBd;IAExB,IAAIY,sBAAsBC,sBAAsB;QAC9CjB,MACE,CAAC,4CAA4C,EAAEI,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEW,oBAAoB,aAAa,YAAY,gBAAgB,EAAEF,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEE,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBb;IAEtB;IAEA,MAAMc,MAAML,WAAWI;IAEvBlB,MAAM,mBAAmBmB;IAEzB,MAAMC,2BAA2BD,MAAMR;IAEvCX,MAAM,4BAA4BoB;IAElC,IAAIT,mBAAmB,GACrBX,MACE,CAAC,kCAAkC,EAAEW,iBAAiB,YAAY,EAAEG,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEH,mBAAmB,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAEA,mBAAmB,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC;IAItL,OAAO;QACL,UAAUA,mBAAmB,QAAQ;QACrC,eAAeO;QACf,YAAYE,eAAe,MAAM,CAC/BT,mBAAmB,MAAM,EACzBH;QAEFW;IACF;AACF;AAEO,eAAeE,+BACpBhC,gBAAwB,EACxBiC,GAEC;IAED,MAAMC,6BACJnC,0BAA0BC;IAC5B,MAAMmC,uBAAuB,MAAMC,kBACjCF;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKE,qBAAqB,KAAK,IACtDF,IAAI,cAAc,CAAC,MAAM,KAAKE,qBAAqB,MAAK,GAE1DzC,WACE,mEACA;QACE,UAAUuC,IAAI,cAAc;QAC5B,QAAQE;IACV;IAIJ,OAAO;QACL,YAAYJ,eAAe,MAAM,CAACG,4BAA4Bd,KAAK,GAAG;QACtE,UAAUe;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASE,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAUO,SAASG,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAIrC,MAAM;IAGlB,OAAOiC,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAI7C,MACR,CAAC,gBAAgB,EAAE4C,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAUO,SAASE,mCACdC,eAAuD;IAEvD,MAAMC,SAASD,gBAAgB,kBAAkB;IACjD,IAAI,CAACC,UAAU,AAAkB,YAAlB,OAAOA,UAAuB,CAAE,aAAYA,MAAK,GAC9D,OAAO;IAET,MAAMC,QAAQD,OAAO,MAAM;IAC3B,OACEE,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,IAAM,AAAa,YAAb,OAAOA,KAAkBrD,OAAO,QAAQ,CAACqD;AAEhE;AAEO,SAASC,qBACdL,eAAkD;IAElD,IAAI,CAACA,iBACH;IAGF,MAAM,EAAEM,MAAM,EAAEC,IAAI,EAAE,GAAGP,gBAAgB,kBAAkB;IAC3D,OAAO;QACL,QAAQ;eAAIM;SAAO;QACnB,aACE,AAAkC,YAAlC,OAAON,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QACxC,GAAIO,OAAO;YAAEA;QAAK,IAAI,CAAC,CAAC;IAC1B;AACF;AAEO,eAAeC,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMN,OACJ,MAAME,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMI,UAA+B;YACnC,QAAQ;gBAACP,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBAAGA,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aAAE;YAChEA;YACA,aACE,AAAuB,YAAvB,OAAOI,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOG;IACT,EAAE,OAAOC,OAAO;QACdF,8BAAW,qCAAqCE;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDP,SACAlD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOkD;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNA,QAAQ,MAAM,CAAC,EAAE,GAAGlD;YACpBkD,QAAQ,MAAM,CAAC,EAAE,GAAGlD;SACrB;QACD,GAAIkD,QAAQ,IAAI,GACZ;YACE,MAAMQ,qCACJR,QAAQ,IAAI,EACZlD;QAEJ,IACA,CAAC,CAAC;IACR;AACF;AAEO,MAAM0D,uCAAuC,CAClDf,MACA3C;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO2C;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMgB,KAAK,KAAK,CAAChB,KAAK,IAAI,GAAG3C;QAC7B,KAAK2D,KAAK,KAAK,CAAChB,KAAK,GAAG,GAAG3C;QAC3B,OAAO2D,KAAK,KAAK,CAAChB,KAAK,KAAK,GAAG3C;QAC/B,QAAQ2D,KAAK,KAAK,CAAChB,KAAK,MAAM,GAAG3C;IACnC;AACF"}

package/dist/es/ai-model/connectivity/run-connectivity-test.mjs CHANGED Viewed

@@ -7,11 +7,6 @@ const TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';
 function isFiniteNumber(value) {
     return 'number' == typeof value && Number.isFinite(value);
 }
-function hasValidRect(value) {
-    if (!value || 'object' != typeof value) return false;
-    const rect = value;
-    return isFiniteNumber(rect.left) && isFiniteNumber(rect.top) && isFiniteNumber(rect.width) && isFiniteNumber(rect.height);
-}
 function hasValidCenter(value) {
     return Array.isArray(value) && 2 === value.length && isFiniteNumber(value[0]) && isFiniteNumber(value[1]);
 }
@@ -121,14 +116,12 @@ async function runAiLocateConnectivityCheck(modelRuntime) {
         const locateResult = await service.locate({
             prompt: 'the main todo input box'
         }, {}, modelRuntime);
-        const targetRect = locateResult.rect || locateResult.element?.rect;
         const center = locateResult.element?.center;
-        const passed = hasValidRect(targetRect) && hasValidCenter(center);
+        const passed = hasValidCenter(center);
         return buildCheckResult('aiLocate', modelRuntime, {
             passed,
             durationMs: Date.now() - startTime,
             message: passed ? '' : `Invalid locate result: ${JSON.stringify({
-                rect: targetRect,
                 center
             })}`
         });

package/dist/es/ai-model/connectivity/run-connectivity-test.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/connectivity/run-connectivity-test.mjs","sources":["../../../../src/ai-model/connectivity/run-connectivity-test.ts"],"sourcesContent":["import { ScreenshotItem } from '@/screenshot-item';\nimport Service from '@/service';\nimport type { UIContext } from '@/types';\nimport type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getModelRuntime } from '../models';\nimport type { ModelRuntime } from '../models';\nimport { callAI } from '../service-caller';\nimport {\n CONNECTIVITY_FIXTURE_IMAGE,\n CONNECTIVITY_FIXTURE_SHOT_SIZE,\n} from './fixture';\n\nconst TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';\n\ninterface ConnectivityCheckResultItem {\n name: 'text' \| 'vision' \| 'aiLocate';\n intent: TIntent;\n modelName: string;\n modelFamily?: string;\n passed: boolean;\n durationMs: number;\n message: string;\n}\n\nexport interface ConnectivityTestResult {\n passed: boolean;\n message?: string;\n}\n\nexport interface ConnectivityTestConfig {\n defaultModelConfig: IModelConfig;\n planningModelConfig: IModelConfig;\n insightModelConfig: IModelConfig;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction hasValidRect(value: unknown): boolean {\n if (!value \|\| typeof value !== 'object') {\n return false;\n }\n\n const rect = value as {\n left?: unknown;\n top?: unknown;\n width?: unknown;\n height?: unknown;\n };\n\n return (\n isFiniteNumber(rect.left) &&\n isFiniteNumber(rect.top) &&\n isFiniteNumber(rect.width) &&\n isFiniteNumber(rect.height)\n );\n}\n\nfunction hasValidCenter(value: unknown): boolean {\n return (\n Array.isArray(value) &&\n value.length === 2 &&\n isFiniteNumber(value[0]) &&\n isFiniteNumber(value[1])\n );\n}\n\nfunction buildFixtureContext(): UIContext {\n return {\n screenshot: ScreenshotItem.create(CONNECTIVITY_FIXTURE_IMAGE, Date.now()),\n shotSize: CONNECTIVITY_FIXTURE_SHOT_SIZE,\n shrunkShotToLogicalRatio: 1,\n };\n}\n\nfunction buildCheckResult(\n name: ConnectivityCheckResultItem['name'],\n modelRuntime: ModelRuntime,\n result: Omit<\n ConnectivityCheckResultItem,\n 'name' \| 'intent' \| 'modelName' \| 'modelFamily'\n >,\n): ConnectivityCheckResultItem {\n const { config } = modelRuntime;\n return {\n name,\n intent: config.intent,\n modelName: config.modelName,\n modelFamily: config.modelFamily,\n ...result,\n };\n}\n\nfunction buildConnectivityModelRuntime(\n modelConfig: IModelConfig,\n): ModelRuntime {\n return getModelRuntime({ ...modelConfig, retryCount: 0 });\n}\n\nfunction formatConnectivityCheckName(\n check: ConnectivityCheckResultItem,\n): string {\n const checkName =\n check.name === 'aiLocate'\n ? 'AI locate check'\n : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;\n const modelLabel = check.modelName \|\| check.intent;\n return `${checkName} - ${modelLabel} (${check.intent})`;\n}\n\nfunction buildConnectivityFailureMessage(\n checks: ConnectivityCheckResultItem[],\n): string {\n const failedChecks = checks.filter((item) => !item.passed);\n if (failedChecks.length === 0) {\n return 'Connectivity test failed, but no failed check details were generated.';\n }\n\n return failedChecks\n .map((item) => {\n const detail = item.message \|\| 'Failed without details.';\n return `[${formatConnectivityCheckName(item)}]: ${detail}`;\n })\n .join('\\n');\n}\n\nasync function runTextConnectivityCheck(\n modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n const startTime = Date.now();\n try {\n const result = await callAI(\n [\n {\n role: 'system',\n content: 'Reply with the exact token the user asks for.',\n },\n {\n role: 'user',\n content: `Return exactly ${TEXT_EXPECTED_TOKEN}`,\n },\n ],\n modelRuntime,\n );\n const content = result.content.trim();\n const passed = content.includes(TEXT_EXPECTED_TOKEN);\n return buildCheckResult('text', modelRuntime, {\n passed,\n durationMs: Date.now() - startTime,\n message: passed ? '' : `Unexpected response: ${content}`,\n });\n } catch (error) {\n return buildCheckResult('text', modelRuntime, {\n passed: false,\n durationMs: Date.now() - startTime,\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\nasync function runVisionConnectivityCheck(\n modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n const startTime = Date.now();\n try {\n await callAI(\n [\n {\n role: 'user',\n content: [\n {\n type: 'text',\n text: 'What is the main content of this image ? It is a photo or a form ?',\n },\n {\n type: 'image_url',\n image_url: {\n url: CONNECTIVITY_FIXTURE_IMAGE,\n detail: 'high',\n },\n },\n ],\n },\n ],\n modelRuntime,\n );\n return buildCheckResult('vision', modelRuntime, {\n passed: true,\n durationMs: Date.now() - startTime,\n message: '',\n });\n } catch (error) {\n return buildCheckResult('vision', modelRuntime, {\n passed: false,\n durationMs: Date.now() - startTime,\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\nasync function runAiLocateConnectivityCheck(\n modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n const startTime = Date.now();\n try {\n const context = buildFixtureContext();\n const service = new Service(context);\n const locateResult = await service.locate(\n { prompt: 'the main todo input box' },\n {},\n modelRuntime,\n );\n const targetRect = locateResult.rect \|\| locateResult.element?.rect;\n const center = locateResult.element?.center;\n const passed = hasValidRect(targetRect) && hasValidCenter(center);\n return buildCheckResult('aiLocate', modelRuntime, {\n passed,\n durationMs: Date.now() - startTime,\n message: passed\n ? ''\n : `Invalid locate result: ${JSON.stringify({\n rect: targetRect,\n center,\n })}`,\n });\n } catch (error) {\n return buildCheckResult('aiLocate', modelRuntime, {\n passed: false,\n durationMs: Date.now() - startTime,\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\nexport async function runConnectivityTest(\n config: ConnectivityTestConfig,\n): Promise<ConnectivityTestResult> {\n const planningModelRuntime = buildConnectivityModelRuntime(\n config.planningModelConfig,\n );\n const insightModelRuntime = buildConnectivityModelRuntime(\n config.insightModelConfig,\n );\n const defaultModelRuntime = buildConnectivityModelRuntime(\n config.defaultModelConfig,\n );\n const checks = await Promise.all([\n runTextConnectivityCheck(planningModelRuntime),\n runVisionConnectivityCheck(insightModelRuntime),\n runAiLocateConnectivityCheck(defaultModelRuntime),\n ]);\n\n const passed = checks.every((item) => item.passed);\n return {\n passed,\n message: passed ? undefined : buildConnectivityFailureMessage(checks),\n };\n}\n"],"names":["TEXT_EXPECTED_TOKEN","isFiniteNumber","value","Number","hasValidRect","rect","hasValidCenter","Array","buildFixtureContext","ScreenshotItem","CONNECTIVITY_FIXTURE_IMAGE","Date","CONNECTIVITY_FIXTURE_SHOT_SIZE","buildCheckResult","name","modelRuntime","result","config","buildConnectivityModelRuntime","modelConfig","getModelRuntime","formatConnectivityCheckName","check","checkName","modelLabel","buildConnectivityFailureMessage","checks","failedChecks","item","detail","runTextConnectivityCheck","startTime","callAI","content","passed","error","Error","String","runVisionConnectivityCheck","runAiLocateConnectivityCheck","context","service","Service","locateResult","targetRect","center","JSON","runConnectivityTest","planningModelRuntime","insightModelRuntime","defaultModelRuntime","Promise","undefined"],"mappings":";;;;;AAYA,MAAMA,sBAAsB;AAuB5B,SAASC,eAAeC,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AACtD;AAEA,SAASE,aAAaF,KAAc;IAClC,IAAI,CAACA,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAMG,OAAOH;IAOb,OACED,eAAeI,KAAK,IAAI,KACxBJ,eAAeI,KAAK,GAAG,KACvBJ,eAAeI,KAAK,KAAK,KACzBJ,eAAeI,KAAK,MAAM;AAE9B;AAEA,SAASC,eAAeJ,KAAc;IACpC,OACEK,MAAM,OAAO,CAACL,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZD,eAAeC,KAAK,CAAC,EAAE,KACvBD,eAAeC,KAAK,CAAC,EAAE;AAE3B;AAEA,SAASM;IACP,OAAO;QACL,YAAYC,eAAe,MAAM,CAACC,4BAA4BC,KAAK,GAAG;QACtE,UAAUC;QACV,0BAA0B;IAC5B;AACF;AAEA,SAASC,iBACPC,IAAyC,EACzCC,YAA0B,EAC1BC,MAGC;IAED,MAAM,EAAEC,MAAM,EAAE,GAAGF;IACnB,OAAO;QACLD;QACA,QAAQG,OAAO,MAAM;QACrB,WAAWA,OAAO,SAAS;QAC3B,aAAaA,OAAO,WAAW;QAC/B,GAAGD,MAAM;IACX;AACF;AAEA,SAASE,8BACPC,WAAyB;IAEzB,OAAOC,gBAAgB;QAAE,GAAGD,WAAW;QAAE,YAAY;IAAE;AACzD;AAEA,SAASE,4BACPC,KAAkC;IAElC,MAAMC,YACJD,AAAe,eAAfA,MAAM,IAAI,GACN,oBACA,GAAGA,MAAM,IAAI,CAAC,EAAE,EAAE,gBAAgBA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACnE,MAAME,aAAaF,MAAM,SAAS,IAAIA,MAAM,MAAM;IAClD,OAAO,GAAGC,UAAU,GAAG,EAAEC,WAAW,EAAE,EAAEF,MAAM,MAAM,CAAC,CAAC,CAAC;AACzD;AAEA,SAASG,gCACPC,MAAqC;IAErC,MAAMC,eAAeD,OAAO,MAAM,CAAC,CAACE,OAAS,CAACA,KAAK,MAAM;IACzD,IAAID,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAO;IAGT,OAAOA,aACJ,GAAG,CAAC,CAACC;QACJ,MAAMC,SAASD,KAAK,OAAO,IAAI;QAC/B,OAAO,CAAC,CAAC,EAAEP,4BAA4BO,MAAM,GAAG,EAAEC,QAAQ;IAC5D,GACC,IAAI,CAAC;AACV;AAEA,eAAeC,yBACbf,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMK,SAAS,MAAMgB,OACnB;YACE;gBACE,MAAM;gBACN,SAAS;YACX;YACA;gBACE,MAAM;gBACN,SAAS,CAAC,eAAe,EAAEhC,qBAAqB;YAClD;SACD,EACDe;QAEF,MAAMkB,UAAUjB,OAAO,OAAO,CAAC,IAAI;QACnC,MAAMkB,SAASD,QAAQ,QAAQ,CAACjC;QAChC,OAAOa,iBAAiB,QAAQE,cAAc;YAC5CmB;YACA,YAAYvB,KAAK,GAAG,KAAKoB;YACzB,SAASG,SAAS,KAAK,CAAC,qBAAqB,EAAED,SAAS;QAC1D;IACF,EAAE,OAAOE,OAAO;QACd,OAAOtB,iBAAiB,QAAQE,cAAc;YAC5C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeG,2BACbvB,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMqB,OACJ;YACE;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM;oBACR;oBACA;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKtB;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD,EACDK;QAEF,OAAOF,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAAS;QACX;IACF,EAAE,OAAOI,OAAO;QACd,OAAOtB,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeI,6BACbxB,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAM6B,UAAUhC;QAChB,MAAMiC,UAAU,IAAIC,UAAQF;QAC5B,MAAMG,eAAe,MAAMF,QAAQ,MAAM,CACvC;YAAE,QAAQ;QAA0B,GACpC,CAAC,GACD1B;QAEF,MAAM6B,aAAaD,aAAa,IAAI,IAAIA,aAAa,OAAO,EAAE;QAC9D,MAAME,SAASF,aAAa,OAAO,EAAE;QACrC,MAAMT,SAAS9B,aAAawC,eAAetC,eAAeuC;QAC1D,OAAOhC,iBAAiB,YAAYE,cAAc;YAChDmB;YACA,YAAYvB,KAAK,GAAG,KAAKoB;YACzB,SAASG,SACL,KACA,CAAC,uBAAuB,EAAEY,KAAK,SAAS,CAAC;gBACvC,MAAMF;gBACNC;YACF,IAAI;QACV;IACF,EAAE,OAAOV,OAAO;QACd,OAAOtB,iBAAiB,YAAYE,cAAc;YAChD,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEO,eAAeY,oBACpB9B,MAA8B;IAE9B,MAAM+B,uBAAuB9B,8BAC3BD,OAAO,mBAAmB;IAE5B,MAAMgC,sBAAsB/B,8BAC1BD,OAAO,kBAAkB;IAE3B,MAAMiC,sBAAsBhC,8BAC1BD,OAAO,kBAAkB;IAE3B,MAAMS,SAAS,MAAMyB,QAAQ,GAAG,CAAC;QAC/BrB,yBAAyBkB;QACzBV,2BAA2BW;QAC3BV,6BAA6BW;KAC9B;IAED,MAAMhB,SAASR,OAAO,KAAK,CAAC,CAACE,OAASA,KAAK,MAAM;IACjD,OAAO;QACLM;QACA,SAASA,SAASkB,SAAY3B,gCAAgCC;IAChE;AACF"}
1	+ {"version":3,"file":"ai-model/connectivity/run-connectivity-test.mjs","sources":["../../../../src/ai-model/connectivity/run-connectivity-test.ts"],"sourcesContent":["import { ScreenshotItem } from '@/screenshot-item';\nimport Service from '@/service';\nimport type { UIContext } from '@/types';\nimport type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getModelRuntime } from '../models';\nimport type { ModelRuntime } from '../models';\nimport { callAI } from '../service-caller';\nimport {\n CONNECTIVITY_FIXTURE_IMAGE,\n CONNECTIVITY_FIXTURE_SHOT_SIZE,\n} from './fixture';\n\nconst TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';\n\ninterface ConnectivityCheckResultItem {\n name: 'text' \| 'vision' \| 'aiLocate';\n intent: TIntent;\n modelName: string;\n modelFamily?: string;\n passed: boolean;\n durationMs: number;\n message: string;\n}\n\nexport interface ConnectivityTestResult {\n passed: boolean;\n message?: string;\n}\n\nexport interface ConnectivityTestConfig {\n defaultModelConfig: IModelConfig;\n planningModelConfig: IModelConfig;\n insightModelConfig: IModelConfig;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction hasValidCenter(value: unknown): boolean {\n return (\n Array.isArray(value) &&\n value.length === 2 &&\n isFiniteNumber(value[0]) &&\n isFiniteNumber(value[1])\n );\n}\n\nfunction buildFixtureContext(): UIContext {\n return {\n screenshot: ScreenshotItem.create(CONNECTIVITY_FIXTURE_IMAGE, Date.now()),\n shotSize: CONNECTIVITY_FIXTURE_SHOT_SIZE,\n shrunkShotToLogicalRatio: 1,\n };\n}\n\nfunction buildCheckResult(\n name: ConnectivityCheckResultItem['name'],\n modelRuntime: ModelRuntime,\n result: Omit<\n ConnectivityCheckResultItem,\n 'name' \| 'intent' \| 'modelName' \| 'modelFamily'\n >,\n): ConnectivityCheckResultItem {\n const { config } = modelRuntime;\n return {\n name,\n intent: config.intent,\n modelName: config.modelName,\n modelFamily: config.modelFamily,\n ...result,\n };\n}\n\nfunction buildConnectivityModelRuntime(\n modelConfig: IModelConfig,\n): ModelRuntime {\n return getModelRuntime({ ...modelConfig, retryCount: 0 });\n}\n\nfunction formatConnectivityCheckName(\n check: ConnectivityCheckResultItem,\n): string {\n const checkName =\n check.name === 'aiLocate'\n ? 'AI locate check'\n : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;\n const modelLabel = check.modelName \|\| check.intent;\n return `${checkName} - ${modelLabel} (${check.intent})`;\n}\n\nfunction buildConnectivityFailureMessage(\n checks: ConnectivityCheckResultItem[],\n): string {\n const failedChecks = checks.filter((item) => !item.passed);\n if (failedChecks.length === 0) {\n return 'Connectivity test failed, but no failed check details were generated.';\n }\n\n return failedChecks\n .map((item) => {\n const detail = item.message \|\| 'Failed without details.';\n return `[${formatConnectivityCheckName(item)}]: ${detail}`;\n })\n .join('\\n');\n}\n\nasync function runTextConnectivityCheck(\n modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n const startTime = Date.now();\n try {\n const result = await callAI(\n [\n {\n role: 'system',\n content: 'Reply with the exact token the user asks for.',\n },\n {\n role: 'user',\n content: `Return exactly ${TEXT_EXPECTED_TOKEN}`,\n },\n ],\n modelRuntime,\n );\n const content = result.content.trim();\n const passed = content.includes(TEXT_EXPECTED_TOKEN);\n return buildCheckResult('text', modelRuntime, {\n passed,\n durationMs: Date.now() - startTime,\n message: passed ? '' : `Unexpected response: ${content}`,\n });\n } catch (error) {\n return buildCheckResult('text', modelRuntime, {\n passed: false,\n durationMs: Date.now() - startTime,\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\nasync function runVisionConnectivityCheck(\n modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n const startTime = Date.now();\n try {\n await callAI(\n [\n {\n role: 'user',\n content: [\n {\n type: 'text',\n text: 'What is the main content of this image ? It is a photo or a form ?',\n },\n {\n type: 'image_url',\n image_url: {\n url: CONNECTIVITY_FIXTURE_IMAGE,\n detail: 'high',\n },\n },\n ],\n },\n ],\n modelRuntime,\n );\n return buildCheckResult('vision', modelRuntime, {\n passed: true,\n durationMs: Date.now() - startTime,\n message: '',\n });\n } catch (error) {\n return buildCheckResult('vision', modelRuntime, {\n passed: false,\n durationMs: Date.now() - startTime,\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\nasync function runAiLocateConnectivityCheck(\n modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n const startTime = Date.now();\n try {\n const context = buildFixtureContext();\n const service = new Service(context);\n const locateResult = await service.locate(\n { prompt: 'the main todo input box' },\n {},\n modelRuntime,\n );\n const center = locateResult.element?.center;\n const passed = hasValidCenter(center);\n return buildCheckResult('aiLocate', modelRuntime, {\n passed,\n durationMs: Date.now() - startTime,\n message: passed\n ? ''\n : `Invalid locate result: ${JSON.stringify({\n center,\n })}`,\n });\n } catch (error) {\n return buildCheckResult('aiLocate', modelRuntime, {\n passed: false,\n durationMs: Date.now() - startTime,\n message: error instanceof Error ? error.message : String(error),\n });\n }\n}\n\nexport async function runConnectivityTest(\n config: ConnectivityTestConfig,\n): Promise<ConnectivityTestResult> {\n const planningModelRuntime = buildConnectivityModelRuntime(\n config.planningModelConfig,\n );\n const insightModelRuntime = buildConnectivityModelRuntime(\n config.insightModelConfig,\n );\n const defaultModelRuntime = buildConnectivityModelRuntime(\n config.defaultModelConfig,\n );\n const checks = await Promise.all([\n runTextConnectivityCheck(planningModelRuntime),\n runVisionConnectivityCheck(insightModelRuntime),\n runAiLocateConnectivityCheck(defaultModelRuntime),\n ]);\n\n const passed = checks.every((item) => item.passed);\n return {\n passed,\n message: passed ? undefined : buildConnectivityFailureMessage(checks),\n };\n}\n"],"names":["TEXT_EXPECTED_TOKEN","isFiniteNumber","value","Number","hasValidCenter","Array","buildFixtureContext","ScreenshotItem","CONNECTIVITY_FIXTURE_IMAGE","Date","CONNECTIVITY_FIXTURE_SHOT_SIZE","buildCheckResult","name","modelRuntime","result","config","buildConnectivityModelRuntime","modelConfig","getModelRuntime","formatConnectivityCheckName","check","checkName","modelLabel","buildConnectivityFailureMessage","checks","failedChecks","item","detail","runTextConnectivityCheck","startTime","callAI","content","passed","error","Error","String","runVisionConnectivityCheck","runAiLocateConnectivityCheck","context","service","Service","locateResult","center","JSON","runConnectivityTest","planningModelRuntime","insightModelRuntime","defaultModelRuntime","Promise","undefined"],"mappings":";;;;;AAYA,MAAMA,sBAAsB;AAuB5B,SAASC,eAAeC,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AACtD;AAEA,SAASE,eAAeF,KAAc;IACpC,OACEG,MAAM,OAAO,CAACH,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZD,eAAeC,KAAK,CAAC,EAAE,KACvBD,eAAeC,KAAK,CAAC,EAAE;AAE3B;AAEA,SAASI;IACP,OAAO;QACL,YAAYC,eAAe,MAAM,CAACC,4BAA4BC,KAAK,GAAG;QACtE,UAAUC;QACV,0BAA0B;IAC5B;AACF;AAEA,SAASC,iBACPC,IAAyC,EACzCC,YAA0B,EAC1BC,MAGC;IAED,MAAM,EAAEC,MAAM,EAAE,GAAGF;IACnB,OAAO;QACLD;QACA,QAAQG,OAAO,MAAM;QACrB,WAAWA,OAAO,SAAS;QAC3B,aAAaA,OAAO,WAAW;QAC/B,GAAGD,MAAM;IACX;AACF;AAEA,SAASE,8BACPC,WAAyB;IAEzB,OAAOC,gBAAgB;QAAE,GAAGD,WAAW;QAAE,YAAY;IAAE;AACzD;AAEA,SAASE,4BACPC,KAAkC;IAElC,MAAMC,YACJD,AAAe,eAAfA,MAAM,IAAI,GACN,oBACA,GAAGA,MAAM,IAAI,CAAC,EAAE,EAAE,gBAAgBA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACnE,MAAME,aAAaF,MAAM,SAAS,IAAIA,MAAM,MAAM;IAClD,OAAO,GAAGC,UAAU,GAAG,EAAEC,WAAW,EAAE,EAAEF,MAAM,MAAM,CAAC,CAAC,CAAC;AACzD;AAEA,SAASG,gCACPC,MAAqC;IAErC,MAAMC,eAAeD,OAAO,MAAM,CAAC,CAACE,OAAS,CAACA,KAAK,MAAM;IACzD,IAAID,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAO;IAGT,OAAOA,aACJ,GAAG,CAAC,CAACC;QACJ,MAAMC,SAASD,KAAK,OAAO,IAAI;QAC/B,OAAO,CAAC,CAAC,EAAEP,4BAA4BO,MAAM,GAAG,EAAEC,QAAQ;IAC5D,GACC,IAAI,CAAC;AACV;AAEA,eAAeC,yBACbf,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMK,SAAS,MAAMgB,OACnB;YACE;gBACE,MAAM;gBACN,SAAS;YACX;YACA;gBACE,MAAM;gBACN,SAAS,CAAC,eAAe,EAAE9B,qBAAqB;YAClD;SACD,EACDa;QAEF,MAAMkB,UAAUjB,OAAO,OAAO,CAAC,IAAI;QACnC,MAAMkB,SAASD,QAAQ,QAAQ,CAAC/B;QAChC,OAAOW,iBAAiB,QAAQE,cAAc;YAC5CmB;YACA,YAAYvB,KAAK,GAAG,KAAKoB;YACzB,SAASG,SAAS,KAAK,CAAC,qBAAqB,EAAED,SAAS;QAC1D;IACF,EAAE,OAAOE,OAAO;QACd,OAAOtB,iBAAiB,QAAQE,cAAc;YAC5C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeG,2BACbvB,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMqB,OACJ;YACE;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM;oBACR;oBACA;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKtB;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD,EACDK;QAEF,OAAOF,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAAS;QACX;IACF,EAAE,OAAOI,OAAO;QACd,OAAOtB,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeI,6BACbxB,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAM6B,UAAUhC;QAChB,MAAMiC,UAAU,IAAIC,UAAQF;QAC5B,MAAMG,eAAe,MAAMF,QAAQ,MAAM,CACvC;YAAE,QAAQ;QAA0B,GACpC,CAAC,GACD1B;QAEF,MAAM6B,SAASD,aAAa,OAAO,EAAE;QACrC,MAAMT,SAAS5B,eAAesC;QAC9B,OAAO/B,iBAAiB,YAAYE,cAAc;YAChDmB;YACA,YAAYvB,KAAK,GAAG,KAAKoB;YACzB,SAASG,SACL,KACA,CAAC,uBAAuB,EAAEW,KAAK,SAAS,CAAC;gBACvCD;YACF,IAAI;QACV;IACF,EAAE,OAAOT,OAAO;QACd,OAAOtB,iBAAiB,YAAYE,cAAc;YAChD,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEO,eAAeW,oBACpB7B,MAA8B;IAE9B,MAAM8B,uBAAuB7B,8BAC3BD,OAAO,mBAAmB;IAE5B,MAAM+B,sBAAsB9B,8BAC1BD,OAAO,kBAAkB;IAE3B,MAAMgC,sBAAsB/B,8BAC1BD,OAAO,kBAAkB;IAE3B,MAAMS,SAAS,MAAMwB,QAAQ,GAAG,CAAC;QAC/BpB,yBAAyBiB;QACzBT,2BAA2BU;QAC3BT,6BAA6BU;KAC9B;IAED,MAAMf,SAASR,OAAO,KAAK,CAAC,CAACE,OAASA,KAAK,MAAM;IACjD,OAAO;QACLM;QACA,SAASA,SAASiB,SAAY1B,gCAAgCC;IAChE;AACF"}

package/dist/es/ai-model/model-adapter/codex-app-server.mjs ADDED Viewed

@@ -0,0 +1,23 @@
+const buildDefaultCodexAppServerParams = ({ userConfig = {} })=>{
+    if (true !== userConfig.reasoningEnabled) return {
+        config: {
+            effort: 'none'
+        }
+    };
+    const effort = userConfig.reasoningEffort?.trim().toLowerCase();
+    return {
+        config: {
+            effort: effort && [
+                'none',
+                'minimal',
+                'low',
+                'medium',
+                'high',
+                'xhigh'
+            ].includes(effort) ? effort : 'medium'
+        }
+    };
+};
+export { buildDefaultCodexAppServerParams };
+//# sourceMappingURL=codex-app-server.mjs.map

package/dist/es/ai-model/model-adapter/codex-app-server.mjs.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"ai-model/model-adapter/codex-app-server.mjs","sources":["../../../../src/ai-model/model-adapter/codex-app-server.ts"],"sourcesContent":["import type {\n CodexAppServerCallInput,\n CodexAppServerParamsResult,\n} from './types';\n\nexport const buildDefaultCodexAppServerParams = ({\n userConfig = {},\n}: CodexAppServerCallInput): CodexAppServerParamsResult => {\n if (userConfig.reasoningEnabled !== true) {\n return { config: { effort: 'none' } };\n }\n\n const effort = userConfig.reasoningEffort?.trim().toLowerCase();\n return {\n config: {\n effort:\n effort &&\n ['none', 'minimal', 'low', 'medium', 'high', 'xhigh'].includes(effort)\n ? effort\n : 'medium',\n },\n };\n};\n"],"names":["buildDefaultCodexAppServerParams","userConfig","effort"],"mappings":"AAKO,MAAMA,mCAAmC,CAAC,EAC/CC,aAAa,CAAC,CAAC,EACS;IACxB,IAAIA,AAAgC,SAAhCA,WAAW,gBAAgB,EAC7B,OAAO;QAAE,QAAQ;YAAE,QAAQ;QAAO;IAAE;IAGtC,MAAMC,SAASD,WAAW,eAAe,EAAE,OAAO;IAClD,OAAO;QACL,QAAQ;YACN,QACEC,UACA;gBAAC;gBAAQ;gBAAW;gBAAO;gBAAU;gBAAQ;aAAQ,CAAC,QAAQ,CAACA,UAC3DA,SACA;QACR;IACF;AACF"}

package/dist/es/ai-model/model-adapter/custom-planning-action.mjs CHANGED Viewed

@@ -1,6 +1,6 @@
-function getTapLocatedPixelBbox(actions) {
-    for (const action of actions)if ('Tap' === action.type) return action.param.locate.locatedPixelBbox;
+function getTapLocatedPixelResult(actions) {
+    for (const action of actions)if ('Tap' === action.type) return action.param.locate.locatedPixelResult;
 }
-export { getTapLocatedPixelBbox };
+export { getTapLocatedPixelResult };
 //# sourceMappingURL=custom-planning-action.mjs.map