npm - @midscene/core - Versions diffs - 1.8.11 → 1.9.0 - Mend

@midscene/core 1.8.11 → 1.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (304) hide show

package/dist/es/agent/task-builder.mjs CHANGED Viewed

@@ -6,7 +6,7 @@ import { generateElementByRect } from "@midscene/shared/extractor";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { withUsageIntent } from "./usage-intent.mjs";
-import { ifPlanLocateParamIsBbox, matchElementFromCache, matchElementFromPlan, transformLogicalElementToScreenshot, transformLogicalRectToScreenshotRect } from "./utils.mjs";
+import { ifPlanLocateParamHasLocatedPixelBbox, matchElementFromCache, matchElementFromPlan, transformLogicalElementToScreenshot, transformLogicalRectToScreenshotRect } from "./utils.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -21,6 +21,18 @@ const debug = getDebug('agent:task-builder');
 function hasNonEmptyCache(cache) {
     return null != cache && 'object' == typeof cache && Object.keys(cache).length > 0;
 }
+function invalidLocateElementReason(element) {
+    const values = [
+        element.center?.[0],
+        element.center?.[1],
+        element.rect?.left,
+        element.rect?.top,
+        element.rect?.width,
+        element.rect?.height
+    ];
+    if (values.some((value)=>'number' != typeof value || !Number.isFinite(value))) return `Invalid locate result coordinates: ${JSON.stringify(element)}`;
+    if (element.rect.width <= 0 || element.rect.height <= 0) return `Invalid locate result rect size: ${JSON.stringify(element)}`;
+}
 function locatePlanForLocate(param) {
     const locate = 'string' == typeof param ? {
         prompt: param
@@ -33,13 +45,13 @@ function locatePlanForLocate(param) {
     return locatePlan;
 }
 class TaskBuilder {
-    async build(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options) {
+    async build(plans, planningModel, defaultModel, options) {
         const tasks = [];
         const cacheable = options?.cacheable;
         const context = {
             tasks,
-            modelConfigForPlanning,
-            modelConfigForDefaultIntent,
+            planningModel,
+            defaultModel,
             cacheable,
             deepLocate: options?.deepLocate,
             abortSignal: options?.abortSignal
@@ -88,7 +100,7 @@ class TaskBuilder {
         locateFields.forEach((field)=>{
             if (param[field]) {
                 const locatePlan = locatePlanForLocate(param[field]);
-                debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`, `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`);
+                debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`, `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`);
                 const locateTask = this.createLocateTask(locatePlan, param[field], context, (result)=>{
                     param[field] = result;
                 });
@@ -172,7 +184,7 @@ class TaskBuilder {
         context.tasks.push(task);
     }
     createLocateTask(plan, detailedLocateParam, context, onResult) {
-        const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } = context;
+        const { cacheable, defaultModel, deepLocate, abortSignal } = context;
         let locateParam = detailedLocateParam;
         if ('string' == typeof locateParam) locateParam = {
             prompt: locateParam
@@ -193,7 +205,8 @@ class TaskBuilder {
             executor: async (param, taskContext)=>{
                 const { task } = taskContext;
                 let { uiContext } = taskContext;
-                assert(param?.prompt || param?.bbox, `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
+                const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(param) ? param : void 0;
+                assert(param?.prompt || paramWithLocatedPixelBbox, `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(param)}`);
                 if (!uiContext) uiContext = await this.service.contextRetrieverFn();
                 assert(uiContext, 'uiContext is required for Service task');
                 const { shrunkShotToLogicalRatio } = uiContext;
@@ -211,11 +224,11 @@ class TaskBuilder {
                     if (dump.taskInfo?.searchAreaUsage) task.searchAreaUsage = dump.taskInfo.searchAreaUsage;
                     if (dump.taskInfo?.reasoning_content) task.reasoning_content = dump.taskInfo.reasoning_content;
                 };
-                const planLocatedElement = ifPlanLocateParamIsBbox(param) ? matchElementFromPlan(param) : void 0;
-                const elementFromBbox = param.deepLocate ? void 0 : planLocatedElement;
-                const isPlanHit = !!elementFromBbox;
+                const planLocatedElement = paramWithLocatedPixelBbox ? matchElementFromPlan(paramWithLocatedPixelBbox) : void 0;
+                const elementFromPlan = param.deepLocate ? void 0 : planLocatedElement;
+                const isPlanDirectHit = !!elementFromPlan;
                 let rectFromXpath;
-                if (!isPlanHit && param.xpath && this.interface.rectMatchesCacheFeature) try {
+                if (!isPlanDirectHit && param.xpath && this.interface.rectMatchesCacheFeature) try {
                     rectFromXpath = await this.interface.rectMatchesCacheFeature({
                         xpaths: [
                             param.xpath
@@ -227,7 +240,7 @@ class TaskBuilder {
                 const cachePrompt = param.prompt;
                 const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
                 const cacheEntry = locateCacheRecord?.cacheContent?.cache;
-                const elementFromCacheResult = isPlanHit || isXpathHit ? null : await matchElementFromCache({
+                const elementFromCacheResult = isPlanDirectHit || isXpathHit ? null : await matchElementFromCache({
                     taskCache: this.taskCache,
                     interfaceInstance: this.interface
                 }, cacheEntry, cachePrompt, param.cacheable);
@@ -235,12 +248,12 @@ class TaskBuilder {
                 const isCacheHit = !!elementFromCache;
                 let elementFromAiLocate;
                 const timing = taskContext.task.timing;
-                if (!isXpathHit && !isCacheHit && !isPlanHit) try {
+                if (!isXpathHit && !isCacheHit && !isPlanDirectHit) try {
                     setTimingFieldOnce(timing, 'callAiStart');
                     locateResult = await this.service.locate(param, {
                         context: uiContext,
                         planLocatedElement
-                    }, modelConfigForDefaultIntent, abortSignal);
+                    }, defaultModel, abortSignal);
                     applyDump(locateResult.dump);
                     elementFromAiLocate = locateResult.element;
                 } catch (error) {
@@ -249,10 +262,17 @@ class TaskBuilder {
                 } finally{
                     setTimingFieldOnce(timing, 'callAiEnd');
                 }
-                const element = elementFromBbox || elementFromXpath || elementFromCache || elementFromAiLocate;
+                const element = elementFromPlan || elementFromXpath || elementFromCache || elementFromAiLocate;
+                if (element) {
+                    const invalidElementReason = invalidLocateElementReason(element);
+                    if (invalidElementReason) {
+                        if (locateDump) throw new ServiceError(invalidElementReason, locateDump);
+                        throw new Error(invalidElementReason);
+                    }
+                }
                 const locateCacheAlreadyExists = hasNonEmptyCache(locateCacheRecord?.cacheContent?.cache);
                 let currentCacheEntry;
-                if (element && this.taskCache && !isCacheHit && (!isPlanHit || !locateCacheAlreadyExists) && param?.cacheable !== false) if (this.interface.cacheFeatureForPoint) try {
+                if (element && this.taskCache && !isCacheHit && (!isPlanDirectHit || !locateCacheAlreadyExists) && param?.cacheable !== false) if (this.interface.cacheFeatureForPoint) try {
                     let pointForCache = element.center;
                     if (1 !== shrunkShotToLogicalRatio) {
                         pointForCache = [
@@ -263,7 +283,7 @@ class TaskBuilder {
                     }
                     const feature = await this.interface.cacheFeatureForPoint(pointForCache, {
                         targetDescription: 'string' == typeof param.prompt ? param.prompt : param.prompt?.prompt,
-                        modelConfig: modelConfigForDefaultIntent
+                        modelRuntime: defaultModel
                     });
                     if (hasNonEmptyCache(feature)) {
                         debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
@@ -283,10 +303,10 @@ class TaskBuilder {
                     throw new Error(`Element not found: ${param.prompt}`);
                 }
                 let hitBy;
-                if (isPlanHit) hitBy = {
+                if (isPlanDirectHit && paramWithLocatedPixelBbox) hitBy = {
                     from: 'Plan',
                     context: {
-                        bbox: param.bbox
+                        locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox
                     }
                 };
                 else if (isXpathHit) hitBy = {

package/dist/es/agent/task-builder.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n AIUsageInfo,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n PlanningLocateParam,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n ifPlanLocateParamIsBbox,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = typeof param === 'string' ? { prompt: param } : param;\n const locatePlan: PlanningAction<PlanningLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n modelConfigForPlanning: IModelConfig;\n modelConfigForDefaultIntent: IModelConfig;\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n\n constructor({\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n }\n\n public async build(\n plans: PlanningAction[],\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n cacheable,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<PlanningLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<PlanningLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = actionSpace.find((item) => item.name === planType);\n const param = plan.param;\n\n if (!action) {\n throw new Error(`Action type '${planType}' not found`);\n }\n\n const locateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema)\n : [];\n\n const requiredLocateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema, true)\n : [];\n\n locateFields.forEach((field) => {\n if (param[field]) {\n // Always use createLocateTask for all locate params (including bbox)\n // This ensures cache writing happens even when bbox is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n executor: async (param, taskContext) => {\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug(\n `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.beforeInvokeAction(action.name, param);\n debug(\n `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n })(),\n delayBeforeRunner > 0\n ? sleep(delayBeforeRunner)\n : Promise.resolve(),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n\n if (action.paramSchema) {\n try {\n param = parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n }\n\n setTimingFieldOnce(timing, 'callActionStart');\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(param, taskContext);\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug(\n `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.afterInvokeAction(action.name, param);\n debug(\n `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<PlanningLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } =\n context;\n\n let locateParam = detailedLocateParam;\n\n if (typeof locateParam === 'string') {\n locateParam = {\n prompt: locateParam,\n };\n }\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n\n assert(\n param?.prompt \|\| param?.bbox,\n `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n param,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n if (dump.taskInfo?.searchAreaUsage) {\n task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n const planLocatedElement = ifPlanLocateParamIsBbox(param)\n ? matchElementFromPlan(param)\n : undefined;\n\n // from bbox (plan hit)\n // when deepLocate is enabled, bbox should be used as search area hint,\n // not as a final direct hit\n const elementFromBbox = param.deepLocate\n ? undefined\n : planLocatedElement;\n const isPlanHit = !!elementFromBbox;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanHit &&\n param.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [param.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? generateElementByRect(\n // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n transformLogicalRectToScreenshotRect(\n rectFromXpath,\n shrunkShotToLogicalRatio,\n ),\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt \|\| '',\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = param.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n param.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n const elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isCacheHit = !!elementFromCache;\n\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n const timing = taskContext.task.timing;\n if (!isXpathHit && !isCacheHit && !isPlanHit) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n param,\n {\n context: uiContext,\n planLocatedElement,\n },\n modelConfigForDefaultIntent,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n\n const element =\n elementFromBbox \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read)\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n !isCacheHit &&\n (!isPlanHit \|\| !locateCacheAlreadyExists) &&\n param?.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n let pointForCache: [number, number] = element.center;\n if (shrunkShotToLogicalRatio !== 1) {\n pointForCache = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n debug(\n 'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n element.center,\n pointForCache,\n );\n }\n\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt,\n modelConfig: modelConfigForDefaultIntent,\n },\n );\n if (hasNonEmptyCache(feature)) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${param.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${param.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanHit) {\n hitBy = {\n from: 'Plan',\n context: {\n bbox: param.bbox,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: param.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n onResult?.(element);\n\n return {\n output: {\n element: {\n ...element,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromBbox","isPlanHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;AAoCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDhC,MACE,uCACA,CAAC,YAAY,EAAEwB,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAC3B,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAACzB,aAAa,EAC1C,CAAC,QAAQ,EAAE0B,wBAAwB5B,KAAK,CAAC0B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtC3B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACoB;oBACC9B,KAAK,CAAC0B,MAAM,GAAGI;gBACjB;gBAEFpB,QAAQ,KAAK,CAAC,IAAI,CAACmB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3ExB,MAAM,CAAC,OAAO,EAAEgC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOiC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtCvC,MACE,oBACAwB,UACAlB,OACA,CAAC,4BAA4B,EAAEiC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBV,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACE/B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAkB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBjB,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMkB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC5C,MACE,CAAC,8DAA8D,EAAE0B,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDN,MACE,CAAC,2DAA2D,EAAE0B,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAiB,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIlB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEqB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAES,wBAAwB,EAAE,GAAGR;gBACrC,IAAIQ,AAA6BC,WAA7BD,0BACF,MAAM,IAAIrB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ6C,iBAAiB7C,OAAOoB,OAAO,WAAW,EAAE;wBAClDuB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIxB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAE0B,MAAM,OAAO,CAAC,cAAc,EAAEnB,KAAK,SAAS,CAAC3B,QAAQ,EACtG;wBAAE,OAAO8C;oBAAM;gBAEnB;gBAGFV,mBAAmBF,QAAQ;gBAE3BxC,MAAM,kBAAkB0B,OAAO,IAAI;gBACnC,MAAM2B,WAAW3B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM4B,eAAe,MAAMD,SAAS/C,OAAOiC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BxC,MAAM,iBAAiB0B,OAAO,IAAI,EAAE,WAAW4B;gBAE/CZ,mBAAmBF,QAAQ;gBAE3B,MAAMe,mBACJ7B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI6B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCvD,MACE,CAAC,6DAA6D,EAAE0B,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDN,MACE,CAAC,0DAA0D,EAAE0B,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOoB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIlB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEqB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQc;gBACV;YACF;QACF;QAEAtC,QAAQ,KAAK,CAAC,IAAI,CAACsB;IACrB;IAEQ,iBACNnB,IAAyC,EACzCqC,mBAAiD,EACjDxC,OAAyB,EACzByC,QAAgD,EACd;QAClC,MAAM,EAAE1C,SAAS,EAAEH,2BAA2B,EAAE8C,UAAU,EAAEC,WAAW,EAAE,GACvE3C;QAEF,IAAI4C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI7C,AAAcmC,WAAdnC,WACF6C,cAAc;YACZ,GAAGA,WAAW;YACd7C;QACF;QAGF,IAAI2C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASzC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOiC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBAEpBF,OACE/B,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAE2B,KAAK,SAAS,CACpE3B,QACC;gBAGL,IAAI,CAACmC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEQ,wBAAwB,EAAE,GAAGR;gBAErC,IAAIQ,AAA6BC,WAA7BD,0BACF,MAAM,IAAIrB,MACR;gBAIJ,IAAIkC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb3B,KAAK,GAAG,GAAG;wBACT2B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA3B,KAAK,KAAK,GAAG4B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,iBACjB3B,KAAK,eAAe,GAAG2B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB3B,KAAK,iBAAiB,GAAG2B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBjC,wBAAwB5B,SAC/C8D,qBAAqB9D,SACrB4C;gBAKJ,MAAMmB,kBAAkB/D,MAAM,UAAU,GACpC4C,SACAiB;gBACJ,MAAMG,YAAY,CAAC,CAACD;gBAGpB,IAAIE;gBACJ,IACE,CAACD,aACDhE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFiE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACjE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMkE,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACAtB,2BAEF,AAAwB,YAAxB,OAAO3C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B4C;gBAEJ,MAAMyB,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAActE,MAAM,MAAM;gBAChC,MAAMuE,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,aAAaK,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAtE,MAAM,SAAS;gBAIvB,MAAM2E,mBAAmBF,yBACrBG,oCACEH,wBACA9B,4BAEFC;gBAEJ,MAAMiC,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM5C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACQ,cAAc,CAACb,WACjC,IAAI;oBACF5B,mBAAmBF,QAAQ;oBAC3BuB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCzD,OACA;wBACE,SAASmC;wBACT0B;oBACF,GACAvD,6BACA+C;oBAEFK,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,cACnBrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRV,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM8C,UACJjB,mBACAG,oBACAS,oBACAG;gBAGF,MAAMG,2BAA2BrF,iBAC/B2E,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACd,CAACH,cACA,EAACb,aAAa,CAACiB,wBAAuB,KACvCjF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAImF,gBAAkCH,QAAQ,MAAM;oBACpD,IAAIrC,AAA6B,MAA7BA,0BAAgC;wBAClCwC,gBAAgB;4BACdC,KAAK,KAAK,CAACJ,QAAQ,MAAM,CAAC,EAAE,GAAGrC;4BAC/ByC,KAAK,KAAK,CAACJ,QAAQ,MAAM,CAAC,EAAE,GAAGrC;yBAChC;wBACDjD,MACE,8DACAsF,QAAQ,MAAM,EACdG;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOnF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiByF,UAAU;wBAC7B3F,MACE,uCACA4E,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACE7E,MACE,yDACA4E;gBAGN,EAAE,OAAOxB,OAAO;oBACdpD,MAAM,mCAAmCoD;gBAC3C;qBAEApD,MAAM;gBAIV,IAAI,CAACsF,SAAS;oBACZ,IAAIxB,YACF,MAAM,IAAIuB,aACR,CAAC,oBAAoB,EAAE/E,MAAM,MAAM,EAAE,EACrCwD;oBAGJ,MAAM,IAAIlC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIsF;gBAEJ,IAAItB,WACFsB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAMtF,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAIqE,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOtF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI6E,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF/B,WAAW6B;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK7C,UAAU,aAAa;wBAC9B;oBACF;oBACAmD;gBACF;YACF;QACF;QAEA,OAAO/B;IACT;IAxjBA,YAAY,EACVgC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTtE,WAAW,EACXuE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGtE;QACnB,IAAI,CAAC,eAAe,GAAGuE;IACzB;AA6iBF"}
1	+ {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n AIUsageInfo,\n DetailedLocateParam,\n DeviceAction,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n PlanningLocateParam,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n ifPlanLocateParamHasLocatedPixelBbox,\n matchElementFromCache,\n matchElementFromPlan,\n transformLogicalElementToScreenshot,\n transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/*\n Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nfunction invalidLocateElementReason(\n element: LocateResultElement,\n): string \| undefined {\n const values = [\n element.center?.[0],\n element.center?.[1],\n element.rect?.left,\n element.rect?.top,\n element.rect?.width,\n element.rect?.height,\n ];\n if (\n values.some((value) => typeof value !== 'number' \|\| !Number.isFinite(value))\n ) {\n return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n }\n if (element.rect.width <= 0 \|\| element.rect.height <= 0) {\n return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n }\n return undefined;\n}\n\nexport function locatePlanForLocate(param: string \| DetailedLocateParam) {\n const locate = typeof param === 'string' ? { prompt: param } : param;\n const locatePlan: PlanningAction<PlanningLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n actionSpace: DeviceAction[];\n waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n planningModel: ModelRuntime;\n defaultModel: ModelRuntime;\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n private readonly actionSpace: DeviceAction[];\n\n private readonly waitAfterAction?: number;\n\n constructor({\n interfaceInstance,\n service,\n taskCache,\n actionSpace,\n waitAfterAction,\n }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n this.actionSpace = actionSpace;\n this.waitAfterAction = waitAfterAction;\n }\n\n public async build(\n plans: PlanningAction[],\n planningModel: ModelRuntime,\n defaultModel: ModelRuntime,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n planningModel,\n defaultModel,\n cacheable,\n deepLocate: options?.deepLocate,\n abortSignal: options?.abortSignal,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> \| void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<PlanningLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<PlanningLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = this.actionSpace;\n const action = actionSpace.find((item) => item.name === planType);\n const param = plan.param;\n\n if (!action) {\n throw new Error(`Action type '${planType}' not found`);\n }\n\n const locateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema)\n : [];\n\n const requiredLocateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema, true)\n : [];\n\n locateFields.forEach((field) => {\n if (param[field]) {\n // Always use createLocateTask for all locate params.\n // This ensures cache writing happens even when locatedPixelBbox is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n executor: async (param, taskContext) => {\n const timing = taskContext.task.timing;\n\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug(\n `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.beforeInvokeAction(action.name, param);\n debug(\n `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n })(),\n delayBeforeRunner > 0\n ? sleep(delayBeforeRunner)\n : Promise.resolve(),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in Action task',\n );\n }\n\n if (action.paramSchema) {\n try {\n param = parseActionParam(param, action.paramSchema, {\n shrunkShotToLogicalRatio,\n });\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n }\n\n setTimingFieldOnce(timing, 'callActionStart');\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(param, taskContext);\n setTimingFieldOnce(timing, 'callActionEnd');\n debug('called action', action.name, 'result:', actionResult);\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n const delayAfterRunner =\n action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug(\n `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n await this.interface.afterInvokeAction(action.name, param);\n debug(\n `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n );\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message \|\| String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<PlanningLocateParam>,\n detailedLocateParam: DetailedLocateParam \| string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n let locateParam = detailedLocateParam;\n\n if (typeof locateParam === 'string') {\n locateParam = {\n prompt: locateParam,\n };\n }\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n if (deepLocate && !locateParam.deepLocate) {\n locateParam = {\n ...locateParam,\n deepLocate: true,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n param: locateParam,\n thought: plan.thought,\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(\n param,\n )\n ? param\n : undefined;\n\n assert(\n param?.prompt \|\| paramWithLocatedPixelBbox,\n `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n param,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n const { shrunkShotToLogicalRatio } = uiContext;\n\n if (shrunkShotToLogicalRatio === undefined) {\n throw new Error(\n 'shrunkShotToLogicalRatio is not defined in locate task',\n );\n }\n\n let locateDump: ServiceDump \| undefined;\n let locateResult: LocateResultWithDump \| undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n if (dump.taskInfo?.searchAreaUsage) {\n task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n }\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n const planLocatedElement = paramWithLocatedPixelBbox\n ? matchElementFromPlan(paramWithLocatedPixelBbox)\n : undefined;\n\n // from locatedPixelBbox (direct plan hit)\n // when deepLocate is enabled, locatedPixelBbox should be used as search\n // area hint, not as a final direct hit\n const elementFromPlan = param.deepLocate\n ? undefined\n : planLocatedElement;\n const isPlanDirectHit = !!elementFromPlan;\n\n // from xpath\n let rectFromXpath: Rect \| undefined;\n if (\n !isPlanDirectHit &&\n param.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [param.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n\n const elementFromXpath = rectFromXpath\n ? generateElementByRect(\n // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n transformLogicalRectToScreenshotRect(\n rectFromXpath,\n shrunkShotToLogicalRatio,\n ),\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt \|\| '',\n )\n : undefined;\n\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = param.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCacheResult =\n isPlanDirectHit \|\| isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n param.cacheable,\n );\n\n // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n const elementFromCache = elementFromCacheResult\n ? transformLogicalElementToScreenshot(\n elementFromCacheResult,\n shrunkShotToLogicalRatio,\n )\n : undefined;\n\n const isCacheHit = !!elementFromCache;\n\n let elementFromAiLocate: LocateResultElement \| null \| undefined;\n const timing = taskContext.task.timing;\n if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n locateResult = await this.service.locate(\n param,\n {\n context: uiContext,\n planLocatedElement,\n },\n defaultModel,\n abortSignal,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n }\n\n const element =\n elementFromPlan \|\|\n elementFromXpath \|\|\n elementFromCache \|\|\n elementFromAiLocate;\n\n if (element) {\n const invalidElementReason = invalidLocateElementReason(element);\n if (invalidElementReason) {\n if (locateDump) {\n throw new ServiceError(invalidElementReason, locateDump);\n }\n throw new Error(invalidElementReason);\n }\n }\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature \| undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read)\n // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n !isCacheHit &&\n (!isPlanDirectHit \|\| !locateCacheAlreadyExists) &&\n param?.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForPoint) {\n try {\n // Transform coordinates to logical space for cacheFeatureForPoint\n // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n let pointForCache: [number, number] = element.center;\n if (shrunkShotToLogicalRatio !== 1) {\n pointForCache = [\n Math.round(element.center[0] / shrunkShotToLogicalRatio),\n Math.round(element.center[1] / shrunkShotToLogicalRatio),\n ];\n debug(\n 'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n element.center,\n pointForCache,\n );\n }\n\n const feature = await this.interface.cacheFeatureForPoint(\n pointForCache,\n {\n targetDescription:\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt,\n modelRuntime: defaultModel,\n },\n );\n if (hasNonEmptyCache(feature)) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForPoint failed: %s', error);\n }\n } else {\n debug('cacheFeatureForPoint is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${param.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${param.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy \| undefined;\n\n if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n hitBy = {\n from: 'Plan',\n context: {\n locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: param.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n onResult?.(element);\n\n return {\n output: {\n element: {\n ...element,\n // backward compatibility for aiLocate, which return value needs a dpr field\n dpr: uiContext.deprecatedDpr,\n },\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","shrunkShotToLogicalRatio","undefined","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","deepLocate","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;AAoCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAEO,SAASK,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMlB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACO,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI1B,KAAK,CAAC0B,MAAM,EAAE;gBAGhB,MAAMxB,aAAaH,oBAAoBC,KAAK,CAAC0B,MAAM;gBACnDtC,MACE,uCACA,CAAC,YAAY,EAAE8B,UAAU,EACzB,CAAC,MAAM,EAAEpB,KAAK,SAAS,CAACE,KAAK,CAAC0B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAE5B,KAAK,SAAS,CAACI,aAAa,EAC1C,CAAC,oBAAoB,EAAEyB,qCAAqC3B,KAAK,CAAC0B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtC1B,YACAF,KAAK,CAAC0B,MAAM,EACZhB,SACA,CAACmB;oBACC7B,KAAK,CAAC0B,MAAM,GAAGG;gBACjB;gBAEFnB,QAAQ,KAAK,CAAC,IAAI,CAACkB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3E9B,MAAM,CAAC,OAAO,EAAEsC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMa,OAKF;YACF,MAAM;YACN,SAASb;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOb,OAAOgC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtC5C,MACE,oBACA8B,UACAlB,OACA,CAAC,4BAA4B,EAAEgC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACE9B,KAAK,CAAC0B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAiB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBhB,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMiB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCjD,MACE,CAAC,8DAA8D,EAAEgC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEpB;gCACrDZ,MACE,CAAC,2DAA2D,EAAEgC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAgB,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAES,wBAAwB,EAAE,GAAGR;gBACrC,IAAIQ,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIF,OAAO,WAAW,EACpB,IAAI;oBACFpB,QAAQ4C,iBAAiB5C,OAAOoB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOG,OAAY;oBACnB,MAAM,IAAIvB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEyB,MAAM,OAAO,CAAC,cAAc,EAAE/C,KAAK,SAAS,CAACE,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFV,mBAAmBF,QAAQ;gBAE3B7C,MAAM,kBAAkBgC,OAAO,IAAI;gBACnC,MAAM0B,WAAW1B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM2B,eAAe,MAAMD,SAAS9C,OAAOgC;gBAC3CG,mBAAmBF,QAAQ;gBAC3B7C,MAAM,iBAAiBgC,OAAO,IAAI,EAAE,WAAW2B;gBAE/CZ,mBAAmBF,QAAQ;gBAE3B,MAAMe,mBACJ5B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI4B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpC5D,MACE,CAAC,6DAA6D,EAAEgC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEpB;wBACpDZ,MACE,CAAC,0DAA0D,EAAEgC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOmB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIjB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQc;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAACqB;IACrB;IAEQ,iBACNlB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,YAAY,EAAE6C,UAAU,EAAEC,WAAW,EAAE,GAAG1C;QAE7D,IAAI2C,cAAcJ;QAElB,IAAI,AAAuB,YAAvB,OAAOI,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAckC,WAAdlC,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,IAAI0C,cAAc,CAACE,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOgC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMuB,4BAA4B5B,qCAChC3B,SAEEA,QACA2C;gBAEJb,OACE9B,OAAO,UAAUuD,2BACjB,CAAC,iEAAiE,EAAEzD,KAAK,SAAS,CAChFE,QACC;gBAGL,IAAI,CAACkC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAEQ,wBAAwB,EAAE,GAAGR;gBAErC,IAAIQ,AAA6BC,WAA7BD,0BACF,MAAM,IAAIpB,MACR;gBAIJ,IAAIkC;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb5B,KAAK,GAAG,GAAG;wBACT4B;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA5B,KAAK,KAAK,GAAG6B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,iBACjB5B,KAAK,eAAe,GAAG4B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB5B,KAAK,iBAAiB,GAAG4B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,qBAAqBP,6BACrBZ;gBAKJ,MAAMoB,kBAAkB/D,MAAM,UAAU,GACpC2C,SACAkB;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDhE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFiE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACjE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMkE,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACAvB,2BAEF,AAAwB,YAAxB,OAAO1C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9B2C;gBAEJ,MAAM0B,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAActE,MAAM,MAAM;gBAChC,MAAMuE,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAtE,MAAM,SAAS;gBAIvB,MAAM2E,mBAAmBF,yBACrBG,oCACEH,wBACA/B,4BAEFC;gBAEJ,MAAMkC,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM7C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACqC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF7B,mBAAmBF,QAAQ;oBAC3BwB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCzD,OACA;wBACE,SAASkC;wBACT2B;oBACF,GACAvD,cACA8C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOZ,OAAO;oBACd,IAAIA,iBAAiBkC,cACnBrB,UAAUb,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRV,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAMvC,UACJqE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIpF,SAAS;oBACX,MAAMsF,uBAAuBvF,2BAA2BC;oBACxD,IAAIsF,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,aAAaC,sBAAsBxB;wBAE/C,MAAM,IAAIlC,MAAM0D;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B3F,iBAC/BiF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACExF,WACA,IAAI,CAAC,SAAS,IACd,CAACmF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7CjF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAImF,gBAAkCzF,QAAQ,MAAM;oBACpD,IAAIgD,AAA6B,MAA7BA,0BAAgC;wBAClCyC,gBAAgB;4BACdC,KAAK,KAAK,CAAC1F,QAAQ,MAAM,CAAC,EAAE,GAAGgD;4BAC/B0C,KAAK,KAAK,CAAC1F,QAAQ,MAAM,CAAC,EAAE,GAAGgD;yBAChC;wBACDtD,MACE,8DACAM,QAAQ,MAAM,EACdyF;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOnF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,cAAcM;oBAChB;oBAEF,IAAIhB,iBAAiB+F,UAAU;wBAC7BjG,MACE,uCACAkF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACEnF,MACE,yDACAkF;gBAGN,EAAE,OAAOzB,OAAO;oBACdzD,MAAM,mCAAmCyD;gBAC3C;qBAEAzD,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAI8D,YACF,MAAM,IAAIuB,aACR,CAAC,oBAAoB,EAAE/E,MAAM,MAAM,EAAE,EACrCwD;oBAGJ,MAAM,IAAIlC,MAAM,CAAC,mBAAmB,EAAEtB,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIsF;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOtF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAI6E,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGFhC,WAAWxD;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAKwC,UAAU,aAAa;wBAC9B;oBACF;oBACAoD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IAtkBA,YAAY,EACViC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTtE,WAAW,EACXuE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGtE;QACnB,IAAI,CAAC,eAAe,GAAGuE;IACzB;AA2jBF"}

package/dist/es/agent/tasks.mjs CHANGED Viewed

@@ -1,6 +1,6 @@
-import { AIResponseParseError, ConversationHistory, autoGLMPlanning, plan, uiTarsPlanning } from "../ai-model/index.mjs";
-import { isAutoGLM, isUITars } from "../ai-model/auto-glm/util.mjs";
+import { AIResponseParseError, ConversationHistory } from "../ai-model/index.mjs";
 import { buildTypeQueryDemandValue } from "../ai-model/prompt/extraction.mjs";
+import { genericXmlPlan } from "../ai-model/workflows/planning/index.mjs";
 import { getReadableTimeString } from "../common.mjs";
 import { TaskExecutionError } from "../task-runner.mjs";
 import { ServiceError } from "../types.mjs";
@@ -51,8 +51,8 @@ class TaskExecutor {
         else warnLog('useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.');
         return getReadableTimeString(format);
     }
-    async convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options) {
-        return this.taskBuilder.build(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options);
+    async convertPlanToExecutable(plans, planningModel, defaultModel, options) {
+        return this.taskBuilder.build(plans, planningModel, defaultModel, options);
     }
     async loadYamlFlowAsPlanning(userInstruction, yamlString) {
         const session = this.createExecutionSession(taskTitleStr('Act', userInstruction));
@@ -90,9 +90,9 @@ class TaskExecutor {
             runner
         };
     }
-    async runPlans(title, plans, modelConfigForPlanning, modelConfigForDefaultIntent) {
+    async runPlans(title, plans, planningModel, defaultModel) {
         const session = this.createExecutionSession(title);
-        const { tasks } = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent);
+        const { tasks } = await this.convertPlanToExecutable(plans, planningModel, defaultModel);
         const runner = session.getRunner();
         const result = await session.appendAndRun(tasks);
         const { output } = result ?? {};
@@ -101,10 +101,14 @@ class TaskExecutor {
             runner
         };
     }
-    async action(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, fileChooserAccept, deepLocate, abortSignal) {
-        return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal));
+    async action(userPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, fileChooserAccept, deepLocate, abortSignal) {
+        return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal));
     }
-    async runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal) {
+    async runAction(userPrompt, planningModel, defaultModel, includeLocateInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal) {
+        if (deepLocate && !planningModel.adapter.planning.supportsActionDeepLocate) {
+            warnLog(`The "deepLocate" option is not supported for aiAct with the current planning adapter (modelFamily: ${planningModel.config.modelFamily ?? 'unknown'}). It will be ignored.`);
+            deepLocate = false;
+        }
         const conversationHistory = new ConversationHistory();
         const session = this.createExecutionSession(taskTitleStr('Act', userPrompt));
         const runner = session.getRunner();
@@ -136,24 +140,22 @@ class TaskExecutor {
                 executor: async (param, executorContext)=>{
                     const { uiContext } = executorContext;
                     assert(uiContext, 'uiContext is required for Planning task');
-                    const { modelFamily } = modelConfigForPlanning;
                     const timing = executorContext.task.timing;
                     const actionSpace = this.getActionSpace();
                     debug('actionSpace for this interface is:', actionSpace.map((action)=>action.name).join(', '));
                     assert(Array.isArray(actionSpace), 'actionSpace must be an array');
                     if (0 === actionSpace.length) console.warn(`ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`);
-                    const planImpl = isUITars(modelFamily) ? uiTarsPlanning : isAutoGLM(modelFamily) ? autoGLMPlanning : plan;
+                    const planImpl = 'custom' === planningModel.adapter.planning.kind ? planningModel.adapter.planning.planFn : genericXmlPlan;
                     let planResult;
                     try {
                         setTimingFieldOnce(timing, 'callAiStart');
                         planResult = await planImpl(param.userInstruction, {
                             context: uiContext,
                             actionContext: param.aiActContext,
-                            interfaceType: this.interface.interfaceType,
                             actionSpace,
-                            modelConfig: modelConfigForPlanning,
+                            modelRuntime: planningModel,
                             conversationHistory,
-                            includeBbox: includeBboxInPlanning,
+                            includeLocateInPlanning,
                             imagesIncludeCount,
                             deepThink,
                             abortSignal
@@ -207,7 +209,7 @@ class TaskExecutor {
             yamlFlow.push(...planResult?.yamlFlow || []);
             let executables;
             try {
-                executables = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, {
+                executables = await this.convertPlanToExecutable(plans, planningModel, defaultModel, {
                     cacheable,
                     deepLocate,
                     abortSignal
@@ -247,7 +249,7 @@ class TaskExecutor {
             runner
         };
     }
-    createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt) {
+    createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt) {
         const queryTask = {
             type: 'Insight',
             subType: type,
@@ -294,7 +296,7 @@ class TaskExecutor {
                     extraPageDescription = await descriptionOfTree(tree, 200, false, opt?.domIncluded === 'visible-only');
                 }
                 try {
-                    extractResult = await this.service.extract(demandInput, modelConfig, opt, extraPageDescription, multimodalPrompt, uiContext);
+                    extractResult = await this.service.extract(demandInput, modelRuntime, opt, extraPageDescription, multimodalPrompt, uiContext);
                 } catch (error) {
                     if (error instanceof ServiceError) applyDump(error.dump);
                     throw error;
@@ -321,9 +323,9 @@ class TaskExecutor {
         };
         return queryTask;
     }
-    async createTypeQueryExecution(type, demand, modelConfig, opt, multimodalPrompt) {
+    async createTypeQueryExecution(type, demand, modelRuntime, opt, multimodalPrompt) {
         const session = this.createExecutionSession(taskTitleStr(type, 'string' == typeof demand ? demand : JSON.stringify(demand)));
-        const queryTask = await this.createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt);
+        const queryTask = await this.createTypeQueryTask(type, demand, modelRuntime, opt, multimodalPrompt);
         const runner = session.getRunner();
         const result = await session.appendAndRun(queryTask);
         if (!result) throw new Error('result of taskExecutor.flush() is undefined in function createTypeQueryTask');
@@ -334,7 +336,7 @@ class TaskExecutor {
             runner
         };
     }
-    async waitFor(assertion, opt, modelConfig) {
+    async waitFor(assertion, opt, modelRuntime) {
         const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
         const description = `waitFor: ${textPrompt}`;
         const session = this.createExecutionSession(taskTitleStr('WaitFor', description));
@@ -355,7 +357,7 @@ class TaskExecutor {
         while(lastCheckStart - overallStartTime <= timeoutMs){
             const currentCheckStart = Date.now();
             lastCheckStart = currentCheckStart;
-            const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, modelConfig, serviceExtractOpt, multimodalPrompt);
+            const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, modelRuntime, serviceExtractOpt, multimodalPrompt);
             const result = await session.appendAndRun(queryTask);
             if (result?.output) return {
                 output: void 0,
@@ -375,7 +377,7 @@ class TaskExecutor {
                         },
                         thought
                     }
-                ], modelConfig, modelConfig);
+                ], modelRuntime, modelRuntime);
                 if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
             }
         }