@midscene/core 0.30.10 → 1.0.0

package/dist/lib/agent/agent.js

@@ -45,6 +45,7 @@ const env_namespaceObject = require("@midscene/shared/env");
 const img_namespaceObject = require("@midscene/shared/img");
 const logger_namespaceObject = require("@midscene/shared/logger");
 const utils_namespaceObject = require("@midscene/shared/utils");
+const external_device_index_js_namespaceObject = require("../device/index.js");
 const external_task_cache_js_namespaceObject = require("./task-cache.js");
 const external_tasks_js_namespaceObject = require("./tasks.js");
 const external_ui_utils_js_namespaceObject = require("./ui-utils.js");
@@ -70,7 +71,7 @@ const includedInRect = (point, rect)=>{
     const { left, top, width, height } = rect;
     return x >= left && x <= left + width && y >= top && y <= top + height;
 };
-const defaultInsightExtractOption = {
+const defaultServiceExtractOption = {
     domIncluded: false,
     screenshotIncluded: true
 };
@@ -81,7 +82,31 @@ const CACHE_STRATEGIES = [
 ];
 const isValidCacheStrategy = (strategy)=>CACHE_STRATEGIES.some((value)=>value === strategy);
 const CACHE_STRATEGY_VALUES = CACHE_STRATEGIES.map((value)=>`"${value}"`).join(', ');
+const legacyScrollTypeMap = {
+    once: 'singleAction',
+    untilBottom: 'scrollToBottom',
+    untilTop: 'scrollToTop',
+    untilRight: 'scrollToRight',
+    untilLeft: 'scrollToLeft'
+};
+const normalizeScrollType = (scrollType)=>{
+    if (!scrollType) return scrollType;
+    if (scrollType in legacyScrollTypeMap) return legacyScrollTypeMap[scrollType];
+    return scrollType;
+};
+const defaultReplanningCycleLimit = 20;
+const defaultVlmUiTarsReplanningCycleLimit = 40;
 class Agent {
+    get onDumpUpdate() {
+        return this.dumpUpdateListeners[0];
+    }
+    set onDumpUpdate(callback) {
+        this.dumpUpdateListeners = [];
+        if (callback) this.dumpUpdateListeners.push(callback);
+    }
+    get aiActContext() {
+        return this.opts.aiActContext ?? this.opts.aiActionContext;
+    }
     get page() {
         return this.interface;
     }
@@ -94,10 +119,11 @@ class Agent {
     async getScreenshotScale(context) {
         if (void 0 !== this.screenshotScale) return this.screenshotScale;
         if (!this.screenshotScalePromise) this.screenshotScalePromise = (async ()=>{
-            var _context_size;
-            const pageWidth = null == (_context_size = context.size) ? void 0 : _context_size.width;
+            const pageWidth = context.size?.width;
             (0, utils_namespaceObject.assert)(pageWidth && pageWidth > 0, `Invalid page width when computing screenshot scale: ${pageWidth}`);
+            debug('will get image info of base64');
             const { width: screenshotWidth } = await (0, img_namespaceObject.imageInfoOfBase64)(context.screenshotBase64);
+            debug('image info of base64 done');
             (0, utils_namespaceObject.assert)(Number.isFinite(screenshotWidth) && screenshotWidth > 0, `Invalid screenshot width when computing screenshot scale: ${screenshotWidth}`);
             const computedScale = screenshotWidth / pageWidth;
             (0, utils_namespaceObject.assert)(Number.isFinite(computedScale) && computedScale > 0, `Invalid computed screenshot scale: ${computedScale}`);
@@ -111,8 +137,16 @@ class Agent {
             this.screenshotScalePromise = void 0;
         }
     }
+    resolveReplanningCycleLimit(modelConfigForPlanning) {
+        if (void 0 !== this.opts.replanningCycleLimit) return this.opts.replanningCycleLimit;
+        return 'vlm-ui-tars' === modelConfigForPlanning.vlMode ? defaultVlmUiTarsReplanningCycleLimit : defaultReplanningCycleLimit;
+    }
     async getActionSpace() {
-        return this.interface.actionSpace();
+        const commonAssertionAction = (0, external_device_index_js_namespaceObject.defineActionAssert)();
+        return [
+            ...this.interface.actionSpace(),
+            commonAssertionAction
+        ];
     }
     async getUIContext(action) {
         this.ensureVLModelWarning();
@@ -125,12 +159,14 @@ class Agent {
             debug('Using page.getContext for action:', action);
             context = await this.interface.getContext();
         } else {
-            debug('Using commonContextParser for action:', action);
+            debug('Using commonContextParser');
             context = await (0, external_utils_js_namespaceObject_1.commonContextParser)(this.interface, {
                 uploadServerUrl: this.modelConfigManager.getUploadTestServerUrl()
             });
         }
+        debug('will get screenshot scale');
         const computedScreenshotScale = await this.getScreenshotScale(context);
+        debug('computedScreenshotScale', computedScreenshotScale);
         if (1 !== computedScreenshotScale) {
             const scaleForLog = Number.parseFloat(computedScreenshotScale.toFixed(4));
             debug(`Applying computed screenshot scale: ${scaleForLog} (resize to logical size)`);
@@ -148,7 +184,11 @@ class Agent {
         return await this.getUIContext('locate');
     }
     async setAIActionContext(prompt) {
-        if (this.opts.aiActionContext) console.warn('aiActionContext is already set, and it is called again, will override the previous setting');
+        await this.setAIActContext(prompt);
+    }
+    async setAIActContext(prompt) {
+        if (this.aiActContext) console.warn('aiActContext is already set, and it is called again, will override the previous setting');
+        this.opts.aiActContext = prompt;
         this.opts.aiActionContext = prompt;
     }
     resetDump() {
@@ -159,12 +199,22 @@ class Agent {
             executions: [],
             modelBriefs: []
         };
+        this.executionDumpIndexByRunner = new WeakMap();
         return this.dump;
     }
-    appendExecutionDump(execution) {
-        const trimmedExecution = (0, external_utils_js_namespaceObject_1.trimContextByViewport)(execution);
+    appendExecutionDump(execution, runner) {
         const currentDump = this.dump;
-        currentDump.executions.push(trimmedExecution);
+        if (runner) {
+            const existingIndex = this.executionDumpIndexByRunner.get(runner);
+            if (void 0 !== existingIndex) {
+                currentDump.executions[existingIndex] = execution;
+                return;
+            }
+            currentDump.executions.push(execution);
+            this.executionDumpIndexByRunner.set(runner, currentDump.executions.length - 1);
+            return;
+        }
+        currentDump.executions.push(execution);
     }
     dumpDataString() {
         this.dump.groupName = this.opts.groupName;
@@ -192,22 +242,8 @@ class Agent {
         const tip = param ? `${(0, external_ui_utils_js_namespaceObject.typeStr)(task)} - ${param}` : (0, external_ui_utils_js_namespaceObject.typeStr)(task);
         if (this.onTaskStartTip) await this.onTaskStartTip(tip);
     }
-    async afterTaskRunning(executor, doNotThrowError = false) {
-        const executionDump = executor.dump();
-        if (this.opts.aiActionContext) executionDump.aiActionContext = this.opts.aiActionContext;
-        this.appendExecutionDump(executionDump);
-        try {
-            if (this.onDumpUpdate) this.onDumpUpdate(this.dumpDataString());
-        } catch (error) {
-            console.error('Error in onDumpUpdate', error);
-        }
-        this.writeOutActionDumps();
-        if (executor.isInErrorState() && !doNotThrowError) {
-            const errorTask = executor.latestErrorTask();
-            throw new Error(`${null == errorTask ? void 0 : errorTask.errorMessage}\n${null == errorTask ? void 0 : errorTask.errorStack}`, {
-                cause: null == errorTask ? void 0 : errorTask.error
-            });
-        }
+    wrapActionInActionSpace(name) {
+        return async (param)=>await this.callActionInActionSpace(name, param);
     }
     async callActionInActionSpace(type, opt) {
         debug('callActionInActionSpace', type, ',', opt);
@@ -220,10 +256,10 @@ class Agent {
         const plans = [
             actionPlan
         ].filter(Boolean);
-        const title = (0, external_ui_utils_js_namespaceObject.taskTitleStr)(type, (0, external_ui_utils_js_namespaceObject.locateParamStr)((null == opt ? void 0 : opt.locate) || {}));
-        const modelConfig = this.modelConfigManager.getModelConfig('grounding');
-        const { output, executor } = await this.taskExecutor.runPlans(title, plans, modelConfig);
-        await this.afterTaskRunning(executor);
+        const title = (0, external_ui_utils_js_namespaceObject.taskTitleStr)(type, (0, external_ui_utils_js_namespaceObject.locateParamStr)(opt?.locate || {}));
+        const defaultIntentModelConfig = this.modelConfigManager.getModelConfig('default');
+        const modelConfigForPlanning = this.modelConfigManager.getModelConfig('planning');
+        const { output } = await this.taskExecutor.runPlans(title, plans, modelConfigForPlanning, defaultIntentModelConfig);
         return output;
     }
     async aiTap(locatePrompt, opt) {
@@ -274,8 +310,10 @@ class Agent {
         (0, utils_namespaceObject.assert)('string' == typeof value || 'number' == typeof value, 'input value must be a string or number, use empty string if you want to clear the input');
         (0, utils_namespaceObject.assert)(locatePrompt, 'missing locate prompt for input');
         const detailedLocateParam = (0, index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt);
+        const stringValue = 'number' == typeof value ? String(value) : value;
         return this.callActionInActionSpace('Input', {
             ...opt || {},
+            value: stringValue,
             locate: detailedLocateParam
         });
     }
@@ -294,7 +332,7 @@ class Agent {
                 keyName
             };
         }
-        (0, utils_namespaceObject.assert)(null == opt ? void 0 : opt.keyName, 'missing keyName for keyboard press');
+        (0, utils_namespaceObject.assert)(opt?.keyName, 'missing keyName for keyboard press');
         const detailedLocateParam = locatePrompt ? (0, index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt, opt) : void 0;
         return this.callActionInActionSpace('KeyboardPress', {
             ...opt || {},
@@ -316,28 +354,45 @@ class Agent {
                 ...scrollParam || {}
             };
         }
+        if (opt) {
+            const normalizedScrollType = normalizeScrollType(opt.scrollType);
+            if (normalizedScrollType !== opt.scrollType) opt = {
+                ...opt || {},
+                scrollType: normalizedScrollType
+            };
+        }
         const detailedLocateParam = (0, index_js_namespaceObject.buildDetailedLocateParam)(locatePrompt || '', opt);
         return this.callActionInActionSpace('Scroll', {
             ...opt || {},
             locate: detailedLocateParam
         });
     }
-    async aiAction(taskPrompt, opt) {
-        var _this_taskCache, _this_taskCache1;
-        const modelConfig = this.modelConfigManager.getModelConfig('planning');
-        const cacheable = null == opt ? void 0 : opt.cacheable;
-        const isVlmUiTars = 'vlm-ui-tars' === modelConfig.vlMode;
-        const matchedCache = isVlmUiTars || false === cacheable ? void 0 : null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchPlanCache(taskPrompt);
-        if (matchedCache && (null == (_this_taskCache1 = this.taskCache) ? void 0 : _this_taskCache1.isCacheResultUsed)) {
-            var _matchedCache_cacheContent, _matchedCache_cacheContent1;
-            const { executor } = await this.taskExecutor.loadYamlFlowAsPlanning(taskPrompt, null == (_matchedCache_cacheContent = matchedCache.cacheContent) ? void 0 : _matchedCache_cacheContent.yamlWorkflow);
-            await this.afterTaskRunning(executor);
+    async aiAct(taskPrompt, opt) {
+        const modelConfigForPlanning = this.modelConfigManager.getModelConfig('planning');
+        const defaultIntentModelConfig = this.modelConfigManager.getModelConfig('default');
+        let planningStrategyToUse = opt?.planningStrategy || 'standard';
+        if (this.aiActContext && 'fast' === planningStrategyToUse) console.warn('using fast planning strategy with aiActContext is not recommended');
+        if (this.opts?._deepThink) {
+            debug('using deep think planning strategy');
+            planningStrategyToUse = 'max';
+        }
+        const includeBboxInPlanning = modelConfigForPlanning.modelName === defaultIntentModelConfig.modelName && 'fast' === planningStrategyToUse;
+        debug('setting includeBboxInPlanning to', includeBboxInPlanning);
+        const cacheable = opt?.cacheable;
+        const replanningCycleLimit = this.resolveReplanningCycleLimit(modelConfigForPlanning);
+        const isVlmUiTars = 'vlm-ui-tars' === modelConfigForPlanning.vlMode;
+        const matchedCache = isVlmUiTars || false === cacheable ? void 0 : this.taskCache?.matchPlanCache(taskPrompt);
+        if (matchedCache && this.taskCache?.isCacheResultUsed && matchedCache.cacheContent?.yamlWorkflow?.trim()) {
+            await this.taskExecutor.loadYamlFlowAsPlanning(taskPrompt, matchedCache.cacheContent.yamlWorkflow);
             debug('matched cache, will call .runYaml to run the action');
-            const yaml = null == (_matchedCache_cacheContent1 = matchedCache.cacheContent) ? void 0 : _matchedCache_cacheContent1.yamlWorkflow;
+            const yaml = matchedCache.cacheContent.yamlWorkflow;
             return this.runYaml(yaml);
         }
-        const { output, executor } = await this.taskExecutor.action(taskPrompt, modelConfig, this.opts.aiActionContext, cacheable);
-        if (this.taskCache && (null == output ? void 0 : output.yamlFlow) && false !== cacheable) {
+        let imagesIncludeCount = 1;
+        if ('standard' === planningStrategyToUse) imagesIncludeCount = 2;
+        else if ('max' === planningStrategyToUse) imagesIncludeCount = void 0;
+        const { output } = await this.taskExecutor.action(taskPrompt, modelConfigForPlanning, defaultIntentModelConfig, includeBboxInPlanning, this.aiActContext, cacheable, replanningCycleLimit, imagesIncludeCount);
+        if (this.taskCache && output?.yamlFlow && false !== cacheable) {
             const yamlContent = {
                 tasks: [
                     {
@@ -353,37 +408,35 @@ class Agent {
                 yamlWorkflow: yamlFlowStr
             }, matchedCache);
         }
-        await this.afterTaskRunning(executor);
         return output;
     }
-    async aiQuery(demand, opt = defaultInsightExtractOption) {
-        const modelConfig = this.modelConfigManager.getModelConfig('VQA');
-        const { output, executor } = await this.taskExecutor.createTypeQueryExecution('Query', demand, modelConfig, opt);
-        await this.afterTaskRunning(executor);
+    async aiAction(taskPrompt, opt) {
+        return this.aiAct(taskPrompt, opt);
+    }
+    async aiQuery(demand, opt = defaultServiceExtractOption) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
+        const { output } = await this.taskExecutor.createTypeQueryExecution('Query', demand, modelConfig, opt);
         return output;
     }
-    async aiBoolean(prompt, opt = defaultInsightExtractOption) {
-        const modelConfig = this.modelConfigManager.getModelConfig('VQA');
+    async aiBoolean(prompt, opt = defaultServiceExtractOption) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
         const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
-        const { output, executor } = await this.taskExecutor.createTypeQueryExecution('Boolean', textPrompt, modelConfig, opt, multimodalPrompt);
-        await this.afterTaskRunning(executor);
+        const { output } = await this.taskExecutor.createTypeQueryExecution('Boolean', textPrompt, modelConfig, opt, multimodalPrompt);
         return output;
     }
-    async aiNumber(prompt, opt = defaultInsightExtractOption) {
-        const modelConfig = this.modelConfigManager.getModelConfig('VQA');
+    async aiNumber(prompt, opt = defaultServiceExtractOption) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
         const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
-        const { output, executor } = await this.taskExecutor.createTypeQueryExecution('Number', textPrompt, modelConfig, opt, multimodalPrompt);
-        await this.afterTaskRunning(executor);
+        const { output } = await this.taskExecutor.createTypeQueryExecution('Number', textPrompt, modelConfig, opt, multimodalPrompt);
         return output;
     }
-    async aiString(prompt, opt = defaultInsightExtractOption) {
-        const modelConfig = this.modelConfigManager.getModelConfig('VQA');
+    async aiString(prompt, opt = defaultServiceExtractOption) {
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
         const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(prompt);
-        const { output, executor } = await this.taskExecutor.createTypeQueryExecution('String', textPrompt, modelConfig, opt, multimodalPrompt);
-        await this.afterTaskRunning(executor);
+        const { output } = await this.taskExecutor.createTypeQueryExecution('String', textPrompt, modelConfig, opt, multimodalPrompt);
         return output;
     }
-    async aiAsk(prompt, opt = defaultInsightExtractOption) {
+    async aiAsk(prompt, opt = defaultServiceExtractOption) {
         return this.aiString(prompt, opt);
     }
     async describeElementAtPoint(center, opt) {
@@ -391,13 +444,13 @@ class Agent {
         let success = false;
         let retryCount = 0;
         let resultPrompt = '';
-        let deepThink = (null == opt ? void 0 : opt.deepThink) || false;
+        let deepThink = opt?.deepThink || false;
         let verifyResult;
         while(!success && retryCount < retryLimit){
             if (retryCount >= 2) deepThink = true;
             debug('aiDescribe', center, 'verifyPrompt', verifyPrompt, 'retryCount', retryCount, 'deepThink', deepThink);
-            const modelConfig = this.modelConfigManager.getModelConfig('grounding');
-            const text = await this.insight.describe(center, modelConfig, {
+            const modelConfig = this.modelConfigManager.getModelConfig('insight');
+            const text = await this.service.describe(center, modelConfig, {
                 deepThink
             });
             debug('aiDescribe text', text);
@@ -420,7 +473,7 @@ class Agent {
         const { center: verifyCenter, rect: verifyRect } = await this.aiLocate(prompt, locateOpt);
         const distance = distanceOfTwoPoints(expectCenter, verifyCenter);
         const included = includedInRect(expectCenter, verifyRect);
-        const pass = distance <= ((null == verifyLocateOption ? void 0 : verifyLocateOption.centerDistanceThreshold) || 20) || included;
+        const pass = distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;
         const verifyResult = {
             pass,
             rect: verifyRect,
@@ -437,59 +490,67 @@ class Agent {
         const plans = [
             locatePlan
         ];
-        const modelConfig = this.modelConfigManager.getModelConfig('grounding');
-        const { executor, output } = await this.taskExecutor.runPlans((0, external_ui_utils_js_namespaceObject.taskTitleStr)('Locate', (0, external_ui_utils_js_namespaceObject.locateParamStr)(locateParam)), plans, modelConfig);
-        await this.afterTaskRunning(executor);
+        const defaultIntentModelConfig = this.modelConfigManager.getModelConfig('default');
+        const modelConfigForPlanning = this.modelConfigManager.getModelConfig('planning');
+        const { output } = await this.taskExecutor.runPlans((0, external_ui_utils_js_namespaceObject.taskTitleStr)('Locate', (0, external_ui_utils_js_namespaceObject.locateParamStr)(locateParam)), plans, modelConfigForPlanning, defaultIntentModelConfig);
         const { element } = output;
         const dprValue = await this.interface.size().dpr;
         const dprEntry = dprValue ? {
             dpr: dprValue
         } : {};
         return {
-            rect: null == element ? void 0 : element.rect,
-            center: null == element ? void 0 : element.center,
+            rect: element?.rect,
+            center: element?.center,
             ...dprEntry
         };
     }
     async aiAssert(assertion, msg, opt) {
-        var _executor_latestErrorTask;
-        const modelConfig = this.modelConfigManager.getModelConfig('VQA');
-        const insightOpt = {
-            domIncluded: (null == opt ? void 0 : opt.domIncluded) ?? defaultInsightExtractOption.domIncluded,
-            screenshotIncluded: (null == opt ? void 0 : opt.screenshotIncluded) ?? defaultInsightExtractOption.screenshotIncluded,
-            doNotThrowError: null == opt ? void 0 : opt.doNotThrowError
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
+        const serviceOpt = {
+            domIncluded: opt?.domIncluded ?? defaultServiceExtractOption.domIncluded,
+            screenshotIncluded: opt?.screenshotIncluded ?? defaultServiceExtractOption.screenshotIncluded
         };
         const { textPrompt, multimodalPrompt } = (0, external_utils_js_namespaceObject_1.parsePrompt)(assertion);
-        const { output, executor, thought } = await this.taskExecutor.createTypeQueryExecution('Assert', textPrompt, modelConfig, insightOpt, multimodalPrompt);
-        await this.afterTaskRunning(executor, true);
-        const message = output ? void 0 : `Assertion failed: ${msg || ('string' == typeof assertion ? assertion : assertion.prompt)}\nReason: ${thought || (null == (_executor_latestErrorTask = executor.latestErrorTask()) ? void 0 : _executor_latestErrorTask.error) || '(no_reason)'}`;
-        if (null == opt ? void 0 : opt.keepRawResponse) return {
-            pass: output,
-            thought,
-            message
-        };
-        if (!output) throw new Error(message);
+        const assertionText = 'string' == typeof assertion ? assertion : assertion.prompt;
+        try {
+            const { output, thought } = await this.taskExecutor.createTypeQueryExecution('Assert', textPrompt, modelConfig, serviceOpt, multimodalPrompt);
+            const pass = Boolean(output);
+            const message = pass ? void 0 : `Assertion failed: ${msg || assertionText}\nReason: ${thought || '(no_reason)'}`;
+            if (opt?.keepRawResponse) return {
+                pass,
+                thought,
+                message
+            };
+            if (!pass) throw new Error(message);
+        } catch (error) {
+            if (error instanceof external_tasks_js_namespaceObject.TaskExecutionError) {
+                const errorTask = error.errorTask;
+                const thought = errorTask?.thought;
+                const rawError = errorTask?.error;
+                const rawMessage = errorTask?.errorMessage || (rawError instanceof Error ? rawError.message : rawError ? String(rawError) : void 0);
+                const reason = thought || rawMessage || '(no_reason)';
+                const message = `Assertion failed: ${msg || assertionText}\nReason: ${reason}`;
+                if (opt?.keepRawResponse) return {
+                    pass: false,
+                    thought,
+                    message
+                };
+                throw new Error(message, {
+                    cause: rawError ?? error
+                });
+            }
+            throw error;
+        }
     }
     async aiWaitFor(assertion, opt) {
-        const modelConfig = this.modelConfigManager.getModelConfig('VQA');
-        const { executor } = await this.taskExecutor.waitFor(assertion, {
-            timeoutMs: (null == opt ? void 0 : opt.timeoutMs) || 15000,
-            checkIntervalMs: (null == opt ? void 0 : opt.checkIntervalMs) || 3000
+        const modelConfig = this.modelConfigManager.getModelConfig('insight');
+        await this.taskExecutor.waitFor(assertion, {
+            timeoutMs: opt?.timeoutMs || 15000,
+            checkIntervalMs: opt?.checkIntervalMs || 3000
         }, modelConfig);
-        await this.afterTaskRunning(executor, true);
-        if (executor.isInErrorState()) {
-            const errorTask = executor.latestErrorTask();
-            throw new Error(`${null == errorTask ? void 0 : errorTask.error}\n${null == errorTask ? void 0 : errorTask.errorStack}`);
-        }
     }
-    async ai(taskPrompt, type = 'action') {
-        if ('action' === type) return this.aiAction(taskPrompt);
-        if ('query' === type) return this.aiQuery(taskPrompt);
-        if ('assert' === type) return this.aiAssert(taskPrompt);
-        if ('tap' === type) return this.aiTap(taskPrompt);
-        if ('rightClick' === type) return this.aiRightClick(taskPrompt);
-        if ('doubleClick' === type) return this.aiDoubleClick(taskPrompt);
-        throw new Error(`Unknown type: ${type}, only support 'action', 'query', 'assert', 'tap', 'rightClick', 'doubleClick'`);
+    async ai(...args) {
+        return this.aiAct(...args);
     }
     async runYaml(yamlScriptContent) {
         const script = (0, index_js_namespaceObject.parseYamlScript)(yamlScriptContent, 'yaml');
@@ -499,10 +560,7 @@ class Agent {
             }));
         await player.run();
         if ('error' === player.status) {
-            const errors = player.taskStatusList.filter((task)=>'error' === task.status).map((task)=>{
-                var _task_error;
-                return `task - ${task.name}: ${null == (_task_error = task.error) ? void 0 : _task_error.message}`;
-            }).join('\n');
+            const errors = player.taskStatusList.filter((task)=>'error' === task.status).map((task)=>`task - ${task.name}: ${task.error?.message}`).join('\n');
             throw new Error(`Error(s) occurred in running yaml script:\n${errors}`);
         }
         return {
@@ -513,14 +571,26 @@ class Agent {
         (0, utils_namespaceObject.assert)(this.interface.evaluateJavaScript, 'evaluateJavaScript is not supported in current agent');
         return this.interface.evaluateJavaScript(script);
     }
+    addDumpUpdateListener(listener) {
+        this.dumpUpdateListeners.push(listener);
+        return ()=>{
+            this.removeDumpUpdateListener(listener);
+        };
+    }
+    removeDumpUpdateListener(listener) {
+        const index = this.dumpUpdateListeners.indexOf(listener);
+        if (index > -1) this.dumpUpdateListeners.splice(index, 1);
+    }
+    clearDumpUpdateListeners() {
+        this.dumpUpdateListeners = [];
+    }
     async destroy() {
-        var _this_interface_destroy, _this_interface;
         if (this.destroyed) return;
-        await (null == (_this_interface_destroy = (_this_interface = this.interface).destroy) ? void 0 : _this_interface_destroy.call(_this_interface));
+        await this.interface.destroy?.();
         this.resetDump();
         this.destroyed = true;
     }
-    async logScreenshot(title, opt) {
+    async recordToReport(title, opt) {
         const base64 = await this.interface.screenshotBase64();
         const now = Date.now();
         const recorder = [
@@ -541,48 +611,36 @@ class Agent {
                 cost: 0
             },
             param: {
-                content: (null == opt ? void 0 : opt.content) || ''
+                content: opt?.content || ''
             },
             executor: async ()=>{}
         };
         const executionDump = {
             logTime: now,
             name: `Log - ${title || 'untitled'}`,
-            description: (null == opt ? void 0 : opt.content) || '',
+            description: opt?.content || '',
             tasks: [
                 task
             ]
         };
-        if (this.opts.aiActionContext) executionDump.aiActionContext = this.opts.aiActionContext;
         this.appendExecutionDump(executionDump);
-        try {
-            var _this_onDumpUpdate, _this;
-            null == (_this_onDumpUpdate = (_this = this).onDumpUpdate) || _this_onDumpUpdate.call(_this, this.dumpDataString());
+        const dumpString = this.dumpDataString();
+        for (const listener of this.dumpUpdateListeners)try {
+            listener(dumpString);
         } catch (error) {
-            console.error('Failed to update dump', error);
+            console.error('Error in onDumpUpdate listener', error);
         }
         this.writeOutActionDumps();
     }
+    async logScreenshot(title, opt) {
+        await this.recordToReport(title, opt);
+    }
     _unstableLogContent() {
         const { groupName, groupDescription, executions } = this.dump;
-        const newExecutions = Array.isArray(executions) ? executions.map((execution)=>{
-            const { tasks, ...restExecution } = execution;
-            let newTasks = tasks;
-            if (Array.isArray(tasks)) newTasks = tasks.map((task)=>{
-                const { uiContext, log, ...restTask } = task;
-                return restTask;
-            });
-            return {
-                ...restExecution,
-                ...newTasks ? {
-                    tasks: newTasks
-                } : {}
-            };
-        }) : [];
         return {
             groupName,
             groupDescription,
-            executions: newExecutions
+            executions: executions || []
         };
     }
     async freezePageContext() {
@@ -627,7 +685,7 @@ class Agent {
     }
     constructor(interfaceInstance, opts){
         _define_property(this, "interface", void 0);
-        _define_property(this, "insight", void 0);
+        _define_property(this, "service", void 0);
         _define_property(this, "dump", void 0);
         _define_property(this, "reportFile", void 0);
         _define_property(this, "reportFileName", void 0);
@@ -636,45 +694,76 @@ class Agent {
         _define_property(this, "dryMode", false);
         _define_property(this, "onTaskStartTip", void 0);
         _define_property(this, "taskCache", void 0);
-        _define_property(this, "onDumpUpdate", void 0);
+        _define_property(this, "dumpUpdateListeners", []);
         _define_property(this, "destroyed", false);
         _define_property(this, "modelConfigManager", void 0);
         _define_property(this, "frozenUIContext", void 0);
         _define_property(this, "hasWarnedNonVLModel", false);
         _define_property(this, "screenshotScale", void 0);
         _define_property(this, "screenshotScalePromise", void 0);
+        _define_property(this, "executionDumpIndexByRunner", new WeakMap());
         this.interface = interfaceInstance;
+        const envConfig = env_namespaceObject.globalConfigManager.getAllEnvConfig();
+        const envReplanningCycleLimitRaw = envConfig[env_namespaceObject.MIDSCENE_REPLANNING_CYCLE_LIMIT];
+        const envReplanningCycleLimit = void 0 !== envReplanningCycleLimitRaw ? Number(envReplanningCycleLimitRaw) : void 0;
         this.opts = Object.assign({
             generateReport: true,
             autoPrintReportMsg: true,
             groupName: 'Midscene Report',
             groupDescription: ''
-        }, opts || {});
-        if ((null == opts ? void 0 : opts.modelConfig) && 'function' != typeof (null == opts ? void 0 : opts.modelConfig)) throw new Error(`opts.modelConfig must be one of function or undefined, but got ${typeof (null == opts ? void 0 : opts.modelConfig)}`);
-        this.modelConfigManager = (null == opts ? void 0 : opts.modelConfig) ? new env_namespaceObject.ModelConfigManager(opts.modelConfig) : env_namespaceObject.globalModelConfigManager;
+        }, opts || {}, opts?.replanningCycleLimit !== void 0 || void 0 === envReplanningCycleLimit || Number.isNaN(envReplanningCycleLimit) ? {} : {
+            replanningCycleLimit: envReplanningCycleLimit
+        });
+        const resolvedAiActContext = this.opts.aiActContext ?? this.opts.aiActionContext;
+        if (void 0 !== resolvedAiActContext) {
+            this.opts.aiActContext = resolvedAiActContext;
+            this.opts.aiActionContext ??= resolvedAiActContext;
+        }
+        if (opts?.modelConfig && ('object' != typeof opts?.modelConfig || Array.isArray(opts.modelConfig))) throw new Error(`opts.modelConfig must be a plain object map of env keys to values, but got ${typeof opts?.modelConfig}`);
+        const hasCustomConfig = opts?.modelConfig || opts?.createOpenAIClient;
+        this.modelConfigManager = hasCustomConfig ? new env_namespaceObject.ModelConfigManager(opts?.modelConfig, opts?.createOpenAIClient) : env_namespaceObject.globalModelConfigManager;
         this.onTaskStartTip = this.opts.onTaskStartTip;
-        this.insight = new external_index_js_namespaceObject.Insight(async (action)=>this.getUIContext(action));
+        this.service = new external_index_js_namespaceObject.Service(async ()=>this.getUIContext());
         const cacheConfigObj = this.processCacheConfig(opts || {});
         if (cacheConfigObj) this.taskCache = new external_task_cache_js_namespaceObject.TaskCache(cacheConfigObj.id, cacheConfigObj.enabled, void 0, {
             readOnly: cacheConfigObj.readOnly,
             writeOnly: cacheConfigObj.writeOnly
         });
-        this.taskExecutor = new external_tasks_js_namespaceObject.TaskExecutor(this.interface, this.insight, {
+        const baseActionSpace = this.interface.actionSpace();
+        const fullActionSpace = [
+            ...baseActionSpace,
+            (0, external_device_index_js_namespaceObject.defineActionAssert)()
+        ];
+        this.taskExecutor = new external_tasks_js_namespaceObject.TaskExecutor(this.interface, this.service, {
             taskCache: this.taskCache,
             onTaskStart: this.callbackOnTaskStartTip.bind(this),
-            replanningCycleLimit: this.opts.replanningCycleLimit
+            replanningCycleLimit: this.opts.replanningCycleLimit,
+            actionSpace: fullActionSpace,
+            hooks: {
+                onTaskUpdate: (runner)=>{
+                    const executionDump = runner.dump();
+                    this.appendExecutionDump(executionDump, runner);
+                    const dumpString = this.dumpDataString();
+                    for (const listener of this.dumpUpdateListeners)try {
+                        listener(dumpString, executionDump);
+                    } catch (error) {
+                        console.error('Error in onDumpUpdate listener', error);
+                    }
+                    this.writeOutActionDumps();
+                }
+            }
         });
         this.dump = this.resetDump();
-        this.reportFileName = (null == opts ? void 0 : opts.reportFileName) || (0, external_utils_js_namespaceObject_1.getReportFileName)((null == opts ? void 0 : opts.testId) || this.interface.interfaceType || 'web');
+        this.reportFileName = opts?.reportFileName || (0, external_utils_js_namespaceObject_1.getReportFileName)(opts?.testId || this.interface.interfaceType || 'web');
     }
 }
 const createAgent = (interfaceInstance, opts)=>new Agent(interfaceInstance, opts);
 exports.Agent = __webpack_exports__.Agent;
 exports.createAgent = __webpack_exports__.createAgent;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "Agent",
     "createAgent"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });