@midscene/core 0.30.10 → 1.0.0

package/dist/es/agent/tasks.mjs

@@ -1,11 +1,13 @@
-import { ConversationHistory, findAllMidsceneLocatorField, parseActionParam, plan as index_mjs_plan, uiTarsPlanning } from "../ai-model/index.mjs";
-import { Executor } from "../ai-model/action-executor.mjs";
-import { sleep as external_utils_mjs_sleep } from "../utils.mjs";
-import { MIDSCENE_REPLANNING_CYCLE_LIMIT, globalConfigManager } from "@midscene/shared/env";
+import { ConversationHistory, plan, uiTarsPlanning } from "../ai-model/index.mjs";
+import { TaskExecutionError } from "../task-runner.mjs";
+import { ServiceError } from "../types.mjs";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
+import { ExecutionSession } from "./execution-session.mjs";
+import { TaskBuilder, locatePlanForLocate } from "./task-builder.mjs";
+import { descriptionOfTree } from "@midscene/shared/extractor";
 import { taskTitleStr } from "./ui-utils.mjs";
-import { matchElementFromCache, matchElementFromPlan, parsePrompt } from "./utils.mjs";
+import { parsePrompt } from "./utils.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -17,337 +19,35 @@ function _define_property(obj, key, value) {
     return obj;
 }
 const debug = getDebug('device-task-executor');
-const defaultReplanningCycleLimit = 10;
-const defaultVlmUiTarsReplanningCycleLimit = 40;
-function locatePlanForLocate(param) {
-    const locate = 'string' == typeof param ? {
-        prompt: param
-    } : param;
-    const locatePlan = {
-        type: 'Locate',
-        locate,
-        param: locate,
-        thought: ''
-    };
-    return locatePlan;
-}
+const maxErrorCountAllowedInOnePlanningLoop = 5;
 class TaskExecutor {
     get page() {
         return this.interface;
     }
-    async recordScreenshot(timing) {
-        const base64 = await this.interface.screenshotBase64();
-        const item = {
-            type: 'screenshot',
-            ts: Date.now(),
-            screenshot: base64,
-            timing
-        };
-        return item;
-    }
-    prependExecutorWithScreenshot(taskApply, appendAfterExecution = false) {
-        const taskWithScreenshot = {
-            ...taskApply,
-            executor: async (param, context, ...args)=>{
-                const recorder = [];
-                const { task } = context;
-                task.recorder = recorder;
-                const shot = await this.recordScreenshot(`before ${task.type}`);
-                recorder.push(shot);
-                const result = await taskApply.executor(param, context, ...args);
-                if (appendAfterExecution) {
-                    const shot2 = await this.recordScreenshot('after Action');
-                    recorder.push(shot2);
-                }
-                return result;
-            }
-        };
-        return taskWithScreenshot;
-    }
-    async convertPlanToExecutable(plans, modelConfig, cacheable) {
-        const tasks = [];
-        const taskForLocatePlan = (plan, detailedLocateParam, onResult)=>{
-            if ('string' == typeof detailedLocateParam) detailedLocateParam = {
-                prompt: detailedLocateParam
-            };
-            if (void 0 !== cacheable) detailedLocateParam = {
-                ...detailedLocateParam,
-                cacheable
-            };
-            const taskFind = {
-                type: 'Insight',
-                subType: 'Locate',
-                param: detailedLocateParam,
-                thought: plan.thought,
-                executor: async (param, taskContext)=>{
-                    var _this_taskCache, _locateCacheRecord_cacheContent;
-                    const { task } = taskContext;
-                    assert((null == param ? void 0 : param.prompt) || (null == param ? void 0 : param.id) || (null == param ? void 0 : param.bbox), `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
-                    let insightDump;
-                    let usage;
-                    const dumpCollector = (dump)=>{
-                        var _dump_taskInfo, _dump_taskInfo1;
-                        insightDump = dump;
-                        usage = null == dump ? void 0 : null == (_dump_taskInfo = dump.taskInfo) ? void 0 : _dump_taskInfo.usage;
-                        task.log = {
-                            dump: insightDump
-                        };
-                        task.usage = usage;
-                        if (null == dump ? void 0 : null == (_dump_taskInfo1 = dump.taskInfo) ? void 0 : _dump_taskInfo1.searchAreaUsage) task.searchAreaUsage = dump.taskInfo.searchAreaUsage;
-                    };
-                    this.insight.onceDumpUpdatedFn = dumpCollector;
-                    const shotTime = Date.now();
-                    const uiContext = await this.insight.contextRetrieverFn('locate');
-                    task.uiContext = uiContext;
-                    const recordItem = {
-                        type: 'screenshot',
-                        ts: shotTime,
-                        screenshot: uiContext.screenshotBase64,
-                        timing: 'before Insight'
-                    };
-                    task.recorder = [
-                        recordItem
-                    ];
-                    const elementFromXpath = param.xpath && this.interface.getElementInfoByXpath ? await this.interface.getElementInfoByXpath(param.xpath) : void 0;
-                    const userExpectedPathHitFlag = !!elementFromXpath;
-                    const cachePrompt = param.prompt;
-                    const locateCacheRecord = null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchLocateCache(cachePrompt);
-                    const cacheEntry = null == locateCacheRecord ? void 0 : null == (_locateCacheRecord_cacheContent = locateCacheRecord.cacheContent) ? void 0 : _locateCacheRecord_cacheContent.cache;
-                    const elementFromCache = userExpectedPathHitFlag ? null : await matchElementFromCache(this, cacheEntry, cachePrompt, param.cacheable);
-                    const cacheHitFlag = !!elementFromCache;
-                    const elementFromPlan = userExpectedPathHitFlag || cacheHitFlag ? void 0 : matchElementFromPlan(param, uiContext.tree);
-                    const planHitFlag = !!elementFromPlan;
-                    const elementFromAiLocate = userExpectedPathHitFlag || cacheHitFlag || planHitFlag ? void 0 : (await this.insight.locate(param, {
-                        context: uiContext
-                    }, modelConfig)).element;
-                    const aiLocateHitFlag = !!elementFromAiLocate;
-                    const element = elementFromXpath || elementFromCache || elementFromPlan || elementFromAiLocate;
-                    let currentCacheEntry;
-                    if (element && this.taskCache && !cacheHitFlag && (null == param ? void 0 : param.cacheable) !== false) if (this.interface.cacheFeatureForRect) try {
-                        const feature = await this.interface.cacheFeatureForRect(element.rect, void 0 !== element.isOrderSensitive ? {
-                            _orderSensitive: element.isOrderSensitive
-                        } : void 0);
-                        if (feature && Object.keys(feature).length > 0) {
-                            debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
-                            currentCacheEntry = feature;
-                            this.taskCache.updateOrAppendCacheRecord({
-                                type: 'locate',
-                                prompt: cachePrompt,
-                                cache: feature
-                            }, locateCacheRecord);
-                        } else debug('no cache data returned, skip cache update, prompt: %s', cachePrompt);
-                    } catch (error) {
-                        debug('cacheFeatureForRect failed: %s', error);
-                    }
-                    else debug('cacheFeatureForRect is not supported, skip cache update');
-                    if (!element) throw new Error(`Element not found: ${param.prompt}`);
-                    let hitBy;
-                    if (userExpectedPathHitFlag) hitBy = {
-                        from: 'User expected path',
-                        context: {
-                            xpath: param.xpath
-                        }
-                    };
-                    else if (cacheHitFlag) hitBy = {
-                        from: 'Cache',
-                        context: {
-                            cacheEntry,
-                            cacheToSave: currentCacheEntry
-                        }
-                    };
-                    else if (planHitFlag) hitBy = {
-                        from: 'Planning',
-                        context: {
-                            id: null == elementFromPlan ? void 0 : elementFromPlan.id,
-                            bbox: null == elementFromPlan ? void 0 : elementFromPlan.bbox
-                        }
-                    };
-                    else if (aiLocateHitFlag) hitBy = {
-                        from: 'AI model',
-                        context: {
-                            prompt: param.prompt
-                        }
-                    };
-                    null == onResult || onResult(element);
-                    return {
-                        output: {
-                            element
-                        },
-                        uiContext,
-                        hitBy
-                    };
-                }
-            };
-            return taskFind;
-        };
-        for (const plan of plans)if ('Locate' === plan.type) {
-            var _plan_locate, _plan_locate1;
-            if (!plan.locate || null === plan.locate || (null == (_plan_locate = plan.locate) ? void 0 : _plan_locate.id) === null || (null == (_plan_locate1 = plan.locate) ? void 0 : _plan_locate1.id) === 'null') {
-                debug('Locate action with id is null, will be ignored', plan);
-                continue;
-            }
-            const taskLocate = taskForLocatePlan(plan, plan.locate);
-            tasks.push(taskLocate);
-        } else if ('Error' === plan.type) {
-            var _plan_param;
-            const taskActionError = {
-                type: 'Action',
-                subType: 'Error',
-                param: plan.param,
-                thought: plan.thought || (null == (_plan_param = plan.param) ? void 0 : _plan_param.thought),
-                locate: plan.locate,
-                executor: async ()=>{
-                    var _plan_param;
-                    throw new Error((null == plan ? void 0 : plan.thought) || (null == (_plan_param = plan.param) ? void 0 : _plan_param.thought) || 'error without thought');
-                }
-            };
-            tasks.push(taskActionError);
-        } else if ('Finished' === plan.type) {
-            const taskActionFinished = {
-                type: 'Action',
-                subType: 'Finished',
-                param: null,
-                thought: plan.thought,
-                locate: plan.locate,
-                executor: async (param)=>{}
-            };
-            tasks.push(taskActionFinished);
-        } else if ('Sleep' === plan.type) {
-            const taskActionSleep = {
-                type: 'Action',
-                subType: 'Sleep',
-                param: plan.param,
-                thought: plan.thought,
-                locate: plan.locate,
-                executor: async (taskParam)=>{
-                    await external_utils_mjs_sleep((null == taskParam ? void 0 : taskParam.timeMs) || 3000);
-                }
-            };
-            tasks.push(taskActionSleep);
-        } else {
-            const planType = plan.type;
-            const actionSpace = await this.interface.actionSpace();
-            const action = actionSpace.find((action)=>action.name === planType);
-            const param = plan.param;
-            if (!action) throw new Error(`Action type '${planType}' not found`);
-            const locateFields = action ? findAllMidsceneLocatorField(action.paramSchema) : [];
-            const requiredLocateFields = action ? findAllMidsceneLocatorField(action.paramSchema, true) : [];
-            locateFields.forEach((field)=>{
-                if (param[field]) {
-                    const locatePlan = locatePlanForLocate(param[field]);
-                    debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`);
-                    const locateTask = taskForLocatePlan(locatePlan, param[field], (result)=>{
-                        param[field] = result;
-                    });
-                    tasks.push(locateTask);
-                } else {
-                    assert(!requiredLocateFields.includes(field), `Required locate field '${field}' is not provided for action ${planType}`);
-                    debug(`field '${field}' is not provided for action ${planType}`);
-                }
-            });
-            const task = {
-                type: 'Action',
-                subType: planType,
-                thought: plan.thought,
-                param: plan.param,
-                executor: async (param, context)=>{
-                    var _context_element;
-                    debug('executing action', planType, param, `context.element.center: ${null == (_context_element = context.element) ? void 0 : _context_element.center}`);
-                    const uiContext = await this.insight.contextRetrieverFn('locate');
-                    context.task.uiContext = uiContext;
-                    requiredLocateFields.forEach((field)=>{
-                        assert(param[field], `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`);
-                    });
-                    try {
-                        await Promise.all([
-                            (async ()=>{
-                                if (this.interface.beforeInvokeAction) {
-                                    debug('will call "beforeInvokeAction" for interface');
-                                    await this.interface.beforeInvokeAction(action.name, param);
-                                    debug('called "beforeInvokeAction" for interface');
-                                }
-                            })(),
-                            external_utils_mjs_sleep(200)
-                        ]);
-                    } catch (originalError) {
-                        const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
-                        throw new Error(`error in running beforeInvokeAction for ${action.name}: ${originalMessage}`, {
-                            cause: originalError
-                        });
-                    }
-                    if (action.paramSchema) try {
-                        param = parseActionParam(param, action.paramSchema);
-                    } catch (error) {
-                        throw new Error(`Invalid parameters for action ${action.name}: ${error.message}\nParameters: ${JSON.stringify(param)}`, {
-                            cause: error
-                        });
-                    }
-                    debug('calling action', action.name);
-                    const actionFn = action.call.bind(this.interface);
-                    await actionFn(param, context);
-                    debug('called action', action.name);
-                    await external_utils_mjs_sleep(300);
-                    try {
-                        if (this.interface.afterInvokeAction) {
-                            debug('will call "afterInvokeAction" for interface');
-                            await this.interface.afterInvokeAction(action.name, param);
-                            debug('called "afterInvokeAction" for interface');
-                        }
-                    } catch (originalError) {
-                        const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
-                        throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalMessage}`, {
-                            cause: originalError
-                        });
-                    }
-                    return {
-                        output: {
-                            success: true,
-                            action: planType,
-                            param: param
-                        }
-                    };
-                }
-            };
-            tasks.push(task);
-        }
-        const wrappedTasks = tasks.map((task, index)=>{
-            if ('Action' === task.type) return this.prependExecutorWithScreenshot(task, index === tasks.length - 1);
-            return task;
+    createExecutionSession(title, options) {
+        return new ExecutionSession(title, ()=>Promise.resolve(this.service.contextRetrieverFn()), {
+            onTaskStart: this.onTaskStartCallback,
+            tasks: options?.tasks,
+            onTaskUpdate: this.hooks?.onTaskUpdate
         });
-        return {
-            tasks: wrappedTasks
-        };
     }
-    async setupPlanningContext(executorContext) {
-        const shotTime = Date.now();
-        const uiContext = await this.insight.contextRetrieverFn('locate');
-        const recordItem = {
-            type: 'screenshot',
-            ts: shotTime,
-            screenshot: uiContext.screenshotBase64,
-            timing: 'before Planning'
-        };
-        executorContext.task.recorder = [
-            recordItem
-        ];
-        executorContext.task.uiContext = uiContext;
-        return {
-            uiContext
-        };
+    getActionSpace() {
+        return this.providedActionSpace;
+    }
+    async convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options) {
+        return this.taskBuilder.build(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options);
     }
     async loadYamlFlowAsPlanning(userInstruction, yamlString) {
-        const taskExecutor = new Executor(taskTitleStr('Action', userInstruction), {
-            onTaskStart: this.onTaskStartCallback
-        });
+        const session = this.createExecutionSession(taskTitleStr('Action', userInstruction));
         const task = {
             type: 'Planning',
             subType: 'LoadYaml',
-            locate: null,
             param: {
                 userInstruction
             },
             executor: async (param, executorContext)=>{
-                await this.setupPlanningContext(executorContext);
+                const { uiContext } = executorContext;
+                assert(uiContext, 'uiContext is required for Planning task');
                 return {
                     output: {
                         actions: [],
@@ -367,140 +67,137 @@ class TaskExecutor {
                 };
             }
         };
-        await taskExecutor.append(task);
-        await taskExecutor.flush();
+        const runner = session.getRunner();
+        await session.appendAndRun(task);
         return {
-            executor: taskExecutor
+            runner
         };
     }
-    createPlanningTask(userInstruction, actionContext, modelConfig) {
-        const task = {
-            type: 'Planning',
-            subType: 'Plan',
-            locate: null,
-            param: {
-                userInstruction
-            },
-            executor: async (param, executorContext)=>{
-                const startTime = Date.now();
-                const { uiContext } = await this.setupPlanningContext(executorContext);
-                const { vlMode } = modelConfig;
-                const uiTarsModelVersion = 'vlm-ui-tars' === vlMode ? modelConfig.uiTarsModelVersion : void 0;
-                assert(this.interface.actionSpace, 'actionSpace for device is not implemented');
-                const actionSpace = await this.interface.actionSpace();
-                debug('actionSpace for this interface is:', actionSpace.map((action)=>action.name).join(', '));
-                assert(Array.isArray(actionSpace), 'actionSpace must be an array');
-                if (0 === actionSpace.length) console.warn(`ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`);
-                const planResult = await (uiTarsModelVersion ? uiTarsPlanning : index_mjs_plan)(param.userInstruction, {
-                    context: uiContext,
-                    actionContext,
-                    interfaceType: this.interface.interfaceType,
-                    actionSpace,
-                    modelConfig,
-                    conversationHistory: this.conversationHistory
-                });
-                debug('planResult', JSON.stringify(planResult, null, 2));
-                const { actions, log, more_actions_needed_by_instruction, error, usage, rawResponse, sleep } = planResult;
-                executorContext.task.log = {
-                    ...executorContext.task.log || {},
-                    rawResponse
-                };
-                executorContext.task.usage = usage;
-                const finalActions = actions || [];
-                if (sleep) {
-                    const timeNow = Date.now();
-                    const timeRemaining = sleep - (timeNow - startTime);
-                    if (timeRemaining > 0) finalActions.push({
-                        type: 'Sleep',
-                        param: {
-                            timeMs: timeRemaining
-                        },
-                        locate: null
-                    });
-                }
-                if (0 === finalActions.length) assert(!more_actions_needed_by_instruction || sleep, error ? `Failed to plan: ${error}` : 'No plan found');
-                return {
-                    output: {
-                        actions: finalActions,
-                        more_actions_needed_by_instruction,
-                        log,
-                        yamlFlow: planResult.yamlFlow
-                    },
-                    cache: {
-                        hit: false
-                    },
-                    uiContext
-                };
-            }
-        };
-        return task;
-    }
-    async runPlans(title, plans, modelConfig) {
-        const taskExecutor = new Executor(title, {
-            onTaskStart: this.onTaskStartCallback
-        });
-        const { tasks } = await this.convertPlanToExecutable(plans, modelConfig);
-        await taskExecutor.append(tasks);
-        const result = await taskExecutor.flush();
-        const { output } = result;
+    async runPlans(title, plans, modelConfigForPlanning, modelConfigForDefaultIntent) {
+        const session = this.createExecutionSession(title);
+        const { tasks } = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent);
+        const runner = session.getRunner();
+        const result = await session.appendAndRun(tasks);
+        const { output } = result ?? {};
         return {
             output,
-            executor: taskExecutor
+            runner
         };
     }
-    getReplanningCycleLimit(isVlmUiTars) {
-        return this.replanningCycleLimit || globalConfigManager.getEnvConfigInNumber(MIDSCENE_REPLANNING_CYCLE_LIMIT) || (isVlmUiTars ? defaultVlmUiTarsReplanningCycleLimit : defaultReplanningCycleLimit);
-    }
-    async action(userPrompt, modelConfig, actionContext, cacheable) {
+    async action(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount) {
         this.conversationHistory.reset();
-        const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {
-            onTaskStart: this.onTaskStartCallback
-        });
+        const session = this.createExecutionSession(taskTitleStr('Action', userPrompt));
+        const runner = session.getRunner();
         let replanCount = 0;
         const yamlFlow = [];
-        const replanningCycleLimit = this.getReplanningCycleLimit('vlm-ui-tars' === modelConfig.vlMode);
+        const replanningCycleLimit = replanningCycleLimitOverride ?? this.replanningCycleLimit;
+        assert(void 0 !== replanningCycleLimit, 'replanningCycleLimit is required for TaskExecutor.action');
+        let errorCountInOnePlanningLoop = 0;
         while(true){
-            if (replanCount > replanningCycleLimit) {
-                const errorMsg = `Replanning ${replanningCycleLimit} times, which is more than the limit, please split the task into multiple steps`;
-                return this.appendErrorPlan(taskExecutor, errorMsg, modelConfig);
-            }
-            const planningTask = this.createPlanningTask(userPrompt, actionContext, modelConfig);
-            await taskExecutor.append(planningTask);
-            const result = await taskExecutor.flush();
-            const planResult = null == result ? void 0 : result.output;
-            if (taskExecutor.isInErrorState()) return {
-                output: planResult,
-                executor: taskExecutor
-            };
-            const plans = planResult.actions || [];
-            yamlFlow.push(...planResult.yamlFlow || []);
+            const result = await session.appendAndRun({
+                type: 'Planning',
+                subType: 'Plan',
+                param: {
+                    userInstruction: userPrompt,
+                    aiActContext,
+                    imagesIncludeCount
+                },
+                executor: async (param, executorContext)=>{
+                    const startTime = Date.now();
+                    const { uiContext } = executorContext;
+                    assert(uiContext, 'uiContext is required for Planning task');
+                    const { vlMode } = modelConfigForPlanning;
+                    const uiTarsModelVersion = 'vlm-ui-tars' === vlMode ? modelConfigForPlanning.uiTarsModelVersion : void 0;
+                    const actionSpace = this.getActionSpace();
+                    debug('actionSpace for this interface is:', actionSpace.map((action)=>action.name).join(', '));
+                    assert(Array.isArray(actionSpace), 'actionSpace must be an array');
+                    if (0 === actionSpace.length) console.warn(`ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`);
+                    const planResult = await (uiTarsModelVersion ? uiTarsPlanning : plan)(param.userInstruction, {
+                        context: uiContext,
+                        actionContext: param.aiActContext,
+                        interfaceType: this.interface.interfaceType,
+                        actionSpace,
+                        modelConfig: modelConfigForPlanning,
+                        conversationHistory: this.conversationHistory,
+                        includeBbox: includeBboxInPlanning,
+                        imagesIncludeCount
+                    });
+                    debug('planResult', JSON.stringify(planResult, null, 2));
+                    const { actions, log, more_actions_needed_by_instruction, error, usage, rawResponse, sleep } = planResult;
+                    executorContext.task.log = {
+                        ...executorContext.task.log || {},
+                        rawResponse
+                    };
+                    executorContext.task.usage = usage;
+                    executorContext.task.output = {
+                        actions: actions || [],
+                        more_actions_needed_by_instruction,
+                        log,
+                        yamlFlow: planResult.yamlFlow
+                    };
+                    executorContext.uiContext = uiContext;
+                    const finalActions = [
+                        ...actions || []
+                    ];
+                    if (sleep) {
+                        const timeNow = Date.now();
+                        const timeRemaining = sleep - (timeNow - startTime);
+                        if (timeRemaining > 0) finalActions.push(this.sleepPlan(timeRemaining));
+                    }
+                    assert(!error, `Failed to continue: ${error}\n${log || ''}`);
+                    return {
+                        cache: {
+                            hit: false
+                        }
+                    };
+                }
+            }, {
+                allowWhenError: true
+            });
+            const planResult = result?.output;
+            const plans = planResult?.actions || [];
+            yamlFlow.push(...planResult?.yamlFlow || []);
             let executables;
             try {
-                executables = await this.convertPlanToExecutable(plans, modelConfig, cacheable);
-                taskExecutor.append(executables.tasks);
+                executables = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, {
+                    cacheable,
+                    subTask: true
+                });
             } catch (error) {
-                return this.appendErrorPlan(taskExecutor, `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(plans)}`, modelConfig);
+                return session.appendErrorPlan(`Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(plans)}`);
             }
-            await taskExecutor.flush();
-            if (taskExecutor.isInErrorState()) return {
-                output: void 0,
-                executor: taskExecutor
-            };
-            if (!planResult.more_actions_needed_by_instruction) break;
-            replanCount++;
+            if (this.conversationHistory.pendingFeedbackMessage) console.warn('unconsumed pending feedback message detected, this may lead to unexpected planning result:', this.conversationHistory.pendingFeedbackMessage);
+            let errorFlag = false;
+            try {
+                await session.appendAndRun(executables.tasks);
+            } catch (error) {
+                errorFlag = true;
+                errorCountInOnePlanningLoop++;
+                this.conversationHistory.pendingFeedbackMessage = `Error executing running tasks: ${error?.message || String(error)}`;
+                debug('error when executing running tasks, but continue to run if it is not too many errors:', error instanceof Error ? error.message : String(error), 'current error count in one planning loop:', errorCountInOnePlanningLoop);
+            }
+            if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) return session.appendErrorPlan('Too many errors in one planning loop');
+            if (!planResult?.more_actions_needed_by_instruction) if (errorFlag) debug('more_actions_needed_by_instruction is false, but there are errors in one planning loop, continue to run');
+            else break;
+            ++replanCount;
+            if (replanCount > replanningCycleLimit) {
+                const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;
+                return session.appendErrorPlan(errorMsg);
+            }
+            if (!this.conversationHistory.pendingFeedbackMessage) this.conversationHistory.pendingFeedbackMessage = 'I have finished the action previously planned.';
         }
-        return {
+        const finalResult = {
             output: {
                 yamlFlow
             },
-            executor: taskExecutor
+            runner
         };
+        return finalResult;
     }
     createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt) {
         const queryTask = {
             type: 'Insight',
             subType: type,
-            locate: null,
             param: {
                 dataDemand: multimodalPrompt ? {
                     demand,
@@ -509,23 +206,15 @@ class TaskExecutor {
             },
             executor: async (param, taskContext)=>{
                 const { task } = taskContext;
-                let insightDump;
-                const dumpCollector = (dump)=>{
-                    insightDump = dump;
-                };
-                this.insight.onceDumpUpdatedFn = dumpCollector;
-                const shotTime = Date.now();
-                const uiContext = await this.insight.contextRetrieverFn('extract');
-                task.uiContext = uiContext;
-                const recordItem = {
-                    type: 'screenshot',
-                    ts: shotTime,
-                    screenshot: uiContext.screenshotBase64,
-                    timing: 'before Extract'
+                let queryDump;
+                const applyDump = (dump)=>{
+                    queryDump = dump;
+                    task.log = {
+                        dump
+                    };
                 };
-                task.recorder = [
-                    recordItem
-                ];
+                const uiContext = taskContext.uiContext;
+                assert(uiContext, 'uiContext is required for Query task');
                 const ifTypeRestricted = 'Query' !== type;
                 let demandInput = demand;
                 let keyOfResult = 'result';
@@ -538,13 +227,27 @@ class TaskExecutor {
                 } else if (ifTypeRestricted) demandInput = {
                     [keyOfResult]: `${type}, ${demand}`
                 };
-                const { data, usage, thought } = await this.insight.extract(demandInput, modelConfig, opt, multimodalPrompt);
+                let extractResult;
+                let extraPageDescription = '';
+                if (opt?.domIncluded && this.interface.getElementsNodeTree) {
+                    debug('appending tree info for page');
+                    const tree = await this.interface.getElementsNodeTree();
+                    extraPageDescription = await descriptionOfTree(tree, 200, false, opt?.domIncluded === 'visible-only');
+                }
+                try {
+                    extractResult = await this.service.extract(demandInput, modelConfig, opt, extraPageDescription, multimodalPrompt);
+                } catch (error) {
+                    if (error instanceof ServiceError) applyDump(error.dump);
+                    throw error;
+                }
+                const { data, usage, thought, dump } = extractResult;
+                applyDump(dump);
                 let outputResult = data;
                 if (ifTypeRestricted) if ('string' == typeof data) outputResult = data;
                 else if ('WaitFor' === type) outputResult = null == data ? false : data[keyOfResult];
                 else if (null == data) outputResult = null;
                 else {
-                    assert((null == data ? void 0 : data[keyOfResult]) !== void 0, 'No result in query data');
+                    assert(data?.[keyOfResult] !== void 0, 'No result in query data');
                     outputResult = data[keyOfResult];
                 }
                 if ('Assert' === type && !outputResult) {
@@ -554,7 +257,7 @@ class TaskExecutor {
                 }
                 return {
                     output: outputResult,
-                    log: insightDump,
+                    log: queryDump,
                     usage,
                     thought
                 };
@@ -563,101 +266,91 @@ class TaskExecutor {
         return queryTask;
     }
     async createTypeQueryExecution(type, demand, modelConfig, opt, multimodalPrompt) {
-        const taskExecutor = new Executor(taskTitleStr(type, 'string' == typeof demand ? demand : JSON.stringify(demand)), {
-            onTaskStart: this.onTaskStartCallback
-        });
+        const session = this.createExecutionSession(taskTitleStr(type, 'string' == typeof demand ? demand : JSON.stringify(demand)));
         const queryTask = await this.createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt);
-        await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));
-        const result = await taskExecutor.flush();
+        const runner = session.getRunner();
+        const result = await session.appendAndRun(queryTask);
         if (!result) throw new Error('result of taskExecutor.flush() is undefined in function createTypeQueryTask');
         const { output, thought } = result;
         return {
             output,
             thought,
-            executor: taskExecutor
+            runner
         };
     }
-    async appendErrorPlan(taskExecutor, errorMsg, modelConfig) {
-        const errorPlan = {
-            type: 'Error',
-            param: {
-                thought: errorMsg
-            },
-            locate: null
-        };
-        const { tasks } = await this.convertPlanToExecutable([
-            errorPlan
-        ], modelConfig);
-        await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
-        await taskExecutor.flush();
+    sleepPlan(timeMs) {
         return {
-            output: void 0,
-            executor: taskExecutor
-        };
-    }
-    async taskForSleep(timeMs, modelConfig) {
-        const sleepPlan = {
             type: 'Sleep',
             param: {
                 timeMs
-            },
-            locate: null
+            }
         };
-        const { tasks: sleepTasks } = await this.convertPlanToExecutable([
-            sleepPlan
-        ], modelConfig);
-        return this.prependExecutorWithScreenshot(sleepTasks[0]);
+    }
+    async taskForSleep(timeMs, _modelConfig) {
+        return this.taskBuilder.createSleepTask({
+            timeMs
+        });
     }
     async waitFor(assertion, opt, modelConfig) {
         const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
         const description = `waitFor: ${textPrompt}`;
-        const taskExecutor = new Executor(taskTitleStr('WaitFor', description), {
-            onTaskStart: this.onTaskStartCallback
-        });
+        const session = this.createExecutionSession(taskTitleStr('WaitFor', description));
+        const runner = session.getRunner();
         const { timeoutMs, checkIntervalMs } = opt;
         assert(assertion, 'No assertion for waitFor');
         assert(timeoutMs, 'No timeoutMs for waitFor');
         assert(checkIntervalMs, 'No checkIntervalMs for waitFor');
         assert(checkIntervalMs <= timeoutMs, `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`);
         const overallStartTime = Date.now();
-        let startTime = Date.now();
+        let lastCheckStart = overallStartTime;
         let errorThought = '';
-        while(Date.now() - overallStartTime < timeoutMs){
-            startTime = Date.now();
-            const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, modelConfig, {
-                doNotThrowError: true
-            }, multimodalPrompt);
-            await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));
-            const result = await taskExecutor.flush();
-            if (null == result ? void 0 : result.output) return {
+        while(lastCheckStart - overallStartTime <= timeoutMs){
+            const currentCheckStart = Date.now();
+            lastCheckStart = currentCheckStart;
+            const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, modelConfig, void 0, multimodalPrompt);
+            const result = await session.appendAndRun(queryTask);
+            if (result?.output) return {
                 output: void 0,
-                executor: taskExecutor
+                runner
             };
-            errorThought = (null == result ? void 0 : result.thought) || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
+            errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
             const now = Date.now();
-            if (now - startTime < checkIntervalMs) {
-                const timeRemaining = checkIntervalMs - (now - startTime);
-                const sleepTask = await this.taskForSleep(timeRemaining, modelConfig);
-                await taskExecutor.append(sleepTask);
+            if (now - currentCheckStart < checkIntervalMs) {
+                const timeRemaining = checkIntervalMs - (now - currentCheckStart);
+                const sleepTask = this.taskBuilder.createSleepTask({
+                    timeMs: timeRemaining
+                });
+                await session.append(sleepTask);
             }
         }
-        return this.appendErrorPlan(taskExecutor, `waitFor timeout: ${errorThought}`, modelConfig);
+        return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
     }
-    constructor(interfaceInstance, insight, opts){
+    constructor(interfaceInstance, service, opts){
         _define_property(this, "interface", void 0);
-        _define_property(this, "insight", void 0);
+        _define_property(this, "service", void 0);
         _define_property(this, "taskCache", void 0);
+        _define_property(this, "providedActionSpace", void 0);
+        _define_property(this, "taskBuilder", void 0);
         _define_property(this, "conversationHistory", void 0);
         _define_property(this, "onTaskStartCallback", void 0);
+        _define_property(this, "hooks", void 0);
         _define_property(this, "replanningCycleLimit", void 0);
         this.interface = interfaceInstance;
-        this.insight = insight;
+        this.service = service;
         this.taskCache = opts.taskCache;
-        this.onTaskStartCallback = null == opts ? void 0 : opts.onTaskStart;
+        this.onTaskStartCallback = opts?.onTaskStart;
         this.replanningCycleLimit = opts.replanningCycleLimit;
+        this.hooks = opts.hooks;
         this.conversationHistory = new ConversationHistory();
+        this.providedActionSpace = opts.actionSpace;
+        this.taskBuilder = new TaskBuilder({
+            interfaceInstance,
+            service,
+            taskCache: opts.taskCache,
+            actionSpace: this.getActionSpace()
+        });
     }
 }
-export { TaskExecutor, locatePlanForLocate };
+export { TaskExecutionError, TaskExecutor, locatePlanForLocate };
 
 //# sourceMappingURL=tasks.mjs.map