@midscene/core 0.27.0 → 0.27.1-beta-20250822094725.0

package/dist/es/agent/tasks.mjs

@@ -0,0 +1,767 @@
+import { elementByPositionWithElementInfo, findAllMidsceneLocatorField, resizeImageForUiTars, vlmPlanning } from "../ai-model/index.mjs";
+import { Executor, plan as external_index_mjs_plan } from "../index.mjs";
+import { sleep as external_utils_mjs_sleep } from "../utils.mjs";
+import { NodeType } from "@midscene/shared/constants";
+import { MIDSCENE_REPLANNING_CYCLE_LIMIT, getAIConfigInNumber } from "@midscene/shared/env";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { taskTitleStr } from "./ui-utils.mjs";
+import { matchElementFromCache, matchElementFromPlan, parsePrompt } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('device-task-executor');
+const defaultReplanningCycleLimit = 10;
+function locatePlanForLocate(param) {
+    const locate = 'string' == typeof param ? {
+        prompt: param
+    } : param;
+    const locatePlan = {
+        type: 'Locate',
+        locate,
+        param: locate,
+        thought: ''
+    };
+    return locatePlan;
+}
+class PageTaskExecutor {
+    async recordScreenshot(timing) {
+        const base64 = await this.page.screenshotBase64();
+        const item = {
+            type: 'screenshot',
+            ts: Date.now(),
+            screenshot: base64,
+            timing
+        };
+        return item;
+    }
+    async getElementXpath(pageContext, element) {
+        var _element_attributes;
+        if (!this.page.getXpathsByPoint) return void debug('getXpathsByPoint is not supported for this page');
+        let elementId = null == element ? void 0 : element.id;
+        if ((null == element ? void 0 : element.isOrderSensitive) !== void 0) {
+            const xpaths = await this.page.getXpathsByPoint({
+                left: element.center[0],
+                top: element.center[1]
+            }, null == element ? void 0 : element.isOrderSensitive);
+            return xpaths;
+        }
+        if ((null == element ? void 0 : null == (_element_attributes = element.attributes) ? void 0 : _element_attributes.nodeType) === NodeType.POSITION) {
+            await this.insight.contextRetrieverFn('locate');
+            const info = elementByPositionWithElementInfo(pageContext.tree, {
+                x: element.center[0],
+                y: element.center[1]
+            }, {
+                requireStrictDistance: false,
+                filterPositionElements: true
+            });
+            if (null == info ? void 0 : info.id) elementId = info.id;
+            else debug('no element id found for position node, will not update cache', element);
+        }
+        if (!elementId) return;
+        try {
+            const result = await this.page.getXpathsById(elementId);
+            return result;
+        } catch (error) {
+            debug('getXpathsById error: ', error);
+        }
+    }
+    prependExecutorWithScreenshot(taskApply, appendAfterExecution = false) {
+        const taskWithScreenshot = {
+            ...taskApply,
+            executor: async (param, context, ...args)=>{
+                const recorder = [];
+                const { task } = context;
+                task.recorder = recorder;
+                const shot = await this.recordScreenshot(`before ${task.type}`);
+                recorder.push(shot);
+                const result = await taskApply.executor(param, context, ...args);
+                if ('Action' === taskApply.type) await Promise.all([
+                    (async ()=>{
+                        await external_utils_mjs_sleep(100);
+                        if (this.page.beforeAction) {
+                            debug('will call "beforeAction" for page');
+                            await this.page.beforeAction();
+                        }
+                    })(),
+                    external_utils_mjs_sleep(200)
+                ]);
+                if (appendAfterExecution) {
+                    const shot2 = await this.recordScreenshot('after Action');
+                    recorder.push(shot2);
+                }
+                return result;
+            }
+        };
+        return taskWithScreenshot;
+    }
+    async convertPlanToExecutable(plans) {
+        const tasks = [];
+        const taskForLocatePlan = (plan, detailedLocateParam, onResult)=>{
+            if ('string' == typeof detailedLocateParam) detailedLocateParam = {
+                prompt: detailedLocateParam
+            };
+            const taskFind = {
+                type: 'Insight',
+                subType: 'Locate',
+                param: detailedLocateParam,
+                thought: plan.thought,
+                executor: async (param, taskContext)=>{
+                    var _this_taskCache, _locateCacheRecord_cacheContent;
+                    const { task } = taskContext;
+                    assert((null == param ? void 0 : param.prompt) || (null == param ? void 0 : param.id) || (null == param ? void 0 : param.bbox), `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
+                    let insightDump;
+                    let usage;
+                    const dumpCollector = (dump)=>{
+                        var _dump_taskInfo;
+                        insightDump = dump;
+                        usage = null == dump ? void 0 : null == (_dump_taskInfo = dump.taskInfo) ? void 0 : _dump_taskInfo.usage;
+                        task.log = {
+                            dump: insightDump
+                        };
+                        task.usage = usage;
+                    };
+                    this.insight.onceDumpUpdatedFn = dumpCollector;
+                    const shotTime = Date.now();
+                    const pageContext = await this.insight.contextRetrieverFn('locate');
+                    task.pageContext = pageContext;
+                    const recordItem = {
+                        type: 'screenshot',
+                        ts: shotTime,
+                        screenshot: pageContext.screenshotBase64,
+                        timing: 'before Insight'
+                    };
+                    task.recorder = [
+                        recordItem
+                    ];
+                    const elementFromXpath = param.xpath && this.page.getElementInfoByXpath ? await this.page.getElementInfoByXpath(param.xpath) : void 0;
+                    const userExpectedPathHitFlag = !!elementFromXpath;
+                    const cachePrompt = param.prompt;
+                    const locateCacheRecord = null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchLocateCache(cachePrompt);
+                    const xpaths = null == locateCacheRecord ? void 0 : null == (_locateCacheRecord_cacheContent = locateCacheRecord.cacheContent) ? void 0 : _locateCacheRecord_cacheContent.xpaths;
+                    const elementFromCache = userExpectedPathHitFlag ? null : await matchElementFromCache(this, xpaths, cachePrompt, param.cacheable);
+                    const cacheHitFlag = !!elementFromCache;
+                    const elementFromPlan = userExpectedPathHitFlag || cacheHitFlag ? void 0 : matchElementFromPlan(param, pageContext.tree);
+                    const planHitFlag = !!elementFromPlan;
+                    const elementFromAiLocate = userExpectedPathHitFlag || cacheHitFlag || planHitFlag ? void 0 : (await this.insight.locate(param, {
+                        context: pageContext
+                    })).element;
+                    const aiLocateHitFlag = !!elementFromAiLocate;
+                    const element = elementFromXpath || elementFromCache || elementFromPlan || elementFromAiLocate;
+                    let currentXpaths;
+                    if (element && this.taskCache && !cacheHitFlag && (null == param ? void 0 : param.cacheable) !== false) {
+                        const elementXpaths = await this.getElementXpath(pageContext, element);
+                        if (null == elementXpaths ? void 0 : elementXpaths.length) {
+                            currentXpaths = elementXpaths;
+                            this.taskCache.updateOrAppendCacheRecord({
+                                type: 'locate',
+                                prompt: cachePrompt,
+                                xpaths: elementXpaths
+                            }, locateCacheRecord);
+                        } else debug('no xpaths found, will not update cache', cachePrompt, elementXpaths);
+                    }
+                    if (!element) throw new Error(`Element not found: ${param.prompt}`);
+                    let hitBy;
+                    if (userExpectedPathHitFlag) hitBy = {
+                        from: 'User expected path',
+                        context: {
+                            xpath: param.xpath
+                        }
+                    };
+                    else if (cacheHitFlag) hitBy = {
+                        from: 'Cache',
+                        context: {
+                            xpathsFromCache: xpaths,
+                            xpathsToSave: currentXpaths
+                        }
+                    };
+                    else if (planHitFlag) hitBy = {
+                        from: 'Planning',
+                        context: {
+                            id: null == elementFromPlan ? void 0 : elementFromPlan.id,
+                            bbox: null == elementFromPlan ? void 0 : elementFromPlan.bbox
+                        }
+                    };
+                    else if (aiLocateHitFlag) hitBy = {
+                        from: 'AI model',
+                        context: {
+                            prompt: param.prompt
+                        }
+                    };
+                    null == onResult || onResult(element);
+                    return {
+                        output: {
+                            element
+                        },
+                        pageContext,
+                        hitBy
+                    };
+                }
+            };
+            return taskFind;
+        };
+        for (const plan of plans)if ('Locate' === plan.type) {
+            var _plan_locate, _plan_locate1;
+            if (!plan.locate || null === plan.locate || (null == (_plan_locate = plan.locate) ? void 0 : _plan_locate.id) === null || (null == (_plan_locate1 = plan.locate) ? void 0 : _plan_locate1.id) === 'null') {
+                debug('Locate action with id is null, will be ignored', plan);
+                continue;
+            }
+            const taskLocate = taskForLocatePlan(plan, plan.locate);
+            tasks.push(taskLocate);
+        } else if ('Error' === plan.type) {
+            var _plan_param;
+            const taskActionError = {
+                type: 'Action',
+                subType: 'Error',
+                param: plan.param,
+                thought: plan.thought || (null == (_plan_param = plan.param) ? void 0 : _plan_param.thought),
+                locate: plan.locate,
+                executor: async ()=>{
+                    var _plan_param;
+                    throw new Error((null == plan ? void 0 : plan.thought) || (null == (_plan_param = plan.param) ? void 0 : _plan_param.thought) || 'error without thought');
+                }
+            };
+            tasks.push(taskActionError);
+        } else if ('Finished' === plan.type) {
+            const taskActionFinished = {
+                type: 'Action',
+                subType: 'Finished',
+                param: null,
+                thought: plan.thought,
+                locate: plan.locate,
+                executor: async (param)=>{}
+            };
+            tasks.push(taskActionFinished);
+        } else if ('Sleep' === plan.type) {
+            const taskActionSleep = {
+                type: 'Action',
+                subType: 'Sleep',
+                param: plan.param,
+                thought: plan.thought,
+                locate: plan.locate,
+                executor: async (taskParam)=>{
+                    await external_utils_mjs_sleep((null == taskParam ? void 0 : taskParam.timeMs) || 3000);
+                }
+            };
+            tasks.push(taskActionSleep);
+        } else if ('Drag' === plan.type) {
+            const taskActionDrag = {
+                type: 'Action',
+                subType: 'Drag',
+                param: plan.param,
+                thought: plan.thought,
+                locate: plan.locate,
+                executor: async (taskParam)=>{
+                    assert((null == taskParam ? void 0 : taskParam.start_box) && (null == taskParam ? void 0 : taskParam.end_box), 'No start_box or end_box to drag');
+                    await this.page.mouse.drag(taskParam.start_box, taskParam.end_box);
+                }
+            };
+            tasks.push(taskActionDrag);
+        } else {
+            const planType = plan.type;
+            const actionSpace = await this.page.actionSpace();
+            const action = actionSpace.find((action)=>action.name === planType);
+            const param = plan.param;
+            if (!action) throw new Error(`Action type '${planType}' not found`);
+            const locateFields = action ? findAllMidsceneLocatorField(action.paramSchema) : [];
+            const requiredLocateFields = action ? findAllMidsceneLocatorField(action.paramSchema, true) : [];
+            locateFields.forEach((field)=>{
+                if (param[field]) {
+                    const locatePlan = locatePlanForLocate(param[field]);
+                    debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`);
+                    const locateTask = taskForLocatePlan(locatePlan, param[field], (result)=>{
+                        param[field] = result;
+                    });
+                    tasks.push(locateTask);
+                } else {
+                    assert(!requiredLocateFields.includes(field), `Required locate field '${field}' is not provided for action ${planType}`);
+                    debug(`field '${field}' is not provided for action ${planType}`);
+                }
+            });
+            const task = {
+                type: 'Action',
+                subType: planType,
+                thought: plan.thought,
+                param: plan.param,
+                executor: async (param, context)=>{
+                    var _context_element;
+                    debug('executing action', planType, param, `context.element.center: ${null == (_context_element = context.element) ? void 0 : _context_element.center}`);
+                    const pageContext = await this.insight.contextRetrieverFn('locate');
+                    context.task.pageContext = pageContext;
+                    requiredLocateFields.forEach((field)=>{
+                        assert(param[field], `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`);
+                    });
+                    const actionFn = action.call.bind(this.page);
+                    await actionFn(param, context);
+                    return {
+                        output: {
+                            success: true,
+                            action: planType,
+                            param: param
+                        }
+                    };
+                }
+            };
+            tasks.push(task);
+        }
+        const wrappedTasks = tasks.map((task, index)=>{
+            if ('Action' === task.type) return this.prependExecutorWithScreenshot(task, index === tasks.length - 1);
+            return task;
+        });
+        return {
+            tasks: wrappedTasks
+        };
+    }
+    async setupPlanningContext(executorContext) {
+        const shotTime = Date.now();
+        const pageContext = await this.insight.contextRetrieverFn('locate');
+        const recordItem = {
+            type: 'screenshot',
+            ts: shotTime,
+            screenshot: pageContext.screenshotBase64,
+            timing: 'before Planning'
+        };
+        executorContext.task.recorder = [
+            recordItem
+        ];
+        executorContext.task.pageContext = pageContext;
+        return {
+            pageContext
+        };
+    }
+    async loadYamlFlowAsPlanning(userInstruction, yamlString) {
+        const taskExecutor = new Executor(taskTitleStr('Action', userInstruction), {
+            onTaskStart: this.onTaskStartCallback
+        });
+        const task = {
+            type: 'Planning',
+            subType: 'LoadYaml',
+            locate: null,
+            param: {
+                userInstruction
+            },
+            executor: async (param, executorContext)=>{
+                await this.setupPlanningContext(executorContext);
+                return {
+                    output: {
+                        actions: [],
+                        more_actions_needed_by_instruction: false,
+                        log: '',
+                        yamlString
+                    },
+                    cache: {
+                        hit: true
+                    },
+                    hitBy: {
+                        from: 'Cache',
+                        context: {
+                            yamlString
+                        }
+                    }
+                };
+            }
+        };
+        await taskExecutor.append(task);
+        await taskExecutor.flush();
+        return {
+            executor: taskExecutor
+        };
+    }
+    planningTaskFromPrompt(userInstruction, log, actionContext) {
+        const task = {
+            type: 'Planning',
+            subType: 'Plan',
+            locate: null,
+            param: {
+                userInstruction,
+                log
+            },
+            executor: async (param, executorContext)=>{
+                const startTime = Date.now();
+                const { pageContext } = await this.setupPlanningContext(executorContext);
+                assert(this.page.actionSpace, 'actionSpace for device is not implemented');
+                const actionSpace = await this.page.actionSpace();
+                debug('actionSpace for page is:', actionSpace.map((action)=>action.name).join(', '));
+                assert(Array.isArray(actionSpace), 'actionSpace must be an array');
+                if (0 === actionSpace.length) console.warn(`ActionSpace for ${this.page.pageType} is empty. This may lead to unexpected behavior.`);
+                const planResult = await external_index_mjs_plan(param.userInstruction, {
+                    context: pageContext,
+                    log: param.log,
+                    actionContext,
+                    pageType: this.page.pageType,
+                    actionSpace
+                });
+                const { actions, log, more_actions_needed_by_instruction, error, usage, rawResponse, sleep } = planResult;
+                executorContext.task.log = {
+                    ...executorContext.task.log || {},
+                    rawResponse
+                };
+                executorContext.task.usage = usage;
+                const finalActions = actions || [];
+                if (sleep) {
+                    const timeNow = Date.now();
+                    const timeRemaining = sleep - (timeNow - startTime);
+                    if (timeRemaining > 0) finalActions.push({
+                        type: 'Sleep',
+                        param: {
+                            timeMs: timeRemaining
+                        },
+                        locate: null
+                    });
+                }
+                if (0 === finalActions.length) assert(!more_actions_needed_by_instruction || sleep, error ? `Failed to plan: ${error}` : 'No plan found');
+                return {
+                    output: {
+                        actions: finalActions,
+                        more_actions_needed_by_instruction,
+                        log,
+                        yamlFlow: planResult.yamlFlow
+                    },
+                    cache: {
+                        hit: false
+                    },
+                    pageContext
+                };
+            }
+        };
+        return task;
+    }
+    planningTaskToGoal(userInstruction, modelPreferences) {
+        const task = {
+            type: 'Planning',
+            subType: 'Plan',
+            locate: null,
+            param: {
+                userInstruction
+            },
+            executor: async (param, executorContext)=>{
+                var _actions_;
+                const { pageContext } = await this.setupPlanningContext(executorContext);
+                const imagePayload = await resizeImageForUiTars(pageContext.screenshotBase64, pageContext.size, modelPreferences);
+                this.appendConversationHistory({
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'image_url',
+                            image_url: {
+                                url: imagePayload
+                            }
+                        }
+                    ]
+                });
+                const planResult = await vlmPlanning({
+                    userInstruction: param.userInstruction,
+                    conversationHistory: this.conversationHistory,
+                    size: pageContext.size,
+                    modelPreferences
+                });
+                const { actions, action_summary, usage } = planResult;
+                executorContext.task.log = {
+                    ...executorContext.task.log || {},
+                    rawResponse: planResult.rawResponse
+                };
+                executorContext.task.usage = usage;
+                this.appendConversationHistory({
+                    role: 'assistant',
+                    content: action_summary
+                });
+                return {
+                    output: {
+                        actions,
+                        thought: null == (_actions_ = actions[0]) ? void 0 : _actions_.thought,
+                        actionType: actions[0].type,
+                        more_actions_needed_by_instruction: true,
+                        log: '',
+                        yamlFlow: planResult.yamlFlow
+                    },
+                    cache: {
+                        hit: false
+                    }
+                };
+            }
+        };
+        return task;
+    }
+    async runPlans(title, plans) {
+        const taskExecutor = new Executor(title, {
+            onTaskStart: this.onTaskStartCallback
+        });
+        const { tasks } = await this.convertPlanToExecutable(plans);
+        await taskExecutor.append(tasks);
+        const result = await taskExecutor.flush();
+        const { output } = result;
+        return {
+            output,
+            executor: taskExecutor
+        };
+    }
+    async action(userPrompt, actionContext) {
+        const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {
+            onTaskStart: this.onTaskStartCallback
+        });
+        let planningTask = this.planningTaskFromPrompt(userPrompt, void 0, actionContext);
+        let replanCount = 0;
+        const logList = [];
+        const yamlFlow = [];
+        const replanningCycleLimit = getAIConfigInNumber(MIDSCENE_REPLANNING_CYCLE_LIMIT) || defaultReplanningCycleLimit;
+        while(planningTask){
+            if (replanCount > replanningCycleLimit) {
+                const errorMsg = 'Replanning too many times, please split the task into multiple steps';
+                return this.appendErrorPlan(taskExecutor, errorMsg);
+            }
+            await taskExecutor.append(planningTask);
+            const result = await taskExecutor.flush();
+            const planResult = null == result ? void 0 : result.output;
+            if (taskExecutor.isInErrorState()) return {
+                output: planResult,
+                executor: taskExecutor
+            };
+            const plans = planResult.actions || [];
+            yamlFlow.push(...planResult.yamlFlow || []);
+            let executables;
+            try {
+                executables = await this.convertPlanToExecutable(plans);
+                taskExecutor.append(executables.tasks);
+            } catch (error) {
+                return this.appendErrorPlan(taskExecutor, `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(plans)}`);
+            }
+            await taskExecutor.flush();
+            if (taskExecutor.isInErrorState()) return {
+                output: void 0,
+                executor: taskExecutor
+            };
+            if (null == planResult ? void 0 : planResult.log) logList.push(planResult.log);
+            if (!planResult.more_actions_needed_by_instruction) {
+                planningTask = null;
+                break;
+            }
+            planningTask = this.planningTaskFromPrompt(userPrompt, logList.length > 0 ? `- ${logList.join('\n- ')}` : void 0, actionContext);
+            replanCount++;
+        }
+        return {
+            output: {
+                yamlFlow
+            },
+            executor: taskExecutor
+        };
+    }
+    async actionToGoal(userPrompt) {
+        const taskExecutor = new Executor(taskTitleStr('Action', userPrompt), {
+            onTaskStart: this.onTaskStartCallback
+        });
+        this.conversationHistory = [];
+        const isCompleted = false;
+        let currentActionCount = 0;
+        const maxActionNumber = 40;
+        const yamlFlow = [];
+        while(!isCompleted && currentActionCount < maxActionNumber){
+            currentActionCount++;
+            debug('actionToGoal, currentActionCount:', currentActionCount, 'userPrompt:', userPrompt);
+            const planningTask = this.planningTaskToGoal(userPrompt, {
+                intent: 'planning'
+            });
+            await taskExecutor.append(planningTask);
+            const result = await taskExecutor.flush();
+            if (taskExecutor.isInErrorState()) return {
+                output: void 0,
+                executor: taskExecutor
+            };
+            if (!result) throw new Error('result of taskExecutor.flush() is undefined in function actionToGoal');
+            const { output } = result;
+            const plans = output.actions;
+            yamlFlow.push(...output.yamlFlow || []);
+            let executables;
+            try {
+                executables = await this.convertPlanToExecutable(plans);
+                taskExecutor.append(executables.tasks);
+            } catch (error) {
+                return this.appendErrorPlan(taskExecutor, `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(plans)}`);
+            }
+            await taskExecutor.flush();
+            if (taskExecutor.isInErrorState()) return {
+                output: void 0,
+                executor: taskExecutor
+            };
+            if ('Finished' === plans[0].type) break;
+        }
+        return {
+            output: {
+                yamlFlow
+            },
+            executor: taskExecutor
+        };
+    }
+    createTypeQueryTask(type, demand, opt, multimodalPrompt) {
+        const queryTask = {
+            type: 'Insight',
+            subType: type,
+            locate: null,
+            param: {
+                dataDemand: multimodalPrompt ? {
+                    demand,
+                    multimodalPrompt
+                } : demand
+            },
+            executor: async (param, taskContext)=>{
+                const { task } = taskContext;
+                let insightDump;
+                const dumpCollector = (dump)=>{
+                    insightDump = dump;
+                };
+                this.insight.onceDumpUpdatedFn = dumpCollector;
+                const shotTime = Date.now();
+                const pageContext = await this.insight.contextRetrieverFn('extract');
+                task.pageContext = pageContext;
+                const recordItem = {
+                    type: 'screenshot',
+                    ts: shotTime,
+                    screenshot: pageContext.screenshotBase64,
+                    timing: 'before Extract'
+                };
+                task.recorder = [
+                    recordItem
+                ];
+                const ifTypeRestricted = 'Query' !== type;
+                let demandInput = demand;
+                if (ifTypeRestricted) {
+                    const returnType = 'Assert' === type ? 'Boolean' : type;
+                    demandInput = {
+                        result: `${returnType}, ${demand}`
+                    };
+                }
+                const { data, usage, thought } = await this.insight.extract(demandInput, opt, multimodalPrompt);
+                let outputResult = data;
+                if (ifTypeRestricted) {
+                    assert((null == data ? void 0 : data.result) !== void 0, 'No result in query data');
+                    outputResult = data.result;
+                }
+                return {
+                    output: outputResult,
+                    log: {
+                        dump: insightDump,
+                        isWaitForAssert: null == opt ? void 0 : opt.isWaitForAssert
+                    },
+                    usage,
+                    thought
+                };
+            }
+        };
+        return queryTask;
+    }
+    async createTypeQueryExecution(type, demand, opt, multimodalPrompt) {
+        const taskExecutor = new Executor(taskTitleStr(type, 'string' == typeof demand ? demand : JSON.stringify(demand)), {
+            onTaskStart: this.onTaskStartCallback
+        });
+        const queryTask = await this.createTypeQueryTask(type, demand, opt, multimodalPrompt);
+        await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));
+        const result = await taskExecutor.flush();
+        if (!result) throw new Error('result of taskExecutor.flush() is undefined in function createTypeQueryTask');
+        const { output, thought } = result;
+        return {
+            output,
+            thought,
+            executor: taskExecutor
+        };
+    }
+    async assert(assertion, opt) {
+        const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
+        return await this.createTypeQueryExecution('Assert', textPrompt, opt, multimodalPrompt);
+    }
+    appendConversationHistory(conversationHistory) {
+        if ('user' === conversationHistory.role) {
+            const userImgItems = this.conversationHistory.filter((item)=>'user' === item.role);
+            if (userImgItems.length >= 4 && 'user' === conversationHistory.role) {
+                const firstUserImgIndex = this.conversationHistory.findIndex((item)=>'user' === item.role);
+                if (firstUserImgIndex >= 0) this.conversationHistory.splice(firstUserImgIndex, 1);
+            }
+        }
+        this.conversationHistory.push(conversationHistory);
+    }
+    async appendErrorPlan(taskExecutor, errorMsg) {
+        const errorPlan = {
+            type: 'Error',
+            param: {
+                thought: errorMsg
+            },
+            locate: null
+        };
+        const { tasks } = await this.convertPlanToExecutable([
+            errorPlan
+        ]);
+        await taskExecutor.append(this.prependExecutorWithScreenshot(tasks[0]));
+        await taskExecutor.flush();
+        return {
+            output: void 0,
+            executor: taskExecutor
+        };
+    }
+    async waitFor(assertion, opt) {
+        const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
+        const description = `waitFor: ${textPrompt}`;
+        const taskExecutor = new Executor(taskTitleStr('WaitFor', description), {
+            onTaskStart: this.onTaskStartCallback
+        });
+        const { timeoutMs, checkIntervalMs } = opt;
+        assert(assertion, 'No assertion for waitFor');
+        assert(timeoutMs, 'No timeoutMs for waitFor');
+        assert(checkIntervalMs, 'No checkIntervalMs for waitFor');
+        assert(checkIntervalMs <= timeoutMs, `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`);
+        const overallStartTime = Date.now();
+        let startTime = Date.now();
+        let errorThought = '';
+        while(Date.now() - overallStartTime < timeoutMs){
+            startTime = Date.now();
+            const queryTask = await this.createTypeQueryTask('Assert', textPrompt, {
+                isWaitForAssert: true,
+                returnThought: true,
+                doNotThrowError: true
+            }, multimodalPrompt);
+            await taskExecutor.append(this.prependExecutorWithScreenshot(queryTask));
+            const result = await taskExecutor.flush();
+            if (!result) throw new Error('result of taskExecutor.flush() is undefined in function waitFor');
+            if (null == result ? void 0 : result.output) return {
+                output: void 0,
+                executor: taskExecutor
+            };
+            errorThought = (null == result ? void 0 : result.thought) || `unknown error when waiting for assertion: ${textPrompt}`;
+            const now = Date.now();
+            if (now - startTime < checkIntervalMs) {
+                const timeRemaining = checkIntervalMs - (now - startTime);
+                const sleepPlan = {
+                    type: 'Sleep',
+                    param: {
+                        timeMs: timeRemaining
+                    },
+                    locate: null
+                };
+                const { tasks: sleepTasks } = await this.convertPlanToExecutable([
+                    sleepPlan
+                ]);
+                await taskExecutor.append(this.prependExecutorWithScreenshot(sleepTasks[0]));
+                await taskExecutor.flush();
+            }
+        }
+        return this.appendErrorPlan(taskExecutor, `waitFor timeout: ${errorThought}`);
+    }
+    constructor(page, insight, opts){
+        _define_property(this, "page", void 0);
+        _define_property(this, "insight", void 0);
+        _define_property(this, "taskCache", void 0);
+        _define_property(this, "conversationHistory", []);
+        _define_property(this, "onTaskStartCallback", void 0);
+        this.page = page;
+        this.insight = insight;
+        this.taskCache = opts.taskCache;
+        this.onTaskStartCallback = null == opts ? void 0 : opts.onTaskStart;
+    }
+}
+export { PageTaskExecutor, locatePlanForLocate };
+
+//# sourceMappingURL=tasks.mjs.map