@midscene/core 1.0.1-beta-20251021060907.0 → 1.0.1-beta-20251022061922.0

package/dist/es/agent/task-builder.mjs

@@ -1,8 +1,8 @@
+import { findAllMidsceneLocatorField, parseActionParam } from "../ai-model/index.mjs";
 import { InsightError } from "../types.mjs";
 import { sleep } from "../utils.mjs";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
-import { findAllMidsceneLocatorField, parseActionParam } from "../ai-model/index.mjs";
 import { matchElementFromCache, matchElementFromPlan } from "./utils.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
@@ -31,263 +31,275 @@ class TaskBuilder {
     async build(plans, modelConfig, options) {
         const tasks = [];
         const cacheable = null == options ? void 0 : options.cacheable;
-        const taskForLocatePlan = (plan, detailedLocateParam, onResult)=>{
-            if ('string' == typeof detailedLocateParam) detailedLocateParam = {
-                prompt: detailedLocateParam
-            };
-            if (void 0 !== cacheable) detailedLocateParam = {
-                ...detailedLocateParam,
-                cacheable
-            };
-            const taskFind = {
-                type: 'Insight',
-                subType: 'Locate',
-                param: detailedLocateParam,
-                thought: plan.thought,
-                executor: async (param, taskContext)=>{
-                    var _this_taskCache, _locateCacheRecord_cacheContent;
-                    const { task } = taskContext;
-                    let { uiContext } = taskContext;
-                    assert((null == param ? void 0 : param.prompt) || (null == param ? void 0 : param.id) || (null == param ? void 0 : param.bbox), `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
-                    if (!uiContext) uiContext = await this.insight.contextRetrieverFn();
-                    assert(uiContext, 'uiContext is required for Insight task');
-                    let locateDump;
-                    let locateResult;
-                    const applyDump = (dump)=>{
-                        var _dump_taskInfo, _dump_taskInfo1;
-                        if (!dump) return;
-                        locateDump = dump;
-                        task.log = {
-                            dump
-                        };
-                        task.usage = null == (_dump_taskInfo = dump.taskInfo) ? void 0 : _dump_taskInfo.usage;
-                        if (null == (_dump_taskInfo1 = dump.taskInfo) ? void 0 : _dump_taskInfo1.searchAreaUsage) task.searchAreaUsage = dump.taskInfo.searchAreaUsage;
-                    };
-                    const elementFromXpath = param.xpath && this.interface.getElementInfoByXpath ? await this.interface.getElementInfoByXpath(param.xpath) : void 0;
-                    const userExpectedPathHitFlag = !!elementFromXpath;
-                    const cachePrompt = param.prompt;
-                    const locateCacheRecord = null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchLocateCache(cachePrompt);
-                    const cacheEntry = null == locateCacheRecord ? void 0 : null == (_locateCacheRecord_cacheContent = locateCacheRecord.cacheContent) ? void 0 : _locateCacheRecord_cacheContent.cache;
-                    const elementFromCache = userExpectedPathHitFlag ? null : await matchElementFromCache({
-                        taskCache: this.taskCache,
-                        interfaceInstance: this.interface
-                    }, cacheEntry, cachePrompt, param.cacheable);
-                    const cacheHitFlag = !!elementFromCache;
-                    const elementFromPlan = userExpectedPathHitFlag || cacheHitFlag ? void 0 : matchElementFromPlan(param, uiContext.tree);
-                    const planHitFlag = !!elementFromPlan;
-                    let elementFromAiLocate;
-                    if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) try {
-                        locateResult = await this.insight.locate(param, {
-                            context: uiContext
-                        }, modelConfig);
-                        applyDump(locateResult.dump);
-                        elementFromAiLocate = locateResult.element;
-                    } catch (error) {
-                        if (error instanceof InsightError) applyDump(error.dump);
-                        throw error;
-                    }
-                    const aiLocateHitFlag = !!elementFromAiLocate;
-                    const element = elementFromXpath || elementFromCache || elementFromPlan || elementFromAiLocate;
-                    let currentCacheEntry;
-                    if (element && this.taskCache && !cacheHitFlag && (null == param ? void 0 : param.cacheable) !== false) if (this.interface.cacheFeatureForRect) try {
-                        const feature = await this.interface.cacheFeatureForRect(element.rect, void 0 !== element.isOrderSensitive ? {
-                            _orderSensitive: element.isOrderSensitive
-                        } : void 0);
-                        if (feature && Object.keys(feature).length > 0) {
-                            debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
-                            currentCacheEntry = feature;
-                            this.taskCache.updateOrAppendCacheRecord({
-                                type: 'locate',
-                                prompt: cachePrompt,
-                                cache: feature
-                            }, locateCacheRecord);
-                        } else debug('no cache data returned, skip cache update, prompt: %s', cachePrompt);
-                    } catch (error) {
-                        debug('cacheFeatureForRect failed: %s', error);
-                    }
-                    else debug('cacheFeatureForRect is not supported, skip cache update');
-                    if (!element) {
-                        if (locateDump) throw new InsightError(`Element not found: ${param.prompt}`, locateDump);
-                        throw new Error(`Element not found: ${param.prompt}`);
-                    }
-                    let hitBy;
-                    if (userExpectedPathHitFlag) hitBy = {
-                        from: 'User expected path',
-                        context: {
-                            xpath: param.xpath
-                        }
-                    };
-                    else if (cacheHitFlag) hitBy = {
-                        from: 'Cache',
-                        context: {
-                            cacheEntry,
-                            cacheToSave: currentCacheEntry
-                        }
-                    };
-                    else if (planHitFlag) hitBy = {
-                        from: 'Planning',
-                        context: {
-                            id: null == elementFromPlan ? void 0 : elementFromPlan.id,
-                            bbox: null == elementFromPlan ? void 0 : elementFromPlan.bbox
-                        }
-                    };
-                    else if (aiLocateHitFlag) hitBy = {
-                        from: 'AI model',
-                        context: {
-                            prompt: param.prompt
-                        }
-                    };
-                    null == onResult || onResult(element);
-                    return {
-                        output: {
-                            element
-                        },
-                        uiContext,
-                        hitBy
-                    };
-                }
-            };
-            return taskFind;
+        const context = {
+            tasks,
+            modelConfig,
+            cacheable,
+            subTask: !!(null == options ? void 0 : options.subTask)
         };
+        const planHandlers = new Map([
+            [
+                'Locate',
+                (plan)=>this.handleLocatePlan(plan, context)
+            ],
+            [
+                'Finished',
+                (plan)=>this.handleFinishedPlan(plan, context)
+            ],
+            [
+                'Sleep',
+                (plan)=>this.handleSleepPlan(plan, context)
+            ]
+        ]);
+        const defaultHandler = (plan)=>this.handleActionPlan(plan, context);
         for (const plan of plans){
-            if ('Locate' === plan.type) {
-                if (!plan.locate || null === plan.locate) {
-                    debug('Locate action with id is null, will be ignored', plan);
-                    continue;
-                }
-                const taskLocate = taskForLocatePlan(plan, plan.locate);
-                tasks.push(taskLocate);
-                continue;
-            }
-            if ('Error' === plan.type) {
-                var _plan_param;
-                const taskActionError = {
-                    type: 'Action',
-                    subType: 'Error',
-                    param: plan.param,
-                    thought: plan.thought || (null == (_plan_param = plan.param) ? void 0 : _plan_param.thought),
-                    locate: plan.locate,
-                    executor: async ()=>{
-                        var _plan_param;
-                        throw new Error((null == plan ? void 0 : plan.thought) || (null == (_plan_param = plan.param) ? void 0 : _plan_param.thought) || 'error without thought');
-                    }
-                };
-                tasks.push(taskActionError);
-                continue;
-            }
-            if ('Finished' === plan.type) {
-                const taskActionFinished = {
-                    type: 'Action',
-                    subType: 'Finished',
-                    param: null,
-                    thought: plan.thought,
-                    locate: plan.locate,
-                    executor: async ()=>{}
-                };
-                tasks.push(taskActionFinished);
-                continue;
+            const handler = planHandlers.get(plan.type) ?? defaultHandler;
+            await handler(plan);
+        }
+        return {
+            tasks
+        };
+    }
+    handleFinishedPlan(plan, context) {
+        const taskActionFinished = {
+            type: 'Action',
+            subType: 'Finished',
+            param: null,
+            thought: plan.thought,
+            locate: plan.locate,
+            subTask: context.subTask || void 0,
+            executor: async ()=>{}
+        };
+        context.tasks.push(taskActionFinished);
+    }
+    handleSleepPlan(plan, context) {
+        const sleepTask = this.createSleepTask(plan.param, {
+            thought: plan.thought,
+            locate: plan.locate
+        });
+        if (context.subTask) sleepTask.subTask = true;
+        context.tasks.push(sleepTask);
+    }
+    createSleepTask(param, meta) {
+        return {
+            type: 'Action',
+            subType: 'Sleep',
+            param,
+            thought: null == meta ? void 0 : meta.thought,
+            locate: (null == meta ? void 0 : meta.locate) ?? null,
+            executor: async (taskParam)=>{
+                await sleep((null == taskParam ? void 0 : taskParam.timeMs) || 3000);
             }
-            if ('Sleep' === plan.type) {
-                const taskActionSleep = {
-                    type: 'Action',
-                    subType: 'Sleep',
-                    param: plan.param,
-                    thought: plan.thought,
-                    locate: plan.locate,
-                    executor: async (taskParam)=>{
-                        await sleep((null == taskParam ? void 0 : taskParam.timeMs) || 3000);
-                    }
-                };
-                tasks.push(taskActionSleep);
-                continue;
+        };
+    }
+    async handleLocatePlan(plan, context) {
+        if (!plan.locate || null === plan.locate) return void debug('Locate action with id is null, will be ignored', plan);
+        const taskLocate = this.createLocateTask(plan, plan.locate, context);
+        context.tasks.push(taskLocate);
+    }
+    async handleActionPlan(plan, context) {
+        const planType = plan.type;
+        const actionSpace = await this.interface.actionSpace();
+        const action = actionSpace.find((item)=>item.name === planType);
+        const param = plan.param;
+        if (!action) throw new Error(`Action type '${planType}' not found`);
+        const locateFields = action ? findAllMidsceneLocatorField(action.paramSchema) : [];
+        const requiredLocateFields = action ? findAllMidsceneLocatorField(action.paramSchema, true) : [];
+        locateFields.forEach((field)=>{
+            if (param[field]) {
+                const locatePlan = locatePlanForLocate(param[field]);
+                debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`);
+                const locateTask = this.createLocateTask(locatePlan, param[field], context, (result)=>{
+                    param[field] = result;
+                });
+                context.tasks.push(locateTask);
+            } else {
+                assert(!requiredLocateFields.includes(field), `Required locate field '${field}' is not provided for action ${planType}`);
+                debug(`field '${field}' is not provided for action ${planType}`);
             }
-            const planType = plan.type;
-            const actionSpace = await this.interface.actionSpace();
-            const action = actionSpace.find((item)=>item.name === planType);
-            const param = plan.param;
-            if (!action) throw new Error(`Action type '${planType}' not found`);
-            const locateFields = action ? findAllMidsceneLocatorField(action.paramSchema) : [];
-            const requiredLocateFields = action ? findAllMidsceneLocatorField(action.paramSchema, true) : [];
-            locateFields.forEach((field)=>{
-                if (param[field]) {
-                    const locatePlan = locatePlanForLocate(param[field]);
-                    debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`);
-                    const locateTask = taskForLocatePlan(locatePlan, param[field], (result)=>{
-                        param[field] = result;
+        });
+        const task = {
+            type: 'Action',
+            subType: planType,
+            thought: plan.thought,
+            param: plan.param,
+            subTask: context.subTask || void 0,
+            executor: async (param, taskContext)=>{
+                var _taskContext_element;
+                debug('executing action', planType, param, `taskContext.element.center: ${null == (_taskContext_element = taskContext.element) ? void 0 : _taskContext_element.center}`);
+                const uiContext = taskContext.uiContext;
+                assert(uiContext, 'uiContext is required for Action task');
+                requiredLocateFields.forEach((field)=>{
+                    assert(param[field], `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`);
+                });
+                try {
+                    await Promise.all([
+                        (async ()=>{
+                            if (this.interface.beforeInvokeAction) {
+                                debug('will call "beforeInvokeAction" for interface');
+                                await this.interface.beforeInvokeAction(action.name, param);
+                                debug('called "beforeInvokeAction" for interface');
+                            }
+                        })(),
+                        sleep(200)
+                    ]);
+                } catch (originalError) {
+                    const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
+                    throw new Error(`error in running beforeInvokeAction for ${action.name}: ${originalMessage}`, {
+                        cause: originalError
                     });
-                    tasks.push(locateTask);
-                } else {
-                    assert(!requiredLocateFields.includes(field), `Required locate field '${field}' is not provided for action ${planType}`);
-                    debug(`field '${field}' is not provided for action ${planType}`);
                 }
-            });
-            const task = {
-                type: 'Action',
-                subType: planType,
-                thought: plan.thought,
-                param: plan.param,
-                executor: async (param, taskContext)=>{
-                    var _taskContext_element;
-                    debug('executing action', planType, param, `taskContext.element.center: ${null == (_taskContext_element = taskContext.element) ? void 0 : _taskContext_element.center}`);
-                    const uiContext = taskContext.uiContext;
-                    assert(uiContext, 'uiContext is required for Action task');
-                    requiredLocateFields.forEach((field)=>{
-                        assert(param[field], `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`);
+                if (action.paramSchema) try {
+                    param = parseActionParam(param, action.paramSchema);
+                } catch (error) {
+                    throw new Error(`Invalid parameters for action ${action.name}: ${error.message}\nParameters: ${JSON.stringify(param)}`, {
+                        cause: error
                     });
-                    try {
-                        await Promise.all([
-                            (async ()=>{
-                                if (this.interface.beforeInvokeAction) {
-                                    debug('will call "beforeInvokeAction" for interface');
-                                    await this.interface.beforeInvokeAction(action.name, param);
-                                    debug('called "beforeInvokeAction" for interface');
-                                }
-                            })(),
-                            sleep(200)
-                        ]);
-                    } catch (originalError) {
-                        const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
-                        throw new Error(`error in running beforeInvokeAction for ${action.name}: ${originalMessage}`, {
-                            cause: originalError
-                        });
-                    }
-                    if (action.paramSchema) try {
-                        param = parseActionParam(param, action.paramSchema);
-                    } catch (error) {
-                        throw new Error(`Invalid parameters for action ${action.name}: ${error.message}\nParameters: ${JSON.stringify(param)}`, {
-                            cause: error
-                        });
+                }
+                debug('calling action', action.name);
+                const actionFn = action.call.bind(this.interface);
+                await actionFn(param, taskContext);
+                debug('called action', action.name);
+                try {
+                    if (this.interface.afterInvokeAction) {
+                        debug('will call "afterInvokeAction" for interface');
+                        await this.interface.afterInvokeAction(action.name, param);
+                        debug('called "afterInvokeAction" for interface');
                     }
-                    debug('calling action', action.name);
-                    const actionFn = action.call.bind(this.interface);
-                    await actionFn(param, taskContext);
-                    debug('called action', action.name);
-                    try {
-                        if (this.interface.afterInvokeAction) {
-                            debug('will call "afterInvokeAction" for interface');
-                            await this.interface.afterInvokeAction(action.name, param);
-                            debug('called "afterInvokeAction" for interface');
-                        }
-                    } catch (originalError) {
-                        const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
-                        throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalMessage}`, {
-                            cause: originalError
-                        });
+                } catch (originalError) {
+                    const originalMessage = (null == originalError ? void 0 : originalError.message) || String(originalError);
+                    throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalMessage}`, {
+                        cause: originalError
+                    });
+                }
+                return {
+                    output: {
+                        success: true,
+                        action: planType,
+                        param: param
                     }
-                    return {
-                        output: {
-                            success: true,
-                            action: planType,
-                            param: param
-                        }
+                };
+            }
+        };
+        context.tasks.push(task);
+    }
+    createLocateTask(plan, detailedLocateParam, context, onResult) {
+        const { cacheable, modelConfig } = context;
+        let locateParam = detailedLocateParam;
+        if ('string' == typeof locateParam) locateParam = {
+            prompt: locateParam
+        };
+        if (void 0 !== cacheable) locateParam = {
+            ...locateParam,
+            cacheable
+        };
+        const taskFind = {
+            type: 'Insight',
+            subType: 'Locate',
+            subTask: context.subTask || void 0,
+            param: locateParam,
+            thought: plan.thought,
+            executor: async (param, taskContext)=>{
+                var _this_taskCache, _locateCacheRecord_cacheContent;
+                const { task } = taskContext;
+                let { uiContext } = taskContext;
+                assert((null == param ? void 0 : param.prompt) || (null == param ? void 0 : param.id) || (null == param ? void 0 : param.bbox), `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
+                if (!uiContext) uiContext = await this.insight.contextRetrieverFn();
+                assert(uiContext, 'uiContext is required for Insight task');
+                let locateDump;
+                let locateResult;
+                const applyDump = (dump)=>{
+                    var _dump_taskInfo, _dump_taskInfo1;
+                    if (!dump) return;
+                    locateDump = dump;
+                    task.log = {
+                        dump
                     };
+                    task.usage = null == (_dump_taskInfo = dump.taskInfo) ? void 0 : _dump_taskInfo.usage;
+                    if (null == (_dump_taskInfo1 = dump.taskInfo) ? void 0 : _dump_taskInfo1.searchAreaUsage) task.searchAreaUsage = dump.taskInfo.searchAreaUsage;
+                };
+                let elementFromXpath;
+                if (param.xpath && this.interface.rectMatchesCacheFeature) elementFromXpath = await this.interface.rectMatchesCacheFeature({
+                    xpaths: [
+                        param.xpath
+                    ]
+                });
+                const userExpectedPathHitFlag = !!elementFromXpath;
+                const cachePrompt = param.prompt;
+                const locateCacheRecord = null == (_this_taskCache = this.taskCache) ? void 0 : _this_taskCache.matchLocateCache(cachePrompt);
+                const cacheEntry = null == locateCacheRecord ? void 0 : null == (_locateCacheRecord_cacheContent = locateCacheRecord.cacheContent) ? void 0 : _locateCacheRecord_cacheContent.cache;
+                const elementFromCache = userExpectedPathHitFlag ? null : await matchElementFromCache({
+                    taskCache: this.taskCache,
+                    interfaceInstance: this.interface
+                }, cacheEntry, cachePrompt, param.cacheable);
+                const cacheHitFlag = !!elementFromCache;
+                const elementFromPlan = userExpectedPathHitFlag || cacheHitFlag ? void 0 : matchElementFromPlan(param, uiContext.tree);
+                const planHitFlag = !!elementFromPlan;
+                let elementFromAiLocate;
+                if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) try {
+                    locateResult = await this.insight.locate(param, {
+                        context: uiContext
+                    }, modelConfig);
+                    applyDump(locateResult.dump);
+                    elementFromAiLocate = locateResult.element;
+                } catch (error) {
+                    if (error instanceof InsightError) applyDump(error.dump);
+                    throw error;
                 }
-            };
-            tasks.push(task);
-        }
-        return {
-            tasks
+                const element = elementFromXpath || elementFromCache || elementFromPlan || elementFromAiLocate;
+                let currentCacheEntry;
+                if (element && this.taskCache && !cacheHitFlag && (null == param ? void 0 : param.cacheable) !== false) if (this.interface.cacheFeatureForRect) try {
+                    const feature = await this.interface.cacheFeatureForRect(element.rect, void 0 !== element.isOrderSensitive ? {
+                        _orderSensitive: element.isOrderSensitive
+                    } : void 0);
+                    if (feature && Object.keys(feature).length > 0) {
+                        debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
+                        currentCacheEntry = feature;
+                        this.taskCache.updateOrAppendCacheRecord({
+                            type: 'locate',
+                            prompt: cachePrompt,
+                            cache: feature
+                        }, locateCacheRecord);
+                    } else debug('no cache data returned, skip cache update, prompt: %s', cachePrompt);
+                } catch (error) {
+                    debug('cacheFeatureForRect failed: %s', error);
+                }
+                else debug('cacheFeatureForRect is not supported, skip cache update');
+                if (!element) {
+                    if (locateDump) throw new InsightError(`Element not found: ${param.prompt}`, locateDump);
+                    throw new Error(`Element not found: ${param.prompt}`);
+                }
+                let hitBy;
+                if (userExpectedPathHitFlag) hitBy = {
+                    from: 'User expected path',
+                    context: {
+                        xpath: param.xpath
+                    }
+                };
+                else if (cacheHitFlag) hitBy = {
+                    from: 'Cache',
+                    context: {
+                        cacheEntry,
+                        cacheToSave: currentCacheEntry
+                    }
+                };
+                else if (planHitFlag) hitBy = {
+                    from: 'Planning',
+                    context: {
+                        id: null == elementFromPlan ? void 0 : elementFromPlan.id,
+                        bbox: null == elementFromPlan ? void 0 : elementFromPlan.bbox
+                    }
+                };
+                null == onResult || onResult(element);
+                return {
+                    output: {
+                        element
+                    },
+                    hitBy
+                };
+            }
         };
+        return taskFind;
     }
     constructor({ interfaceInstance, insight, taskCache }){
         _define_property(this, "interface", void 0);