@godscene/core 1.7.11

package/dist/es/agent/tasks.mjs

@@ -0,0 +1,428 @@
+import { AIResponseParseError, ConversationHistory, autoGLMPlanning, plan, uiTarsPlanning } from "../ai-model/index.mjs";
+import { isAutoGLM, isUITars } from "../ai-model/auto-glm/util.mjs";
+import { getReadableTimeString } from "../common.mjs";
+import { TaskExecutionError } from "../task-runner.mjs";
+import { ServiceError } from "../types.mjs";
+import { getDebug } from "@godscene/shared/logger";
+import { assert } from "@godscene/shared/utils";
+import { ExecutionSession } from "./execution-session.mjs";
+import { TaskBuilder, locatePlanForLocate } from "./task-builder.mjs";
+import { setTimingFieldOnce } from "../task-timing.mjs";
+import { descriptionOfTree } from "@godscene/shared/extractor";
+import { taskTitleStr } from "./ui-utils.mjs";
+import { parsePrompt } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('device-task-executor');
+const warnLog = getDebug('device-task-executor', {
+    console: true
+});
+const maxErrorCountAllowedInOnePlanningLoop = 5;
+class TaskExecutor {
+    get page() {
+        return this.interface;
+    }
+    createExecutionSession(title, options) {
+        return new ExecutionSession(title, ()=>Promise.resolve(this.service.contextRetrieverFn()), {
+            onTaskStart: this.onTaskStartCallback,
+            tasks: options?.tasks,
+            onTaskUpdate: this.hooks?.onTaskUpdate
+        });
+    }
+    getActionSpace() {
+        return this.providedActionSpace;
+    }
+    async getTimeString(format) {
+        if (this.useDeviceTime) if (this.interface.getDeviceLocalTimeString) try {
+            return await this.interface.getDeviceLocalTimeString(format);
+        } catch (error) {
+            warnLog(`Failed to get device time string, falling back to runtime time: ${error}`);
+        }
+        else warnLog('useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.');
+        return getReadableTimeString(format);
+    }
+    async convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options) {
+        return this.taskBuilder.build(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options);
+    }
+    async loadYamlFlowAsPlanning(userInstruction, yamlString) {
+        const session = this.createExecutionSession(taskTitleStr('Act', userInstruction));
+        const task = {
+            type: 'Planning',
+            subType: 'LoadYaml',
+            param: {
+                userInstruction
+            },
+            executor: async (param, executorContext)=>{
+                const { uiContext } = executorContext;
+                assert(uiContext, 'uiContext is required for Planning task');
+                return {
+                    output: {
+                        actions: [],
+                        shouldContinuePlanning: false,
+                        log: '',
+                        yamlString
+                    },
+                    cache: {
+                        hit: true
+                    },
+                    hitBy: {
+                        from: 'Cache',
+                        context: {
+                            yamlString
+                        }
+                    }
+                };
+            }
+        };
+        const runner = session.getRunner();
+        await session.appendAndRun(task);
+        return {
+            runner
+        };
+    }
+    async runPlans(title, plans, modelConfigForPlanning, modelConfigForDefaultIntent) {
+        const session = this.createExecutionSession(title);
+        const { tasks } = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent);
+        const runner = session.getRunner();
+        const result = await session.appendAndRun(tasks);
+        const { output } = result ?? {};
+        return {
+            output,
+            runner
+        };
+    }
+    async action(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, fileChooserAccept, deepLocate, abortSignal) {
+        return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal));
+    }
+    async runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal) {
+        const conversationHistory = new ConversationHistory();
+        const session = this.createExecutionSession(taskTitleStr('Act', userPrompt));
+        const runner = session.getRunner();
+        let replanCount = 0;
+        const yamlFlow = [];
+        const replanningCycleLimit = replanningCycleLimitOverride ?? this.replanningCycleLimit;
+        assert(void 0 !== replanningCycleLimit, 'replanningCycleLimit is required for TaskExecutor.action');
+        let errorCountInOnePlanningLoop = 0;
+        let outputString;
+        while(true){
+            if (abortSignal?.aborted) return session.appendErrorPlan(`Task aborted: ${abortSignal.reason || 'abort signal received'}`);
+            const subGoalStatus = conversationHistory.subGoalsToText() || void 0;
+            const memoriesStatus = conversationHistory.memoriesToText() || void 0;
+            const result = await session.appendAndRun({
+                type: 'Planning',
+                subType: 'Plan',
+                param: {
+                    userInstruction: userPrompt,
+                    aiActContext,
+                    imagesIncludeCount,
+                    deepThink,
+                    ...subGoalStatus ? {
+                        subGoalStatus
+                    } : {},
+                    ...memoriesStatus ? {
+                        memoriesStatus
+                    } : {}
+                },
+                executor: async (param, executorContext)=>{
+                    const { uiContext } = executorContext;
+                    assert(uiContext, 'uiContext is required for Planning task');
+                    const { modelFamily } = modelConfigForPlanning;
+                    const timing = executorContext.task.timing;
+                    const actionSpace = this.getActionSpace();
+                    debug('actionSpace for this interface is:', actionSpace.map((action)=>action.name).join(', '));
+                    assert(Array.isArray(actionSpace), 'actionSpace must be an array');
+                    if (0 === actionSpace.length) console.warn(`ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`);
+                    const planImpl = isUITars(modelFamily) ? uiTarsPlanning : isAutoGLM(modelFamily) ? autoGLMPlanning : plan;
+                    let planResult;
+                    try {
+                        setTimingFieldOnce(timing, 'callAiStart');
+                        planResult = await planImpl(param.userInstruction, {
+                            context: uiContext,
+                            actionContext: param.aiActContext,
+                            interfaceType: this.interface.interfaceType,
+                            actionSpace,
+                            modelConfig: modelConfigForPlanning,
+                            conversationHistory,
+                            includeBbox: includeBboxInPlanning,
+                            imagesIncludeCount,
+                            deepThink,
+                            abortSignal
+                        });
+                    } catch (planError) {
+                        if (planError instanceof AIResponseParseError) {
+                            executorContext.task.usage = planError.usage;
+                            executorContext.task.log = {
+                                ...executorContext.task.log || {},
+                                rawResponse: planError.rawResponse
+                            };
+                        }
+                        throw planError;
+                    } finally{
+                        setTimingFieldOnce(timing, 'callAiEnd');
+                    }
+                    debug('planResult', JSON.stringify(planResult, null, 2));
+                    const { actions, thought, log, memory, error, usage, rawResponse, reasoning_content, finalizeSuccess, finalizeMessage, updateSubGoals, markFinishedIndexes } = planResult;
+                    outputString = finalizeMessage;
+                    executorContext.task.log = {
+                        ...executorContext.task.log || {},
+                        rawResponse
+                    };
+                    executorContext.task.usage = usage;
+                    executorContext.task.reasoning_content = reasoning_content;
+                    executorContext.task.output = {
+                        actions: actions || [],
+                        log,
+                        thought,
+                        memory,
+                        yamlFlow: planResult.yamlFlow,
+                        output: finalizeMessage,
+                        shouldContinuePlanning: planResult.shouldContinuePlanning,
+                        updateSubGoals,
+                        markFinishedIndexes
+                    };
+                    executorContext.uiContext = uiContext;
+                    assert(!error, `Failed to continue: ${error}\n${log || ''}`);
+                    if (false === finalizeSuccess) assert(false, `Task failed: ${finalizeMessage || 'No error message provided'}\n${log || ''}`);
+                    return {
+                        cache: {
+                            hit: false
+                        }
+                    };
+                }
+            }, {
+                allowWhenError: true
+            });
+            const planResult = result?.output;
+            const plans = planResult?.actions || [];
+            yamlFlow.push(...planResult?.yamlFlow || []);
+            let executables;
+            try {
+                executables = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, {
+                    cacheable,
+                    deepLocate,
+                    abortSignal
+                });
+            } catch (error) {
+                return session.appendErrorPlan(`Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(plans)}`);
+            }
+            if (conversationHistory.pendingFeedbackMessage) console.warn('unconsumed pending feedback message detected, this may lead to unexpected planning result:', conversationHistory.pendingFeedbackMessage);
+            const initialTimeString = await this.getTimeString();
+            conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;
+            try {
+                await session.appendAndRun(executables.tasks);
+            } catch (error) {
+                errorCountInOnePlanningLoop++;
+                const timeString = await this.getTimeString();
+                conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message || String(error)}`;
+                debug('error when executing running tasks, but continue to run if it is not too many errors:', error instanceof Error ? error.message : String(error), 'current error count in one planning loop:', errorCountInOnePlanningLoop);
+            }
+            if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) return session.appendErrorPlan('Too many errors in one planning loop');
+            if (abortSignal?.aborted) return session.appendErrorPlan(`Task aborted: ${abortSignal.reason || 'abort signal received'}`);
+            if (!planResult?.shouldContinuePlanning) break;
+            ++replanCount;
+            if (replanCount > replanningCycleLimit) {
+                const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;
+                return session.appendErrorPlan(errorMsg);
+            }
+            if (!conversationHistory.pendingFeedbackMessage) {
+                const timeString = await this.getTimeString();
+                conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;
+            }
+        }
+        return {
+            output: {
+                yamlFlow,
+                output: outputString
+            },
+            runner
+        };
+    }
+    createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt) {
+        const queryTask = {
+            type: 'Insight',
+            subType: type,
+            param: {
+                domIncluded: opt?.domIncluded,
+                dataDemand: multimodalPrompt ? {
+                    demand,
+                    multimodalPrompt
+                } : demand
+            },
+            executor: async (param, taskContext)=>{
+                const { task } = taskContext;
+                let queryDump;
+                const applyDump = (dump)=>{
+                    queryDump = dump;
+                    task.log = {
+                        dump,
+                        rawResponse: dump.taskInfo?.rawResponse
+                    };
+                    task.usage = dump.taskInfo?.usage;
+                    if (dump.taskInfo?.reasoning_content) task.reasoning_content = dump.taskInfo.reasoning_content;
+                };
+                const uiContext = taskContext.uiContext;
+                assert(uiContext, 'uiContext is required for Query task');
+                const ifTypeRestricted = 'Query' !== type;
+                let demandInput = demand;
+                let keyOfResult = 'result';
+                if (ifTypeRestricted && ('Assert' === type || 'WaitFor' === type)) {
+                    keyOfResult = 'StatementIsTruthy';
+                    const booleanPrompt = 'Assert' === type ? `Boolean, whether the following statement is true: ${demand}` : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;
+                    demandInput = {
+                        [keyOfResult]: booleanPrompt
+                    };
+                } else if (ifTypeRestricted) {
+                    keyOfResult = type;
+                    demandInput = {
+                        [keyOfResult]: `${type}, ${demand}`
+                    };
+                }
+                let extractResult;
+                let extraPageDescription = '';
+                if (opt?.domIncluded && this.interface.getElementsNodeTree) {
+                    debug('appending tree info for page');
+                    const tree = await this.interface.getElementsNodeTree();
+                    extraPageDescription = await descriptionOfTree(tree, 200, false, opt?.domIncluded === 'visible-only');
+                }
+                try {
+                    extractResult = await this.service.extract(demandInput, modelConfig, opt, extraPageDescription, multimodalPrompt, uiContext);
+                } catch (error) {
+                    if (error instanceof ServiceError) applyDump(error.dump);
+                    throw error;
+                }
+                const { data, thought, dump } = extractResult;
+                applyDump(dump);
+                let outputResult = data;
+                if (ifTypeRestricted) if ('string' == typeof data) outputResult = data;
+                else if ('WaitFor' === type) outputResult = null == data ? false : data[keyOfResult];
+                else if (null == data) outputResult = null;
+                else if (data?.[keyOfResult] !== void 0) outputResult = data[keyOfResult];
+                else if (data?.result !== void 0) outputResult = data.result;
+                else assert(false, 'No result in query data');
+                if ('Assert' === type && !outputResult) {
+                    task.thought = thought;
+                    throw new Error(`Assertion failed: ${thought}`);
+                }
+                return {
+                    output: outputResult,
+                    log: queryDump,
+                    thought
+                };
+            }
+        };
+        return queryTask;
+    }
+    async createTypeQueryExecution(type, demand, modelConfig, opt, multimodalPrompt) {
+        const session = this.createExecutionSession(taskTitleStr(type, 'string' == typeof demand ? demand : JSON.stringify(demand)));
+        const queryTask = await this.createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt);
+        const runner = session.getRunner();
+        const result = await session.appendAndRun(queryTask);
+        if (!result) throw new Error('result of taskExecutor.flush() is undefined in function createTypeQueryTask');
+        const { output, thought } = result;
+        return {
+            output,
+            thought,
+            runner
+        };
+    }
+    async waitFor(assertion, opt, modelConfig) {
+        const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
+        const description = `waitFor: ${textPrompt}`;
+        const session = this.createExecutionSession(taskTitleStr('WaitFor', description));
+        const runner = session.getRunner();
+        const { timeoutMs, checkIntervalMs, domIncluded, screenshotIncluded, ...restOpt } = opt;
+        const serviceExtractOpt = {
+            domIncluded,
+            screenshotIncluded,
+            ...restOpt
+        };
+        assert(assertion, 'No assertion for waitFor');
+        assert(timeoutMs, 'No timeoutMs for waitFor');
+        assert(checkIntervalMs, 'No checkIntervalMs for waitFor');
+        assert(checkIntervalMs <= timeoutMs, `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`);
+        const overallStartTime = Date.now();
+        let lastCheckStart = overallStartTime;
+        let errorThought = '';
+        while(lastCheckStart - overallStartTime <= timeoutMs){
+            const currentCheckStart = Date.now();
+            lastCheckStart = currentCheckStart;
+            const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, modelConfig, serviceExtractOpt, multimodalPrompt);
+            const result = await session.appendAndRun(queryTask);
+            if (result?.output) return {
+                output: void 0,
+                runner
+            };
+            errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
+            const now = Date.now();
+            if (now - currentCheckStart < checkIntervalMs) {
+                const elapsed = now - currentCheckStart;
+                const timeRemaining = checkIntervalMs - elapsed;
+                const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;
+                const { tasks: sleepTasks } = await this.convertPlanToExecutable([
+                    {
+                        type: 'Sleep',
+                        param: {
+                            timeMs: timeRemaining
+                        },
+                        thought
+                    }
+                ], modelConfig, modelConfig);
+                if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
+            }
+        }
+        return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
+    }
+    constructor(interfaceInstance, service, opts){
+        _define_property(this, "interface", void 0);
+        _define_property(this, "service", void 0);
+        _define_property(this, "taskCache", void 0);
+        _define_property(this, "providedActionSpace", void 0);
+        _define_property(this, "taskBuilder", void 0);
+        _define_property(this, "onTaskStartCallback", void 0);
+        _define_property(this, "hooks", void 0);
+        _define_property(this, "replanningCycleLimit", void 0);
+        _define_property(this, "waitAfterAction", void 0);
+        _define_property(this, "useDeviceTime", void 0);
+        this.interface = interfaceInstance;
+        this.service = service;
+        this.taskCache = opts.taskCache;
+        this.onTaskStartCallback = opts?.onTaskStart;
+        this.replanningCycleLimit = opts.replanningCycleLimit;
+        this.waitAfterAction = opts.waitAfterAction;
+        this.useDeviceTime = opts.useDeviceTime;
+        this.hooks = opts.hooks;
+        this.providedActionSpace = opts.actionSpace;
+        this.taskBuilder = new TaskBuilder({
+            interfaceInstance,
+            service,
+            taskCache: opts.taskCache,
+            actionSpace: this.getActionSpace(),
+            waitAfterAction: opts.waitAfterAction
+        });
+    }
+}
+async function withFileChooser(interfaceInstance, fileChooserAccept, action) {
+    if (!fileChooserAccept?.length) return action();
+    if (!interfaceInstance.registerFileChooserListener) throw new Error(`File upload is not supported on ${interfaceInstance.interfaceType}`);
+    const handler = async (chooser)=>{
+        await chooser.accept(fileChooserAccept);
+    };
+    const { dispose, getError } = await interfaceInstance.registerFileChooserListener(handler);
+    try {
+        const result = await action();
+        const error = getError();
+        if (error) throw error;
+        return result;
+    } finally{
+        dispose();
+    }
+}
+export { TaskExecutionError, TaskExecutor, locatePlanForLocate, withFileChooser };

package/dist/es/agent/ui-utils.mjs

@@ -0,0 +1,101 @@
+function typeStr(task) {
+    return task.subType || task.type;
+}
+function locateParamStr(locate) {
+    if (!locate) return '';
+    if ('string' == typeof locate) return locate;
+    if ('object' == typeof locate) {
+        if ('object' == typeof locate.prompt && null !== locate.prompt && locate.prompt.prompt) {
+            const prompt = locate.prompt.prompt;
+            return prompt;
+        }
+        if ('string' == typeof locate.prompt) return locate.prompt;
+        if ('string' == typeof locate.description) return locate.description;
+    }
+    return '';
+}
+function scrollParamStr(scrollParam) {
+    if (!scrollParam) return '';
+    return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;
+}
+function pullParamStr(pullParam) {
+    if (!pullParam) return '';
+    const parts = [];
+    parts.push(`direction: ${pullParam.direction || 'down'}`);
+    if (pullParam.distance) parts.push(`distance: ${pullParam.distance}`);
+    if (pullParam.duration) parts.push(`duration: ${pullParam.duration}ms`);
+    return parts.join(', ');
+}
+function extractInsightParam(taskParam) {
+    if (!taskParam) return {
+        content: ''
+    };
+    const extractImages = (source)=>source?.multimodalPrompt?.images && Array.isArray(source.multimodalPrompt.images) ? source.multimodalPrompt.images : void 0;
+    const toContent = (value)=>'string' == typeof value ? value : JSON.stringify(value);
+    if (taskParam.demand) return {
+        content: toContent(taskParam.demand),
+        images: extractImages(taskParam)
+    };
+    if (taskParam.assertion) return {
+        content: toContent(taskParam.assertion),
+        images: extractImages(taskParam)
+    };
+    if (taskParam.dataDemand) {
+        const { dataDemand } = taskParam;
+        if ('string' == typeof dataDemand) return {
+            content: dataDemand
+        };
+        if ('object' == typeof dataDemand) return {
+            content: toContent(dataDemand.demand || dataDemand),
+            images: extractImages(dataDemand)
+        };
+    }
+    return {
+        content: ''
+    };
+}
+function taskTitleStr(type, prompt) {
+    if (prompt) return `${type} - ${prompt}`;
+    return type;
+}
+function paramStr(task) {
+    let value;
+    if ('Planning' === task.type) if ('Locate' === task.subType) value = locateParamStr(task?.param);
+    else {
+        const planTask = task;
+        value = planTask.output?.log || planTask.param?.userInstruction;
+    }
+    if ('Insight' === task.type) value = extractInsightParam(task?.param).content;
+    if ('Action Space' === task.type) {
+        const locate = task?.param?.locate;
+        const locateStr = locate ? locateParamStr(locate) : '';
+        value = task.thought || '';
+        if ('number' == typeof task?.param?.timeMs) value = `${task?.param?.timeMs}ms`;
+        else if ('string' == typeof task?.param?.scrollType) value = scrollParamStr(task?.param);
+        else if ('string' == typeof task?.param?.direction && task?.subType === 'PullGesture') value = pullParamStr(task?.param);
+        else if (void 0 !== task?.param?.value) value = task?.param?.value;
+        else if (task?.param && 'object' == typeof task?.param && Object.keys(task?.param || {}).length > 0) value = task?.param;
+        if (locateStr) value = value && 'object' != typeof value ? `${locateStr} - ${value}` : locateStr;
+    }
+    if (void 0 === value) return '';
+    if ('string' == typeof value) return value;
+    if ('object' == typeof value) {
+        const locateStr = locateParamStr(value);
+        if (locateStr) return locateStr;
+        const entries = Object.entries(value);
+        if (0 === entries.length) return '';
+        const formatValue = (v)=>{
+            if ('string' == typeof v) return v;
+            if (null == v) return String(v);
+            if ('object' == typeof v) return JSON.stringify(v);
+            return String(v);
+        };
+        if (1 === entries.length) {
+            const [key, v] = entries[0];
+            return `${key}: ${formatValue(v)}`;
+        }
+        return entries.map(([key, v])=>`${key}: ${formatValue(v)}`).join(', ');
+    }
+    return String(value);
+}
+export { extractInsightParam, locateParamStr, paramStr, pullParamStr, scrollParamStr, taskTitleStr, typeStr };