@godscene/core 1.7.11

package/dist/lib/agent/common.js

@@ -0,0 +1,5 @@
+var __webpack_exports__ = {};
+for(var __rspack_i in __webpack_exports__)exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/agent/execution-session.js

@@ -0,0 +1,73 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    ExecutionSession: ()=>ExecutionSession
+});
+const external_task_runner_js_namespaceObject = require("../task-runner.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class ExecutionSession {
+    async append(tasks, options) {
+        await this.runner.append(tasks, options);
+    }
+    async appendAndRun(tasks, options) {
+        return this.runner.appendAndFlush(tasks, options);
+    }
+    async run(options) {
+        return this.runner.flush(options);
+    }
+    isInErrorState() {
+        return this.runner.isInErrorState();
+    }
+    latestErrorTask() {
+        return this.runner.latestErrorTask();
+    }
+    appendErrorPlan(errorMsg) {
+        return this.runner.appendErrorPlan(errorMsg);
+    }
+    getRunner() {
+        return this.runner;
+    }
+    constructor(name, contextProvider, options){
+        _define_property(this, "runner", void 0);
+        this.runner = new external_task_runner_js_namespaceObject.TaskRunner(name, contextProvider, options);
+    }
+}
+exports.ExecutionSession = __webpack_exports__.ExecutionSession;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "ExecutionSession"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/agent/index.js

@@ -0,0 +1,76 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    typeStr: ()=>external_ui_utils_js_namespaceObject.typeStr,
+    paramStr: ()=>external_ui_utils_js_namespaceObject.paramStr,
+    getReportFileName: ()=>external_utils_js_namespaceObject.getReportFileName,
+    extractInsightParam: ()=>external_ui_utils_js_namespaceObject.extractInsightParam,
+    cacheFileExt: ()=>external_task_cache_js_namespaceObject.cacheFileExt,
+    taskTitleStr: ()=>external_ui_utils_js_namespaceObject.taskTitleStr,
+    commonContextParser: ()=>external_utils_js_namespaceObject.commonContextParser,
+    TaskCache: ()=>external_task_cache_js_namespaceObject.TaskCache,
+    locateParamStr: ()=>external_ui_utils_js_namespaceObject.locateParamStr,
+    createAgent: ()=>external_agent_js_namespaceObject.createAgent,
+    Agent: ()=>external_agent_js_namespaceObject.Agent,
+    TaskExecutor: ()=>external_tasks_js_namespaceObject.TaskExecutor,
+    printReportMsg: ()=>external_utils_js_namespaceObject.printReportMsg
+});
+const external_agent_js_namespaceObject = require("./agent.js");
+const external_utils_js_namespaceObject = require("./utils.js");
+const external_ui_utils_js_namespaceObject = require("./ui-utils.js");
+const external_task_cache_js_namespaceObject = require("./task-cache.js");
+const external_tasks_js_namespaceObject = require("./tasks.js");
+exports.Agent = __webpack_exports__.Agent;
+exports.TaskCache = __webpack_exports__.TaskCache;
+exports.TaskExecutor = __webpack_exports__.TaskExecutor;
+exports.cacheFileExt = __webpack_exports__.cacheFileExt;
+exports.commonContextParser = __webpack_exports__.commonContextParser;
+exports.createAgent = __webpack_exports__.createAgent;
+exports.extractInsightParam = __webpack_exports__.extractInsightParam;
+exports.getReportFileName = __webpack_exports__.getReportFileName;
+exports.locateParamStr = __webpack_exports__.locateParamStr;
+exports.paramStr = __webpack_exports__.paramStr;
+exports.printReportMsg = __webpack_exports__.printReportMsg;
+exports.taskTitleStr = __webpack_exports__.taskTitleStr;
+exports.typeStr = __webpack_exports__.typeStr;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "Agent",
+    "TaskCache",
+    "TaskExecutor",
+    "cacheFileExt",
+    "commonContextParser",
+    "createAgent",
+    "extractInsightParam",
+    "getReportFileName",
+    "locateParamStr",
+    "paramStr",
+    "printReportMsg",
+    "taskTitleStr",
+    "typeStr"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/agent/task-builder.js

@@ -0,0 +1,380 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    locatePlanForLocate: ()=>locatePlanForLocate,
+    TaskBuilder: ()=>TaskBuilder
+});
+const index_js_namespaceObject = require("../ai-model/index.js");
+const external_task_timing_js_namespaceObject = require("../task-timing.js");
+const external_types_js_namespaceObject = require("../types.js");
+const external_utils_js_namespaceObject = require("../utils.js");
+const extractor_namespaceObject = require("@godscene/shared/extractor");
+const logger_namespaceObject = require("@godscene/shared/logger");
+const utils_namespaceObject = require("@godscene/shared/utils");
+const external_utils_js_namespaceObject_1 = require("./utils.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = (0, logger_namespaceObject.getDebug)('agent:task-builder');
+function hasNonEmptyCache(cache) {
+    return null != cache && 'object' == typeof cache && Object.keys(cache).length > 0;
+}
+function locatePlanForLocate(param) {
+    const locate = 'string' == typeof param ? {
+        prompt: param
+    } : param;
+    const locatePlan = {
+        type: 'Locate',
+        param: locate,
+        thought: ''
+    };
+    return locatePlan;
+}
+class TaskBuilder {
+    async build(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options) {
+        const tasks = [];
+        const cacheable = options?.cacheable;
+        const context = {
+            tasks,
+            modelConfigForPlanning,
+            modelConfigForDefaultIntent,
+            cacheable,
+            deepLocate: options?.deepLocate,
+            abortSignal: options?.abortSignal
+        };
+        const planHandlers = new Map([
+            [
+                'Locate',
+                (plan)=>this.handleLocatePlan(plan, context)
+            ],
+            [
+                'Finished',
+                (plan)=>this.handleFinishedPlan(plan, context)
+            ]
+        ]);
+        const defaultHandler = (plan)=>this.handleActionPlan(plan, context);
+        for (const plan of plans){
+            const handler = planHandlers.get(plan.type) ?? defaultHandler;
+            await handler(plan);
+        }
+        return {
+            tasks
+        };
+    }
+    handleFinishedPlan(plan, context) {
+        const taskActionFinished = {
+            type: 'Action Space',
+            subType: 'Finished',
+            param: null,
+            thought: plan.thought,
+            executor: async ()=>{}
+        };
+        context.tasks.push(taskActionFinished);
+    }
+    async handleLocatePlan(plan, context) {
+        const taskLocate = this.createLocateTask(plan, plan.param, context);
+        context.tasks.push(taskLocate);
+    }
+    async handleActionPlan(plan, context) {
+        const planType = plan.type;
+        const actionSpace = this.actionSpace;
+        const action = actionSpace.find((item)=>item.name === planType);
+        const param = plan.param;
+        if (!action) throw new Error(`Action type '${planType}' not found`);
+        const locateFields = action ? (0, index_js_namespaceObject.findAllMidsceneLocatorField)(action.paramSchema) : [];
+        const requiredLocateFields = action ? (0, index_js_namespaceObject.findAllMidsceneLocatorField)(action.paramSchema, true) : [];
+        locateFields.forEach((field)=>{
+            if (param[field]) {
+                const locatePlan = locatePlanForLocate(param[field]);
+                debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`, `hasBbox=${(0, external_utils_js_namespaceObject_1.ifPlanLocateParamIsBbox)(param[field])}`);
+                const locateTask = this.createLocateTask(locatePlan, param[field], context, (result)=>{
+                    param[field] = result;
+                });
+                context.tasks.push(locateTask);
+            } else {
+                (0, utils_namespaceObject.assert)(!requiredLocateFields.includes(field), `Required locate field '${field}' is not provided for action ${planType}`);
+                debug(`field '${field}' is not provided for action ${planType}`);
+            }
+        });
+        const task = {
+            type: 'Action Space',
+            subType: planType,
+            thought: plan.thought,
+            param: plan.param,
+            executor: async (param, taskContext)=>{
+                const timing = taskContext.task.timing;
+                debug('executing action', planType, param, `taskContext.element.center: ${taskContext.element?.center}`);
+                const uiContext = taskContext.uiContext;
+                (0, utils_namespaceObject.assert)(uiContext, 'uiContext is required for Action task');
+                requiredLocateFields.forEach((field)=>{
+                    (0, utils_namespaceObject.assert)(param[field], `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`);
+                });
+                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'beforeInvokeActionHookStart');
+                const delayBeforeRunner = action.delayBeforeRunner ?? 200;
+                try {
+                    await Promise.all([
+                        (async ()=>{
+                            if (this.interface.beforeInvokeAction) {
+                                debug(`will call "beforeInvokeAction" for interface with action name ${action.name}`);
+                                await this.interface.beforeInvokeAction(action.name, param);
+                                debug(`called "beforeInvokeAction" for interface with action name ${action.name}`);
+                            }
+                        })(),
+                        delayBeforeRunner > 0 ? (0, external_utils_js_namespaceObject.sleep)(delayBeforeRunner) : Promise.resolve()
+                    ]);
+                } catch (originalError) {
+                    const originalMessage = originalError?.message || String(originalError);
+                    throw new Error(`error in running beforeInvokeAction for ${action.name}: ${originalMessage}`, {
+                        cause: originalError
+                    });
+                }
+                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'beforeInvokeActionHookEnd');
+                const { shrunkShotToLogicalRatio } = uiContext;
+                if (void 0 === shrunkShotToLogicalRatio) throw new Error('shrunkShotToLogicalRatio is not defined in Action task');
+                if (action.paramSchema) try {
+                    param = (0, index_js_namespaceObject.parseActionParam)(param, action.paramSchema, {
+                        shrunkShotToLogicalRatio
+                    });
+                } catch (error) {
+                    throw new Error(`Invalid parameters for action ${action.name}: ${error.message}\nParameters: ${JSON.stringify(param)}`, {
+                        cause: error
+                    });
+                }
+                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'callActionStart');
+                debug('calling action', action.name);
+                const actionFn = action.call.bind(this.interface);
+                const actionResult = await actionFn(param, taskContext);
+                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'callActionEnd');
+                debug('called action', action.name, 'result:', actionResult);
+                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'afterInvokeActionHookStart');
+                const delayAfterRunner = action.delayAfterRunner ?? this.waitAfterAction ?? 300;
+                if (delayAfterRunner > 0) await (0, external_utils_js_namespaceObject.sleep)(delayAfterRunner);
+                try {
+                    if (this.interface.afterInvokeAction) {
+                        debug(`will call "afterInvokeAction" for interface with action name ${action.name}`);
+                        await this.interface.afterInvokeAction(action.name, param);
+                        debug(`called "afterInvokeAction" for interface with action name ${action.name}`);
+                    }
+                } catch (originalError) {
+                    const originalMessage = originalError?.message || String(originalError);
+                    throw new Error(`error in running afterInvokeAction for ${action.name}: ${originalMessage}`, {
+                        cause: originalError
+                    });
+                }
+                (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'afterInvokeActionHookEnd');
+                return {
+                    output: actionResult
+                };
+            }
+        };
+        context.tasks.push(task);
+    }
+    createLocateTask(plan, detailedLocateParam, context, onResult) {
+        const { cacheable, modelConfigForDefaultIntent, deepLocate, abortSignal } = context;
+        let locateParam = detailedLocateParam;
+        if ('string' == typeof locateParam) locateParam = {
+            prompt: locateParam
+        };
+        if (void 0 !== cacheable) locateParam = {
+            ...locateParam,
+            cacheable
+        };
+        if (deepLocate && !locateParam.deepLocate) locateParam = {
+            ...locateParam,
+            deepLocate: true
+        };
+        const taskLocator = {
+            type: 'Planning',
+            subType: 'Locate',
+            param: locateParam,
+            thought: plan.thought,
+            executor: async (param, taskContext)=>{
+                const { task } = taskContext;
+                let { uiContext } = taskContext;
+                (0, utils_namespaceObject.assert)(param?.prompt || param?.bbox, `No prompt or id or position or bbox to locate, param=${JSON.stringify(param)}`);
+                if (!uiContext) uiContext = await this.service.contextRetrieverFn();
+                (0, utils_namespaceObject.assert)(uiContext, 'uiContext is required for Service task');
+                const { shrunkShotToLogicalRatio } = uiContext;
+                if (void 0 === shrunkShotToLogicalRatio) throw new Error('shrunkShotToLogicalRatio is not defined in locate task');
+                let locateDump;
+                let locateResult;
+                const applyDump = (dump)=>{
+                    if (!dump) return;
+                    locateDump = dump;
+                    task.log = {
+                        dump,
+                        rawResponse: dump.taskInfo?.rawResponse
+                    };
+                    task.usage = dump.taskInfo?.usage;
+                    if (dump.taskInfo?.searchAreaUsage) task.searchAreaUsage = dump.taskInfo.searchAreaUsage;
+                    if (dump.taskInfo?.reasoning_content) task.reasoning_content = dump.taskInfo.reasoning_content;
+                };
+                const planLocatedElement = (0, external_utils_js_namespaceObject_1.ifPlanLocateParamIsBbox)(param) ? (0, external_utils_js_namespaceObject_1.matchElementFromPlan)(param) : void 0;
+                const elementFromBbox = param.deepLocate ? void 0 : planLocatedElement;
+                const isPlanHit = !!elementFromBbox;
+                let rectFromXpath;
+                if (!isPlanHit && param.xpath && this.interface.rectMatchesCacheFeature) try {
+                    rectFromXpath = await this.interface.rectMatchesCacheFeature({
+                        xpaths: [
+                            param.xpath
+                        ]
+                    });
+                } catch  {}
+                const elementFromXpath = rectFromXpath ? (0, extractor_namespaceObject.generateElementByRect)((0, external_utils_js_namespaceObject_1.transformLogicalRectToScreenshotRect)(rectFromXpath, shrunkShotToLogicalRatio), 'string' == typeof param.prompt ? param.prompt : param.prompt?.prompt || '') : void 0;
+                const isXpathHit = !!elementFromXpath;
+                const cachePrompt = param.prompt;
+                const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
+                const cacheEntry = locateCacheRecord?.cacheContent?.cache;
+                const elementFromCacheResult = isPlanHit || isXpathHit ? null : await (0, external_utils_js_namespaceObject_1.matchElementFromCache)({
+                    taskCache: this.taskCache,
+                    interfaceInstance: this.interface
+                }, cacheEntry, cachePrompt, param.cacheable);
+                const elementFromCache = elementFromCacheResult ? (0, external_utils_js_namespaceObject_1.transformLogicalElementToScreenshot)(elementFromCacheResult, shrunkShotToLogicalRatio) : void 0;
+                const isCacheHit = !!elementFromCache;
+                let elementFromAiLocate;
+                const timing = taskContext.task.timing;
+                if (!isXpathHit && !isCacheHit && !isPlanHit) try {
+                    (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'callAiStart');
+                    locateResult = await this.service.locate(param, {
+                        context: uiContext,
+                        planLocatedElement
+                    }, modelConfigForDefaultIntent, abortSignal);
+                    applyDump(locateResult.dump);
+                    elementFromAiLocate = locateResult.element;
+                } catch (error) {
+                    if (error instanceof external_types_js_namespaceObject.ServiceError) applyDump(error.dump);
+                    throw error;
+                } finally{
+                    (0, external_task_timing_js_namespaceObject.setTimingFieldOnce)(timing, 'callAiEnd');
+                }
+                const element = elementFromBbox || elementFromXpath || elementFromCache || elementFromAiLocate;
+                const locateCacheAlreadyExists = hasNonEmptyCache(locateCacheRecord?.cacheContent?.cache);
+                let currentCacheEntry;
+                if (element && this.taskCache && !isCacheHit && (!isPlanHit || !locateCacheAlreadyExists) && param?.cacheable !== false) if (this.interface.cacheFeatureForPoint) try {
+                    let pointForCache = element.center;
+                    if (1 !== shrunkShotToLogicalRatio) {
+                        pointForCache = [
+                            Math.round(element.center[0] / shrunkShotToLogicalRatio),
+                            Math.round(element.center[1] / shrunkShotToLogicalRatio)
+                        ];
+                        debug('Transformed coordinates for cacheFeatureForPoint: %o -> %o', element.center, pointForCache);
+                    }
+                    const feature = await this.interface.cacheFeatureForPoint(pointForCache, {
+                        targetDescription: 'string' == typeof param.prompt ? param.prompt : param.prompt?.prompt,
+                        modelConfig: modelConfigForDefaultIntent
+                    });
+                    if (hasNonEmptyCache(feature)) {
+                        debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
+                        currentCacheEntry = feature;
+                        this.taskCache.updateOrAppendCacheRecord({
+                            type: 'locate',
+                            prompt: cachePrompt,
+                            cache: feature
+                        }, locateCacheRecord);
+                    } else debug('no cache data returned, skip cache update, prompt: %s', cachePrompt);
+                } catch (error) {
+                    debug('cacheFeatureForPoint failed: %s', error);
+                }
+                else debug('cacheFeatureForPoint is not supported, skip cache update');
+                if (!element) {
+                    if (locateDump) throw new external_types_js_namespaceObject.ServiceError(`Element not found : ${param.prompt}`, locateDump);
+                    throw new Error(`Element not found: ${param.prompt}`);
+                }
+                let hitBy;
+                if (isPlanHit) hitBy = {
+                    from: 'Plan',
+                    context: {
+                        bbox: param.bbox
+                    }
+                };
+                else if (isXpathHit) hitBy = {
+                    from: 'User expected path',
+                    context: {
+                        xpath: param.xpath
+                    }
+                };
+                else if (isCacheHit) hitBy = {
+                    from: 'Cache',
+                    context: {
+                        cacheEntry,
+                        cacheToSave: currentCacheEntry
+                    }
+                };
+                if (this.interface.getElementFromPoint && element.center) try {
+                    const [x, y] = element.center;
+                    const data = await this.interface.getElementFromPoint({
+                        x,
+                        y
+                    });
+                    element.allPaths = data?.allPaths || [];
+                    element.containerPaths = data?.containerPaths || [];
+                } catch (error) {
+                    element.allPaths = [];
+                    element.containerPaths = [];
+                }
+                onResult?.(element);
+                return {
+                    output: {
+                        element: {
+                            ...element,
+                            dpr: uiContext.deprecatedDpr
+                        }
+                    },
+                    hitBy
+                };
+            }
+        };
+        return taskLocator;
+    }
+    constructor({ interfaceInstance, service, taskCache, actionSpace, waitAfterAction }){
+        _define_property(this, "interface", void 0);
+        _define_property(this, "service", void 0);
+        _define_property(this, "taskCache", void 0);
+        _define_property(this, "actionSpace", void 0);
+        _define_property(this, "waitAfterAction", void 0);
+        this.interface = interfaceInstance;
+        this.service = service;
+        this.taskCache = taskCache;
+        this.actionSpace = actionSpace;
+        this.waitAfterAction = waitAfterAction;
+    }
+}
+exports.TaskBuilder = __webpack_exports__.TaskBuilder;
+exports.locatePlanForLocate = __webpack_exports__.locatePlanForLocate;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "TaskBuilder",
+    "locatePlanForLocate"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});