@godscene/core 1.7.11

package/dist/es/service/index.mjs

@@ -0,0 +1,272 @@
+import { isAutoGLM } from "../ai-model/auto-glm/util.mjs";
+import { AIResponseParseError, AiExtractElementInfo, AiLocateElement, callAIWithObjectResponse } from "../ai-model/index.mjs";
+import { AiLocateSection, buildSearchAreaConfig } from "../ai-model/inspect.mjs";
+import { elementDescriberInstruction } from "../ai-model/prompt/describe.mjs";
+import { expandSearchArea } from "../common.mjs";
+import { ServiceError } from "../types.mjs";
+import { compositeElementInfoImg, cropByRect } from "@godscene/shared/img";
+import { getDebug } from "@godscene/shared/logger";
+import { assert } from "@godscene/shared/utils";
+import { createServiceDump } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('ai:service');
+class Service {
+    async locate(query, opt, modelConfig, abortSignal) {
+        const queryPrompt = 'string' == typeof query ? query : query.prompt;
+        assert(queryPrompt, 'query is required for locate');
+        assert('object' == typeof query, 'query should be an object for locate');
+        const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;
+        let searchAreaPrompt;
+        if (query.deepLocate && !hasPlanLocatedElement) searchAreaPrompt = query.prompt;
+        const { modelFamily } = modelConfig;
+        if (searchAreaPrompt && !modelFamily) {
+            console.warn('The "deepLocate" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config');
+            searchAreaPrompt = void 0;
+        }
+        if (searchAreaPrompt && isAutoGLM(modelFamily)) {
+            console.warn('The "deepLocate" feature is not supported with AutoGLM.');
+            searchAreaPrompt = void 0;
+        }
+        const context = opt?.context || await this.contextRetrieverFn();
+        let searchArea;
+        let searchAreaRawResponse;
+        let searchAreaUsage;
+        let searchAreaResponse;
+        if (query.deepLocate && hasPlanLocatedElement) {
+            const searchAreaConfig = await buildSearchAreaConfig({
+                context,
+                baseRect: opt.planLocatedElement.rect,
+                modelFamily
+            });
+            searchArea = searchAreaConfig.rect;
+            searchAreaRawResponse = JSON.stringify({
+                source: 'plan-located-element',
+                rect: opt.planLocatedElement.rect
+            });
+            searchAreaResponse = {
+                rect: searchArea,
+                imageBase64: searchAreaConfig.imageBase64,
+                scale: searchAreaConfig.scale,
+                rawResponse: searchAreaRawResponse
+            };
+        } else if (searchAreaPrompt) {
+            searchAreaResponse = await AiLocateSection({
+                context,
+                sectionDescription: searchAreaPrompt,
+                modelConfig,
+                abortSignal
+            });
+            assert(searchAreaResponse.rect, `cannot find search area for "${searchAreaPrompt}"${searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''}`);
+            searchAreaRawResponse = searchAreaResponse.rawResponse;
+            searchAreaUsage = searchAreaResponse.usage;
+            searchArea = searchAreaResponse.rect;
+        }
+        const startTime = Date.now();
+        const { parseResult, rect, rawResponse, usage, reasoning_content } = await AiLocateElement({
+            context,
+            targetElementDescription: queryPrompt,
+            searchConfig: searchAreaResponse,
+            modelConfig,
+            abortSignal
+        });
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse: JSON.stringify(rawResponse),
+            formatResponse: JSON.stringify(parseResult),
+            usage,
+            searchArea,
+            searchAreaRawResponse,
+            searchAreaUsage,
+            reasoning_content
+        };
+        let errorLog;
+        if (parseResult.errors?.length) errorLog = `failed to locate element: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'locate',
+            userQuery: {
+                element: queryPrompt
+            },
+            matchedElement: [],
+            matchedRect: rect,
+            data: null,
+            taskInfo,
+            deepLocate: !!searchArea,
+            error: errorLog
+        };
+        const elements = parseResult.elements || [];
+        const dump = createServiceDump({
+            ...dumpData,
+            matchedElement: elements
+        });
+        if (errorLog) throw new ServiceError(errorLog, dump);
+        if (elements.length > 1) throw new ServiceError(`locate: multiple elements found, length = ${elements.length}`, dump);
+        if (1 === elements.length) return {
+            element: {
+                center: elements[0].center,
+                rect: elements[0].rect,
+                description: elements[0].description
+            },
+            rect,
+            dump
+        };
+        return {
+            element: null,
+            rect,
+            dump
+        };
+    }
+    async extract(dataDemand, modelConfig, opt, pageDescription, multimodalPrompt, context) {
+        assert(context, 'context is required for extract');
+        assert('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
+        const startTime = Date.now();
+        let parseResult;
+        let rawResponse;
+        let usage;
+        let reasoning_content;
+        try {
+            const result = await AiExtractElementInfo({
+                context,
+                dataQuery: dataDemand,
+                multimodalPrompt,
+                extractOption: opt,
+                modelConfig,
+                pageDescription
+            });
+            parseResult = result.parseResult;
+            rawResponse = result.rawResponse;
+            usage = result.usage;
+            reasoning_content = result.reasoning_content;
+        } catch (error) {
+            if (error instanceof AIResponseParseError) {
+                const timeCost = Date.now() - startTime;
+                const taskInfo = {
+                    ...this.taskInfo ? this.taskInfo : {},
+                    durationMs: timeCost,
+                    rawResponse: error.rawResponse,
+                    usage: error.usage
+                };
+                const dump = createServiceDump({
+                    type: 'extract',
+                    userQuery: {
+                        dataDemand
+                    },
+                    matchedElement: [],
+                    data: null,
+                    taskInfo,
+                    error: error.message
+                });
+                throw new ServiceError(error.message, dump);
+            }
+            throw error;
+        }
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse,
+            formatResponse: JSON.stringify(parseResult),
+            usage,
+            reasoning_content
+        };
+        let errorLog;
+        if (parseResult.errors?.length) errorLog = `AI response error: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'extract',
+            userQuery: {
+                dataDemand
+            },
+            matchedElement: [],
+            data: null,
+            taskInfo,
+            error: errorLog
+        };
+        const { data, thought } = parseResult || {};
+        const dump = createServiceDump({
+            ...dumpData,
+            data
+        });
+        if (errorLog && !data) throw new ServiceError(errorLog, dump);
+        return {
+            data,
+            thought,
+            usage,
+            reasoning_content,
+            dump
+        };
+    }
+    async describe(target, modelConfig, opt) {
+        assert(target, 'target is required for service.describe');
+        const context = await this.contextRetrieverFn();
+        const { shotSize } = context;
+        const screenshotBase64 = context.screenshot.base64;
+        assert(screenshotBase64, 'screenshot is required for service.describe');
+        const { modelFamily } = modelConfig;
+        const systemPrompt = elementDescriberInstruction();
+        const defaultRectSize = 30;
+        const targetRect = Array.isArray(target) ? {
+            left: Math.floor(target[0] - defaultRectSize / 2),
+            top: Math.floor(target[1] - defaultRectSize / 2),
+            width: defaultRectSize,
+            height: defaultRectSize
+        } : target;
+        let imagePayload = await compositeElementInfoImg({
+            inputImgBase64: screenshotBase64,
+            size: shotSize,
+            elementsPositionInfo: [
+                {
+                    rect: targetRect
+                }
+            ],
+            borderThickness: 3
+        });
+        if (opt?.deepLocate) {
+            const searchArea = expandSearchArea(targetRect, shotSize);
+            debug('describe: cropping to searchArea', searchArea);
+            const croppedResult = await cropByRect(imagePayload, searchArea, 'qwen2.5-vl' === modelFamily);
+            imagePayload = croppedResult.imageBase64;
+        }
+        const msgs = [
+            {
+                role: 'system',
+                content: systemPrompt
+            },
+            {
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: imagePayload,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            }
+        ];
+        const res = await callAIWithObjectResponse(msgs, modelConfig);
+        const { content } = res;
+        assert(!content.error, `describe failed: ${content.error}`);
+        assert(content.description, 'failed to describe the element');
+        return content;
+    }
+    constructor(context, opt){
+        _define_property(this, "contextRetrieverFn", void 0);
+        _define_property(this, "taskInfo", void 0);
+        assert(context, 'context is required for Service');
+        if ('function' == typeof context) this.contextRetrieverFn = context;
+        else this.contextRetrieverFn = ()=>Promise.resolve(context);
+        if (void 0 !== opt?.taskInfo) this.taskInfo = opt.taskInfo;
+    }
+}
+export { Service as default };

package/dist/es/service/utils.mjs

@@ -0,0 +1,13 @@
+import { uuid } from "@godscene/shared/utils";
+function createServiceDump(data) {
+    const baseData = {
+        logTime: Date.now()
+    };
+    const finalData = {
+        logId: uuid(),
+        ...baseData,
+        ...data
+    };
+    return finalData;
+}
+export { createServiceDump };

package/dist/es/skill/index.mjs

@@ -0,0 +1,35 @@
+import { reportCLIError, runToolsCLI } from "@godscene/shared/cli";
+import { BaseMidsceneTools } from "@godscene/shared/mcp/base-tools";
+import { Agent } from "../agent/agent.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class SkillMidsceneTools extends BaseMidsceneTools {
+    createTemporaryDevice() {
+        return new this.DeviceClass();
+    }
+    async ensureAgent() {
+        if (!this.agent) {
+            const device = new this.DeviceClass();
+            this.agent = new Agent(device);
+        }
+        return this.agent;
+    }
+    constructor(DeviceClass){
+        super(), _define_property(this, "DeviceClass", void 0), this.DeviceClass = DeviceClass;
+    }
+}
+function runSkillCLI(options) {
+    const tools = new SkillMidsceneTools(options.DeviceClass);
+    return runToolsCLI(tools, options.scriptName).catch((e)=>{
+        process.exit(reportCLIError(e));
+    });
+}
+export { runSkillCLI };

package/dist/es/task-runner.mjs

@@ -0,0 +1,261 @@
+import { setTimingFieldOnce } from "./task-timing.mjs";
+import { ExecutionDump } from "./types.mjs";
+import { getDebug } from "@godscene/shared/logger";
+import { assert, uuid } from "@godscene/shared/utils";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('task-runner');
+const UI_CONTEXT_CACHE_TTL_MS = 300;
+class TaskRunner {
+    async emitOnTaskUpdate(error) {
+        if (!this.onTaskUpdate) return;
+        await this.onTaskUpdate(this, error);
+    }
+    async getUiContext(options) {
+        const now = Date.now();
+        const shouldReuse = !options?.forceRefresh && this.lastUiContext && now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;
+        if (shouldReuse && this.lastUiContext?.context) {
+            debug(`reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`);
+            return this.lastUiContext?.context;
+        }
+        try {
+            const uiContext = await this.uiContextBuilder();
+            if (uiContext) this.lastUiContext = {
+                context: uiContext,
+                capturedAt: Date.now()
+            };
+            else this.lastUiContext = void 0;
+            return uiContext;
+        } catch (error) {
+            this.lastUiContext = void 0;
+            throw error;
+        }
+    }
+    async captureScreenshot() {
+        try {
+            const uiContext = await this.getUiContext({
+                forceRefresh: true
+            });
+            return uiContext?.screenshot;
+        } catch (error) {
+            console.error('error while capturing screenshot', error);
+        }
+    }
+    attachRecorderItem(task, screenshot, phase) {
+        if (!phase || !screenshot) return;
+        const recorderItem = {
+            type: 'screenshot',
+            ts: Date.now(),
+            screenshot,
+            timing: phase
+        };
+        if (!task.recorder) {
+            task.recorder = [
+                recorderItem
+            ];
+            return;
+        }
+        task.recorder.push(recorderItem);
+    }
+    markTaskAsPending(task) {
+        return {
+            taskId: uuid(),
+            status: 'pending',
+            ...task
+        };
+    }
+    normalizeStatusFromError(options, errorMessage) {
+        if ('error' !== this.status) return;
+        assert(options?.allowWhenError, errorMessage || `task runner is in error state, cannot proceed\nerror=${this.latestErrorTask()?.error}\n${this.latestErrorTask()?.errorStack}`);
+        this.status = this.tasks.length > 0 ? 'pending' : 'init';
+    }
+    async append(task, options) {
+        this.normalizeStatusFromError(options, `task runner is in error state, cannot append task\nerror=${this.latestErrorTask()?.error}\n${this.latestErrorTask()?.errorStack}`);
+        if (Array.isArray(task)) this.tasks.push(...task.map((item)=>this.markTaskAsPending(item)));
+        else this.tasks.push(this.markTaskAsPending(task));
+        if ('running' !== this.status) this.status = 'pending';
+        await this.emitOnTaskUpdate();
+    }
+    async appendAndFlush(task, options) {
+        await this.append(task, options);
+        return this.flush(options);
+    }
+    async flush(options) {
+        if ('init' === this.status && this.tasks.length > 0) console.warn('illegal state for task runner, status is init but tasks are not empty');
+        this.normalizeStatusFromError(options, 'task runner is in error state');
+        assert('running' !== this.status, 'task runner is already running');
+        assert('completed' !== this.status, 'task runner is already completed');
+        const nextPendingIndex = this.tasks.findIndex((task)=>'pending' === task.status);
+        if (nextPendingIndex < 0) return;
+        this.status = 'running';
+        await this.emitOnTaskUpdate();
+        let taskIndex = nextPendingIndex;
+        let successfullyCompleted = true;
+        let previousFindOutput;
+        while(taskIndex < this.tasks.length){
+            const task = this.tasks[taskIndex];
+            assert('pending' === task.status, `task status should be pending, but got: ${task.status}`);
+            task.timing = {
+                start: Date.now()
+            };
+            try {
+                task.status = 'running';
+                await this.emitOnTaskUpdate();
+                try {
+                    if (this.onTaskStart) await this.onTaskStart(task);
+                } catch (e) {
+                    console.error('error in onTaskStart', e);
+                }
+                assert([
+                    'Insight',
+                    'Action Space',
+                    'Planning'
+                ].indexOf(task.type) >= 0, `unsupported task type: ${task.type}`);
+                const { executor, param } = task;
+                assert(executor, `executor is required for task type: ${task.type}`);
+                let returnValue;
+                const forceRefresh = 'Insight' === task.type;
+                setTimingFieldOnce(task.timing, 'getUiContextStart');
+                const uiContext = await this.getUiContext({
+                    forceRefresh
+                });
+                setTimingFieldOnce(task.timing, 'getUiContextEnd');
+                task.uiContext = uiContext;
+                const executorContext = {
+                    task,
+                    element: previousFindOutput?.element,
+                    uiContext
+                };
+                if ('Insight' === task.type) {
+                    assert('Query' === task.subType || 'Assert' === task.subType || 'WaitFor' === task.subType || 'Boolean' === task.subType || 'Number' === task.subType || 'String' === task.subType, `unsupported service subType: ${task.subType}`);
+                    returnValue = await task.executor(param, executorContext);
+                } else if ('Planning' === task.type) {
+                    returnValue = await task.executor(param, executorContext);
+                    if ('Locate' === task.subType) previousFindOutput = returnValue?.output;
+                } else if ('Action Space' === task.type) returnValue = await task.executor(param, executorContext);
+                else {
+                    console.warn(`unsupported task type: ${task.type}, will try to execute it directly`);
+                    returnValue = await task.executor(param, executorContext);
+                }
+                const isLastTask = taskIndex === this.tasks.length - 1;
+                if (isLastTask) {
+                    setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotStart');
+                    const screenshot = await this.captureScreenshot();
+                    this.attachRecorderItem(task, screenshot, 'after-calling');
+                    setTimingFieldOnce(task.timing, 'captureAfterCallingSnapshotEnd');
+                }
+                Object.assign(task, returnValue);
+                task.status = 'finished';
+                task.timing.end = Date.now();
+                task.timing.cost = task.timing.end - task.timing.start;
+                await this.emitOnTaskUpdate();
+                taskIndex++;
+            } catch (e) {
+                successfullyCompleted = false;
+                task.error = e;
+                task.errorMessage = e?.message || ('string' == typeof e ? e : 'error-without-message');
+                task.errorStack = e.stack;
+                task.status = 'failed';
+                task.timing.end = Date.now();
+                task.timing.cost = task.timing.end - task.timing.start;
+                await this.emitOnTaskUpdate();
+                break;
+            }
+        }
+        for(let i = taskIndex + 1; i < this.tasks.length; i++)this.tasks[i].status = 'cancelled';
+        if (taskIndex + 1 < this.tasks.length) await this.emitOnTaskUpdate();
+        let finalizeError;
+        if (successfullyCompleted) {
+            this.status = 'completed';
+            await this.emitOnTaskUpdate();
+        } else {
+            this.status = 'error';
+            const errorTask = this.latestErrorTask();
+            const messageBase = errorTask?.errorMessage || (errorTask?.error ? String(errorTask.error) : 'Task execution failed');
+            const stack = errorTask?.errorStack;
+            const message = stack ? `${messageBase}\n${stack}` : messageBase;
+            finalizeError = new TaskExecutionError(message, this, errorTask, {
+                cause: errorTask?.error
+            });
+            await this.emitOnTaskUpdate(finalizeError);
+        }
+        if (finalizeError) throw finalizeError;
+        if (this.tasks.length) {
+            const outputIndex = Math.min(taskIndex, this.tasks.length - 1);
+            const { thought, output } = this.tasks[outputIndex];
+            return {
+                thought,
+                output
+            };
+        }
+    }
+    isInErrorState() {
+        return 'error' === this.status;
+    }
+    latestErrorTask() {
+        if ('error' !== this.status) return null;
+        for(let i = this.tasks.length - 1; i >= 0; i--)if ('failed' === this.tasks[i].status) return this.tasks[i];
+        return null;
+    }
+    dump() {
+        return new ExecutionDump({
+            id: this.id,
+            logTime: this.executionLogTime,
+            name: this.name,
+            tasks: this.tasks
+        });
+    }
+    async appendErrorPlan(errorMsg) {
+        const errorTask = {
+            type: 'Action Space',
+            subType: 'Error',
+            param: {
+                thought: errorMsg
+            },
+            thought: errorMsg,
+            executor: async ()=>{
+                throw new Error(errorMsg || 'error without thought');
+            }
+        };
+        await this.appendAndFlush(errorTask);
+        return {
+            output: void 0,
+            runner: this
+        };
+    }
+    constructor(name, uiContextBuilder, options){
+        _define_property(this, "id", void 0);
+        _define_property(this, "name", void 0);
+        _define_property(this, "tasks", void 0);
+        _define_property(this, "status", void 0);
+        _define_property(this, "onTaskStart", void 0);
+        _define_property(this, "uiContextBuilder", void 0);
+        _define_property(this, "onTaskUpdate", void 0);
+        _define_property(this, "executionLogTime", void 0);
+        _define_property(this, "lastUiContext", void 0);
+        this.id = uuid();
+        this.status = options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';
+        this.name = name;
+        this.tasks = (options?.tasks || []).map((item)=>this.markTaskAsPending(item));
+        this.onTaskStart = options?.onTaskStart;
+        this.uiContextBuilder = uiContextBuilder;
+        this.onTaskUpdate = options?.onTaskUpdate;
+        this.executionLogTime = Date.now();
+    }
+}
+class TaskExecutionError extends Error {
+    constructor(message, runner, errorTask, options){
+        super(message, options), _define_property(this, "runner", void 0), _define_property(this, "errorTask", void 0);
+        this.runner = runner;
+        this.errorTask = errorTask;
+    }
+}
+export { TaskExecutionError, TaskRunner };

package/dist/es/task-timing.mjs

@@ -0,0 +1,10 @@
+import { getDebug } from "@godscene/shared/logger";
+const debugTiming = getDebug('task-timing');
+function setTimingFieldOnce(timing, field) {
+    if (!timing) return void debugTiming(`[warning] timing object missing, skip set. field=${field}`);
+    const value = Date.now();
+    const existingValue = timing[field];
+    if (void 0 !== existingValue) return void debugTiming(`[warning] duplicate timing field set ignored. field=${field}, existing=${existingValue}, incoming=${value}`);
+    timing[field] = value;
+}
+export { setTimingFieldOnce };

package/dist/es/tree.mjs

@@ -0,0 +1,11 @@
+import { descriptionOfTree, treeToList, trimAttributes, truncateText } from "@godscene/shared/extractor";
+const ELEMENT_COUNT_WARNING_THRESHOLD = 5000;
+const TREE_SIZE_WARNING_MESSAGE = 'The number of elements is too large, it may cause the prompt to be too long, please use domIncluded: "visible-only" to reduce the number of elements';
+function tree_descriptionOfTree(tree, truncateTextLength, filterNonTextContent = false, visibleOnly = true) {
+    if (!visibleOnly) {
+        const flatElements = treeToList(tree);
+        if (flatElements.length >= ELEMENT_COUNT_WARNING_THRESHOLD) console.warn(TREE_SIZE_WARNING_MESSAGE);
+    }
+    return descriptionOfTree(tree, truncateTextLength, filterNonTextContent, visibleOnly);
+}
+export { tree_descriptionOfTree as descriptionOfTree, trimAttributes, truncateText };