@godscene/core 1.7.11

package/dist/lib/screenshot-item.js

@@ -0,0 +1,154 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    ScreenshotItem: ()=>ScreenshotItem
+});
+const external_node_fs_namespaceObject = require("node:fs");
+const utils_namespaceObject = require("@godscene/shared/utils");
+const html_utils_js_namespaceObject = require("./dump/html-utils.js");
+const screenshot_store_js_namespaceObject = require("./dump/screenshot-store.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+function detectFormat(base64) {
+    if (base64.startsWith('data:image/jpeg')) return 'jpeg';
+    if (base64.startsWith('data:image/jpg')) return 'jpeg';
+    return 'png';
+}
+class ScreenshotItem {
+    static create(base64, capturedAt) {
+        return new ScreenshotItem((0, utils_namespaceObject.uuid)(), base64, capturedAt);
+    }
+    get id() {
+        return this._id;
+    }
+    get format() {
+        return this._format;
+    }
+    get extension() {
+        return 'jpeg' === this._format ? 'jpeg' : 'png';
+    }
+    get capturedAt() {
+        return this._capturedAt;
+    }
+    get base64() {
+        if (null !== this._base64) return this._base64;
+        const loadFromFile = ()=>{
+            if (null === this._persistedPath) throw new Error(`Screenshot ${this._id}: file recovery path missing`);
+            const buffer = (0, external_node_fs_namespaceObject.readFileSync)(this._persistedPath);
+            return `data:image/${this._format};base64,${buffer.toString('base64')}`;
+        };
+        const loadFromInline = ()=>{
+            if (null === this._persistedHtmlPath) throw new Error(`Screenshot ${this._id}: HTML recovery path missing`);
+            const data = (0, html_utils_js_namespaceObject.extractImageByIdSync)(this._persistedHtmlPath, this._id);
+            if (data) return data;
+            throw new Error(`Screenshot ${this._id}: cannot recover from HTML (id not found in ${this._persistedHtmlPath})`);
+        };
+        if (this._serializedRef?.storage === 'file') return loadFromFile();
+        if (this._serializedRef?.storage === 'inline') return loadFromInline();
+        if (null !== this._persistedPath) return loadFromFile();
+        if (null !== this._persistedHtmlPath) return loadFromInline();
+        throw new Error(`Screenshot ${this._id}: base64 data released without recovery path`);
+    }
+    hasBase64() {
+        return null !== this._base64;
+    }
+    markPersistedInline(htmlPath) {
+        const ref = this.createRef('inline');
+        this._serializedRef = ref;
+        this._persistedHtmlPath = htmlPath;
+        this._base64 = null;
+        return ref;
+    }
+    registerPersistedFileCopy(relativePath, absolutePath) {
+        const ref = this.createRef('file', relativePath);
+        this._persistedPath = absolutePath;
+        this._base64 = null;
+        return ref;
+    }
+    markPersistedToPath(relativePath, absolutePath) {
+        const ref = this.registerPersistedFileCopy(relativePath, absolutePath);
+        this._serializedRef = ref;
+        return ref;
+    }
+    toSerializable() {
+        return this._serializedRef ?? {
+            type: 'midscene_screenshot_ref',
+            id: this._id,
+            capturedAt: this._capturedAt,
+            mimeType: 'jpeg' === this._format ? 'image/jpeg' : 'image/png',
+            storage: 'inline'
+        };
+    }
+    static isSerialized(value) {
+        return null !== (0, screenshot_store_js_namespaceObject.normalizeScreenshotRef)(value);
+    }
+    createRef(storage, relativePath) {
+        const baseRef = {
+            type: 'midscene_screenshot_ref',
+            id: this._id,
+            capturedAt: this._capturedAt,
+            mimeType: 'jpeg' === this._format ? 'image/jpeg' : 'image/png',
+            storage
+        };
+        if ('file' === storage) return {
+            ...baseRef,
+            storage,
+            path: relativePath
+        };
+        return baseRef;
+    }
+    get rawBase64() {
+        return this.base64.replace(/^data:image\/(png|jpeg|jpg);base64,/, '');
+    }
+    constructor(id, base64, capturedAt){
+        _define_property(this, "_id", void 0);
+        _define_property(this, "_base64", void 0);
+        _define_property(this, "_format", void 0);
+        _define_property(this, "_capturedAt", void 0);
+        _define_property(this, "_serializedRef", null);
+        _define_property(this, "_persistedPath", null);
+        _define_property(this, "_persistedHtmlPath", null);
+        this._id = id;
+        this._base64 = base64;
+        this._format = detectFormat(base64);
+        this._capturedAt = capturedAt;
+    }
+}
+exports.ScreenshotItem = __webpack_exports__.ScreenshotItem;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "ScreenshotItem"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/service/index.js

@@ -0,0 +1,306 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    default: ()=>Service
+});
+const util_js_namespaceObject = require("../ai-model/auto-glm/util.js");
+const index_js_namespaceObject = require("../ai-model/index.js");
+const inspect_js_namespaceObject = require("../ai-model/inspect.js");
+const describe_js_namespaceObject = require("../ai-model/prompt/describe.js");
+const external_common_js_namespaceObject = require("../common.js");
+const external_types_js_namespaceObject = require("../types.js");
+const img_namespaceObject = require("@godscene/shared/img");
+const logger_namespaceObject = require("@godscene/shared/logger");
+const utils_namespaceObject = require("@godscene/shared/utils");
+const external_utils_js_namespaceObject = require("./utils.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = (0, logger_namespaceObject.getDebug)('ai:service');
+class Service {
+    async locate(query, opt, modelConfig, abortSignal) {
+        const queryPrompt = 'string' == typeof query ? query : query.prompt;
+        (0, utils_namespaceObject.assert)(queryPrompt, 'query is required for locate');
+        (0, utils_namespaceObject.assert)('object' == typeof query, 'query should be an object for locate');
+        const hasPlanLocatedElement = !!opt?.planLocatedElement?.rect;
+        let searchAreaPrompt;
+        if (query.deepLocate && !hasPlanLocatedElement) searchAreaPrompt = query.prompt;
+        const { modelFamily } = modelConfig;
+        if (searchAreaPrompt && !modelFamily) {
+            console.warn('The "deepLocate" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/model-config');
+            searchAreaPrompt = void 0;
+        }
+        if (searchAreaPrompt && (0, util_js_namespaceObject.isAutoGLM)(modelFamily)) {
+            console.warn('The "deepLocate" feature is not supported with AutoGLM.');
+            searchAreaPrompt = void 0;
+        }
+        const context = opt?.context || await this.contextRetrieverFn();
+        let searchArea;
+        let searchAreaRawResponse;
+        let searchAreaUsage;
+        let searchAreaResponse;
+        if (query.deepLocate && hasPlanLocatedElement) {
+            const searchAreaConfig = await (0, inspect_js_namespaceObject.buildSearchAreaConfig)({
+                context,
+                baseRect: opt.planLocatedElement.rect,
+                modelFamily
+            });
+            searchArea = searchAreaConfig.rect;
+            searchAreaRawResponse = JSON.stringify({
+                source: 'plan-located-element',
+                rect: opt.planLocatedElement.rect
+            });
+            searchAreaResponse = {
+                rect: searchArea,
+                imageBase64: searchAreaConfig.imageBase64,
+                scale: searchAreaConfig.scale,
+                rawResponse: searchAreaRawResponse
+            };
+        } else if (searchAreaPrompt) {
+            searchAreaResponse = await (0, inspect_js_namespaceObject.AiLocateSection)({
+                context,
+                sectionDescription: searchAreaPrompt,
+                modelConfig,
+                abortSignal
+            });
+            (0, utils_namespaceObject.assert)(searchAreaResponse.rect, `cannot find search area for "${searchAreaPrompt}"${searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''}`);
+            searchAreaRawResponse = searchAreaResponse.rawResponse;
+            searchAreaUsage = searchAreaResponse.usage;
+            searchArea = searchAreaResponse.rect;
+        }
+        const startTime = Date.now();
+        const { parseResult, rect, rawResponse, usage, reasoning_content } = await (0, index_js_namespaceObject.AiLocateElement)({
+            context,
+            targetElementDescription: queryPrompt,
+            searchConfig: searchAreaResponse,
+            modelConfig,
+            abortSignal
+        });
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse: JSON.stringify(rawResponse),
+            formatResponse: JSON.stringify(parseResult),
+            usage,
+            searchArea,
+            searchAreaRawResponse,
+            searchAreaUsage,
+            reasoning_content
+        };
+        let errorLog;
+        if (parseResult.errors?.length) errorLog = `failed to locate element: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'locate',
+            userQuery: {
+                element: queryPrompt
+            },
+            matchedElement: [],
+            matchedRect: rect,
+            data: null,
+            taskInfo,
+            deepLocate: !!searchArea,
+            error: errorLog
+        };
+        const elements = parseResult.elements || [];
+        const dump = (0, external_utils_js_namespaceObject.createServiceDump)({
+            ...dumpData,
+            matchedElement: elements
+        });
+        if (errorLog) throw new external_types_js_namespaceObject.ServiceError(errorLog, dump);
+        if (elements.length > 1) throw new external_types_js_namespaceObject.ServiceError(`locate: multiple elements found, length = ${elements.length}`, dump);
+        if (1 === elements.length) return {
+            element: {
+                center: elements[0].center,
+                rect: elements[0].rect,
+                description: elements[0].description
+            },
+            rect,
+            dump
+        };
+        return {
+            element: null,
+            rect,
+            dump
+        };
+    }
+    async extract(dataDemand, modelConfig, opt, pageDescription, multimodalPrompt, context) {
+        (0, utils_namespaceObject.assert)(context, 'context is required for extract');
+        (0, utils_namespaceObject.assert)('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
+        const startTime = Date.now();
+        let parseResult;
+        let rawResponse;
+        let usage;
+        let reasoning_content;
+        try {
+            const result = await (0, index_js_namespaceObject.AiExtractElementInfo)({
+                context,
+                dataQuery: dataDemand,
+                multimodalPrompt,
+                extractOption: opt,
+                modelConfig,
+                pageDescription
+            });
+            parseResult = result.parseResult;
+            rawResponse = result.rawResponse;
+            usage = result.usage;
+            reasoning_content = result.reasoning_content;
+        } catch (error) {
+            if (error instanceof index_js_namespaceObject.AIResponseParseError) {
+                const timeCost = Date.now() - startTime;
+                const taskInfo = {
+                    ...this.taskInfo ? this.taskInfo : {},
+                    durationMs: timeCost,
+                    rawResponse: error.rawResponse,
+                    usage: error.usage
+                };
+                const dump = (0, external_utils_js_namespaceObject.createServiceDump)({
+                    type: 'extract',
+                    userQuery: {
+                        dataDemand
+                    },
+                    matchedElement: [],
+                    data: null,
+                    taskInfo,
+                    error: error.message
+                });
+                throw new external_types_js_namespaceObject.ServiceError(error.message, dump);
+            }
+            throw error;
+        }
+        const timeCost = Date.now() - startTime;
+        const taskInfo = {
+            ...this.taskInfo ? this.taskInfo : {},
+            durationMs: timeCost,
+            rawResponse,
+            formatResponse: JSON.stringify(parseResult),
+            usage,
+            reasoning_content
+        };
+        let errorLog;
+        if (parseResult.errors?.length) errorLog = `AI response error: \n${parseResult.errors.join('\n')}`;
+        const dumpData = {
+            type: 'extract',
+            userQuery: {
+                dataDemand
+            },
+            matchedElement: [],
+            data: null,
+            taskInfo,
+            error: errorLog
+        };
+        const { data, thought } = parseResult || {};
+        const dump = (0, external_utils_js_namespaceObject.createServiceDump)({
+            ...dumpData,
+            data
+        });
+        if (errorLog && !data) throw new external_types_js_namespaceObject.ServiceError(errorLog, dump);
+        return {
+            data,
+            thought,
+            usage,
+            reasoning_content,
+            dump
+        };
+    }
+    async describe(target, modelConfig, opt) {
+        (0, utils_namespaceObject.assert)(target, 'target is required for service.describe');
+        const context = await this.contextRetrieverFn();
+        const { shotSize } = context;
+        const screenshotBase64 = context.screenshot.base64;
+        (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshot is required for service.describe');
+        const { modelFamily } = modelConfig;
+        const systemPrompt = (0, describe_js_namespaceObject.elementDescriberInstruction)();
+        const defaultRectSize = 30;
+        const targetRect = Array.isArray(target) ? {
+            left: Math.floor(target[0] - defaultRectSize / 2),
+            top: Math.floor(target[1] - defaultRectSize / 2),
+            width: defaultRectSize,
+            height: defaultRectSize
+        } : target;
+        let imagePayload = await (0, img_namespaceObject.compositeElementInfoImg)({
+            inputImgBase64: screenshotBase64,
+            size: shotSize,
+            elementsPositionInfo: [
+                {
+                    rect: targetRect
+                }
+            ],
+            borderThickness: 3
+        });
+        if (opt?.deepLocate) {
+            const searchArea = (0, external_common_js_namespaceObject.expandSearchArea)(targetRect, shotSize);
+            debug('describe: cropping to searchArea', searchArea);
+            const croppedResult = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea, 'qwen2.5-vl' === modelFamily);
+            imagePayload = croppedResult.imageBase64;
+        }
+        const msgs = [
+            {
+                role: 'system',
+                content: systemPrompt
+            },
+            {
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: imagePayload,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            }
+        ];
+        const res = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, modelConfig);
+        const { content } = res;
+        (0, utils_namespaceObject.assert)(!content.error, `describe failed: ${content.error}`);
+        (0, utils_namespaceObject.assert)(content.description, 'failed to describe the element');
+        return content;
+    }
+    constructor(context, opt){
+        _define_property(this, "contextRetrieverFn", void 0);
+        _define_property(this, "taskInfo", void 0);
+        (0, utils_namespaceObject.assert)(context, 'context is required for Service');
+        if ('function' == typeof context) this.contextRetrieverFn = context;
+        else this.contextRetrieverFn = ()=>Promise.resolve(context);
+        if (void 0 !== opt?.taskInfo) this.taskInfo = opt.taskInfo;
+    }
+}
+exports["default"] = __webpack_exports__["default"];
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "default"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/service/utils.js

@@ -0,0 +1,47 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    createServiceDump: ()=>createServiceDump
+});
+const utils_namespaceObject = require("@godscene/shared/utils");
+function createServiceDump(data) {
+    const baseData = {
+        logTime: Date.now()
+    };
+    const finalData = {
+        logId: (0, utils_namespaceObject.uuid)(),
+        ...baseData,
+        ...data
+    };
+    return finalData;
+}
+exports.createServiceDump = __webpack_exports__.createServiceDump;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "createServiceDump"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/skill/index.js

@@ -0,0 +1,69 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ("u" > typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    runSkillCLI: ()=>runSkillCLI
+});
+const cli_namespaceObject = require("@godscene/shared/cli");
+const base_tools_namespaceObject = require("@godscene/shared/mcp/base-tools");
+const agent_js_namespaceObject = require("../agent/agent.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+class SkillMidsceneTools extends base_tools_namespaceObject.BaseMidsceneTools {
+    createTemporaryDevice() {
+        return new this.DeviceClass();
+    }
+    async ensureAgent() {
+        if (!this.agent) {
+            const device = new this.DeviceClass();
+            this.agent = new agent_js_namespaceObject.Agent(device);
+        }
+        return this.agent;
+    }
+    constructor(DeviceClass){
+        super(), _define_property(this, "DeviceClass", void 0), this.DeviceClass = DeviceClass;
+    }
+}
+function runSkillCLI(options) {
+    const tools = new SkillMidsceneTools(options.DeviceClass);
+    return (0, cli_namespaceObject.runToolsCLI)(tools, options.scriptName).catch((e)=>{
+        process.exit((0, cli_namespaceObject.reportCLIError)(e));
+    });
+}
+exports.runSkillCLI = __webpack_exports__.runSkillCLI;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "runSkillCLI"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});