@midscene/shared 1.0.1-beta-20251208031823.0 → 1.0.1-beta-20251208031856.0

package/dist/lib/mcp/base-tools.js

@@ -0,0 +1,118 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    BaseMidsceneTools: ()=>BaseMidsceneTools
+});
+const img_namespaceObject = require("@midscene/shared/img");
+const logger_namespaceObject = require("@midscene/shared/logger");
+const external_tool_generator_js_namespaceObject = require("./tool-generator.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = (0, logger_namespaceObject.getDebug)('mcp:base-tools');
+class BaseMidsceneTools {
+    preparePlatformTools() {
+        return [];
+    }
+    async initTools() {
+        this.toolDefinitions = [];
+        const platformTools = this.preparePlatformTools();
+        this.toolDefinitions.push(...platformTools);
+        let actionSpace;
+        try {
+            const agent = await this.ensureAgent();
+            actionSpace = await agent.getActionSpace();
+            debug('Action space from connected agent:', actionSpace.map((a)=>a.name).join(', '));
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            errorMessage.includes('requires a URL') || errorMessage.includes('web_connect') ? debug('Bridge mode detected - agent will be initialized on first web_connect call') : debug('Agent not available yet, using temporary device for action space');
+            const tempDevice = this.createTemporaryDevice();
+            actionSpace = tempDevice.actionSpace();
+            debug('Action space from temporary device:', actionSpace.map((a)=>a.name).join(', '));
+            await tempDevice.destroy?.();
+        }
+        const actionTools = (0, external_tool_generator_js_namespaceObject.generateToolsFromActionSpace)(actionSpace, ()=>this.ensureAgent());
+        const commonTools = (0, external_tool_generator_js_namespaceObject.generateCommonTools)(()=>this.ensureAgent());
+        this.toolDefinitions.push(...actionTools, ...commonTools);
+        debug('Total tools prepared:', this.toolDefinitions.length);
+    }
+    attachToServer(server) {
+        this.mcpServer = server;
+        if (0 === this.toolDefinitions.length) debug('Warning: No tools to register. Tools may be initialized lazily.');
+        for (const toolDef of this.toolDefinitions)if (toolDef.autoDestroy) this.toolWithAutoDestroy(toolDef.name, toolDef.description, toolDef.schema, toolDef.handler);
+        else this.mcpServer.tool(toolDef.name, toolDef.description, toolDef.schema, toolDef.handler);
+        debug('Registered', this.toolDefinitions.length, 'tools');
+    }
+    toolWithAutoDestroy(name, description, schema, handler) {
+        if (!this.mcpServer) throw new Error('MCP server not attached');
+        this.mcpServer.tool(name, description, schema, async (...args)=>{
+            try {
+                return await handler(...args);
+            } finally{
+                if (!process.env.MIDSCENE_MCP_DISABLE_AGENT_AUTO_DESTROY) {
+                    try {
+                        await this.agent?.destroy?.();
+                    } catch (error) {
+                        debug('Failed to destroy agent during cleanup:', error);
+                    }
+                    this.agent = void 0;
+                }
+            }
+        });
+    }
+    async closeBrowser() {
+        await this.agent?.destroy?.();
+    }
+    buildScreenshotContent(screenshot) {
+        const { mimeType, body } = (0, img_namespaceObject.parseBase64)(screenshot);
+        return [
+            {
+                type: 'image',
+                data: body,
+                mimeType
+            }
+        ];
+    }
+    constructor(){
+        _define_property(this, "mcpServer", void 0);
+        _define_property(this, "agent", void 0);
+        _define_property(this, "toolDefinitions", []);
+    }
+}
+exports.BaseMidsceneTools = __webpack_exports__.BaseMidsceneTools;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "BaseMidsceneTools"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/mcp/index.js

@@ -0,0 +1,79 @@
+"use strict";
+var __webpack_modules__ = {
+    "./base-server" (module) {
+        module.exports = require("./base-server.js");
+    },
+    "./base-tools" (module) {
+        module.exports = require("./base-tools.js");
+    },
+    "./tool-generator" (module) {
+        module.exports = require("./tool-generator.js");
+    },
+    "./types" (module) {
+        module.exports = require("./types.js");
+    }
+};
+var __webpack_module_cache__ = {};
+function __webpack_require__(moduleId) {
+    var cachedModule = __webpack_module_cache__[moduleId];
+    if (void 0 !== cachedModule) return cachedModule.exports;
+    var module = __webpack_module_cache__[moduleId] = {
+        exports: {}
+    };
+    __webpack_modules__[moduleId](module, module.exports, __webpack_require__);
+    return module.exports;
+}
+(()=>{
+    __webpack_require__.n = (module)=>{
+        var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
+        __webpack_require__.d(getter, {
+            a: getter
+        });
+        return getter;
+    };
+})();
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+(()=>{
+    __webpack_require__.r(__webpack_exports__);
+    var _base_server__rspack_import_0 = __webpack_require__("./base-server");
+    var __rspack_reexport = {};
+    for(const __rspack_import_key in _base_server__rspack_import_0)if ("default" !== __rspack_import_key) __rspack_reexport[__rspack_import_key] = ()=>_base_server__rspack_import_0[__rspack_import_key];
+    __webpack_require__.d(__webpack_exports__, __rspack_reexport);
+    var _base_tools__rspack_import_1 = __webpack_require__("./base-tools");
+    var __rspack_reexport = {};
+    for(const __rspack_import_key in _base_tools__rspack_import_1)if ("default" !== __rspack_import_key) __rspack_reexport[__rspack_import_key] = ()=>_base_tools__rspack_import_1[__rspack_import_key];
+    __webpack_require__.d(__webpack_exports__, __rspack_reexport);
+    var _tool_generator__rspack_import_2 = __webpack_require__("./tool-generator");
+    var __rspack_reexport = {};
+    for(const __rspack_import_key in _tool_generator__rspack_import_2)if ("default" !== __rspack_import_key) __rspack_reexport[__rspack_import_key] = ()=>_tool_generator__rspack_import_2[__rspack_import_key];
+    __webpack_require__.d(__webpack_exports__, __rspack_reexport);
+    var _types__rspack_import_3 = __webpack_require__("./types");
+    var __rspack_reexport = {};
+    for(const __rspack_import_key in _types__rspack_import_3)if ("default" !== __rspack_import_key) __rspack_reexport[__rspack_import_key] = ()=>_types__rspack_import_3[__rspack_import_key];
+    __webpack_require__.d(__webpack_exports__, __rspack_reexport);
+})();
+for(var __rspack_i in __webpack_exports__)exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/mcp/tool-generator.js

@@ -0,0 +1,252 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    generateToolsFromActionSpace: ()=>generateToolsFromActionSpace,
+    generateCommonTools: ()=>generateCommonTools
+});
+const img_namespaceObject = require("@midscene/shared/img");
+const external_zod_namespaceObject = require("zod");
+function getErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function isZodOptional(value) {
+    return '_def' in value && value._def?.typeName === 'ZodOptional';
+}
+function isZodObject(value) {
+    return '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value;
+}
+function unwrapOptional(value) {
+    if (isZodOptional(value)) return {
+        innerValue: value._def.innerType,
+        isOptional: true
+    };
+    return {
+        innerValue: value,
+        isOptional: false
+    };
+}
+function isLocateField(value) {
+    if (!isZodObject(value)) return false;
+    return 'prompt' in value.shape;
+}
+function makePromptOptional(value, wrapInOptional) {
+    const newShape = {
+        ...value.shape
+    };
+    newShape.prompt = value.shape.prompt.optional();
+    let newSchema = external_zod_namespaceObject.z.object(newShape).passthrough();
+    if (wrapInOptional) newSchema = newSchema.optional();
+    return newSchema;
+}
+function transformSchemaField(key, value) {
+    const { innerValue, isOptional } = unwrapOptional(value);
+    if (isZodObject(innerValue) && isLocateField(innerValue)) return [
+        key,
+        makePromptOptional(innerValue, isOptional)
+    ];
+    return [
+        key,
+        value
+    ];
+}
+function extractActionSchema(paramSchema) {
+    if (!paramSchema) return {};
+    const schema = paramSchema;
+    if (!isZodObject(schema)) return schema;
+    return Object.fromEntries(Object.entries(schema.shape).map(([key, value])=>transformSchemaField(key, value)));
+}
+function serializeArgsToDescription(args) {
+    try {
+        return Object.entries(args).map(([key, value])=>{
+            if ('object' == typeof value && null !== value) try {
+                return `${key}: ${JSON.stringify(value)}`;
+            } catch  {
+                return `${key}: [object]`;
+            }
+            return `${key}: "${value}"`;
+        }).join(', ');
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error serializing args:', errorMessage);
+        return `[args serialization failed: ${errorMessage}]`;
+    }
+}
+function buildActionInstruction(actionName, args) {
+    const argsDescription = serializeArgsToDescription(args);
+    return argsDescription ? `Use the action "${actionName}" with ${argsDescription}` : `Use the action "${actionName}"`;
+}
+async function captureScreenshotResult(agent, actionName) {
+    try {
+        const screenshot = await agent.page?.screenshotBase64();
+        if (!screenshot) return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                }
+            ]
+        };
+        const { mimeType, body } = (0, img_namespaceObject.parseBase64)(screenshot);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                },
+                {
+                    type: 'image',
+                    data: body,
+                    mimeType
+                }
+            ]
+        };
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error capturing screenshot:', errorMessage);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
+                }
+            ]
+        };
+    }
+}
+function createErrorResult(message) {
+    return {
+        content: [
+            {
+                type: 'text',
+                text: message
+            }
+        ],
+        isError: true
+    };
+}
+function generateToolsFromActionSpace(actionSpace, getAgent) {
+    return actionSpace.map((action)=>{
+        const schema = extractActionSchema(action.paramSchema);
+        return {
+            name: action.name,
+            description: action.description || `Execute ${action.name} action`,
+            schema,
+            handler: async (args)=>{
+                try {
+                    const agent = await getAgent();
+                    if (agent.aiAction) {
+                        const instruction = buildActionInstruction(action.name, args);
+                        try {
+                            await agent.aiAction(instruction);
+                        } catch (error) {
+                            const errorMessage = getErrorMessage(error);
+                            console.error(`Error executing action "${action.name}":`, errorMessage);
+                            return createErrorResult(`Failed to execute action "${action.name}": ${errorMessage}`);
+                        }
+                    }
+                    return await captureScreenshotResult(agent, action.name);
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error(`Error in handler for "${action.name}":`, errorMessage);
+                    return createErrorResult(`Failed to get agent or execute action "${action.name}": ${errorMessage}`);
+                }
+            },
+            autoDestroy: true
+        };
+    });
+}
+function generateCommonTools(getAgent) {
+    return [
+        {
+            name: 'take_screenshot',
+            description: 'Capture screenshot of current page/screen',
+            schema: {},
+            handler: async ()=>{
+                try {
+                    const agent = await getAgent();
+                    const screenshot = await agent.page?.screenshotBase64();
+                    if (!screenshot) return createErrorResult('Screenshot not available');
+                    const { mimeType, body } = (0, img_namespaceObject.parseBase64)(screenshot);
+                    return {
+                        content: [
+                            {
+                                type: 'image',
+                                data: body,
+                                mimeType
+                            }
+                        ]
+                    };
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error taking screenshot:', errorMessage);
+                    return createErrorResult(`Failed to capture screenshot: ${errorMessage}`);
+                }
+            },
+            autoDestroy: true
+        },
+        {
+            name: 'wait_for',
+            description: 'Wait until condition becomes true',
+            schema: {
+                assertion: external_zod_namespaceObject.z.string().describe('Condition to wait for'),
+                timeoutMs: external_zod_namespaceObject.z.number().optional().default(15000),
+                checkIntervalMs: external_zod_namespaceObject.z.number().optional().default(3000)
+            },
+            handler: async (args)=>{
+                try {
+                    const agent = await getAgent();
+                    const { assertion, timeoutMs, checkIntervalMs } = args;
+                    if (agent.aiWaitFor) await agent.aiWaitFor(assertion, {
+                        timeoutMs,
+                        checkIntervalMs
+                    });
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: `Condition met: "${assertion}"`
+                            }
+                        ]
+                    };
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error in wait_for:', errorMessage);
+                    return createErrorResult(`Wait condition failed: ${errorMessage}`);
+                }
+            },
+            autoDestroy: true
+        }
+    ];
+}
+exports.generateCommonTools = __webpack_exports__.generateCommonTools;
+exports.generateToolsFromActionSpace = __webpack_exports__.generateToolsFromActionSpace;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "generateCommonTools",
+    "generateToolsFromActionSpace"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/mcp/types.js

@@ -0,0 +1,40 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    defaultAppLoadingCheckIntervalMs: ()=>defaultAppLoadingCheckIntervalMs,
+    defaultAppLoadingTimeoutMs: ()=>defaultAppLoadingTimeoutMs
+});
+const defaultAppLoadingTimeoutMs = 10000;
+const defaultAppLoadingCheckIntervalMs = 2000;
+exports.defaultAppLoadingCheckIntervalMs = __webpack_exports__.defaultAppLoadingCheckIntervalMs;
+exports.defaultAppLoadingTimeoutMs = __webpack_exports__.defaultAppLoadingTimeoutMs;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "defaultAppLoadingCheckIntervalMs",
+    "defaultAppLoadingTimeoutMs"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});