@aiscene/shared 1.6.0

package/dist/lib/mcp/inject-report-html-plugin.js

@@ -0,0 +1,98 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.n = (module)=>{
+        var getter = module && module.__esModule ? ()=>module['default'] : ()=>module;
+        __webpack_require__.d(getter, {
+            a: getter
+        });
+        return getter;
+    };
+})();
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    injectReportHtmlFromCore: ()=>injectReportHtmlFromCore
+});
+const external_node_fs_namespaceObject = require("node:fs");
+var external_node_fs_default = /*#__PURE__*/ __webpack_require__.n(external_node_fs_namespaceObject);
+const external_node_path_namespaceObject = require("node:path");
+var external_node_path_default = /*#__PURE__*/ __webpack_require__.n(external_node_path_namespaceObject);
+const MAGIC_STRING = 'REPLACE_ME_WITH_REPORT_HTML';
+const REPLACED_MARK = '/*REPORT_HTML_REPLACED*/';
+const REG_EXP_FOR_REPLACE = /\/\*REPORT_HTML_REPLACED\*\/.*/;
+function injectReportHtmlFromCore(packageDir) {
+    return {
+        name: 'inject-report-html-from-core',
+        setup (api) {
+            api.onAfterBuild(()=>{
+                const coreUtilsPath = external_node_path_default().resolve(packageDir, '..', 'core', 'dist', 'lib', 'utils.js');
+                if (!external_node_fs_default().existsSync(coreUtilsPath)) return void console.warn('[inject-report-html] @midscene/core dist not found, skipping');
+                const coreContent = external_node_fs_default().readFileSync(coreUtilsPath, 'utf-8');
+                if (!coreContent.includes(REPLACED_MARK)) return void console.warn('[inject-report-html] HTML not found in core dist. Ensure report builds first.');
+                const markerIndex = coreContent.indexOf(REPLACED_MARK);
+                const jsonStart = markerIndex + REPLACED_MARK.length;
+                let jsonEnd = jsonStart;
+                if ('"' === coreContent[jsonStart]) {
+                    jsonEnd = jsonStart + 1;
+                    while(jsonEnd < coreContent.length)if ('\\' === coreContent[jsonEnd]) jsonEnd += 2;
+                    else if ('"' === coreContent[jsonEnd]) {
+                        jsonEnd += 1;
+                        break;
+                    } else jsonEnd += 1;
+                }
+                const jsonString = coreContent.slice(jsonStart, jsonEnd);
+                if (!jsonString || jsonString.length < 10) return void console.warn('[inject-report-html] Failed to extract HTML from core');
+                const finalContent = `${REPLACED_MARK}${jsonString}`;
+                const distDir = external_node_path_default().join(packageDir, 'dist');
+                if (!external_node_fs_default().existsSync(distDir)) return;
+                const jsFiles = external_node_fs_default().readdirSync(distDir).filter((f)=>f.endsWith('.js'));
+                let injectedCount = 0;
+                for (const file of jsFiles){
+                    const filePath = external_node_path_default().join(distDir, file);
+                    const content = external_node_fs_default().readFileSync(filePath, 'utf-8');
+                    if (content.includes(REPLACED_MARK)) {
+                        if (REG_EXP_FOR_REPLACE.test(content)) {
+                            external_node_fs_default().writeFileSync(filePath, content.replace(REG_EXP_FOR_REPLACE, ()=>finalContent));
+                            console.log(`[inject-report-html] Updated: ${file}`);
+                            injectedCount++;
+                        }
+                    } else if (content.includes(`'${MAGIC_STRING}'`)) {
+                        external_node_fs_default().writeFileSync(filePath, content.replace(`'${MAGIC_STRING}'`, ()=>finalContent));
+                        console.log(`[inject-report-html] Injected: ${file}`);
+                        injectedCount++;
+                    }
+                }
+                if (injectedCount > 0) console.log(`[inject-report-html] Completed: ${injectedCount} file(s)`);
+            });
+        }
+    };
+}
+exports.injectReportHtmlFromCore = __webpack_exports__.injectReportHtmlFromCore;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "injectReportHtmlFromCore"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/mcp/launcher-helper.js

@@ -0,0 +1,86 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    createMCPServerLauncher: ()=>createMCPServerLauncher
+});
+function createMCPServerLauncher(config) {
+    const { agent, platformName, ToolsManagerClass, MCPServerClass } = config;
+    function validateAgent() {
+        const device = agent.interface;
+        if (!device) throw new Error(`Agent must have an 'interface' property that references the underlying device.
+Please ensure your agent instance is properly initialized with a device interface.
+Expected: agent.interface to be defined, but got: ${typeof device}
+Solution: Check that your agent constructor properly sets the interface property.`);
+    }
+    function createToolsManager() {
+        const toolsManager = new ToolsManagerClass();
+        toolsManager.agent = agent;
+        return toolsManager;
+    }
+    function logStartupInfo(mode, additionalInfo) {
+        const device = agent.interface;
+        console.log(`Starting Midscene ${platformName} MCP Server (${mode})...`);
+        console.log(`Agent: ${agent.constructor.name}`);
+        console.log(`Device: ${device.constructor.name}`);
+        if (additionalInfo?.port !== void 0) console.log(`Port: ${additionalInfo.port}`);
+        if (additionalInfo?.host) console.log(`Host: ${additionalInfo.host}`);
+    }
+    return {
+        async launch (options = {}) {
+            const { verbose = true } = options;
+            validateAgent();
+            if (verbose) logStartupInfo('stdio');
+            const toolsManager = createToolsManager();
+            const server = new MCPServerClass(toolsManager);
+            const result = await server.launch();
+            if (verbose) console.log(`${platformName} MCP Server started (stdio mode)`);
+            return result;
+        },
+        async launchHttp (options) {
+            const { port, host = 'localhost', verbose = true } = options;
+            validateAgent();
+            if (verbose) logStartupInfo('HTTP', {
+                port,
+                host
+            });
+            const toolsManager = createToolsManager();
+            const server = new MCPServerClass(toolsManager);
+            const result = await server.launchHttp({
+                port,
+                host
+            });
+            if (verbose) console.log(`${platformName} MCP Server started on http://${result.host}:${result.port}/mcp`);
+            return result;
+        }
+    };
+}
+exports.createMCPServerLauncher = __webpack_exports__.createMCPServerLauncher;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "createMCPServerLauncher"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/mcp/tool-generator.js

@@ -0,0 +1,334 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    generateToolsFromActionSpace: ()=>generateToolsFromActionSpace,
+    generateCommonTools: ()=>generateCommonTools
+});
+const index_js_namespaceObject = require("../img/index.js");
+const external_zod_namespaceObject = require("zod");
+const external_zod_schema_utils_js_namespaceObject = require("../zod-schema-utils.js");
+function getErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function describeActionForMCP(action) {
+    const actionDesc = action.description || `Execute ${action.name} action`;
+    if (!action.paramSchema) return `${action.name} action, ${actionDesc}`;
+    const schema = action.paramSchema;
+    const isZodObjectType = schema._def?.typeName === 'ZodObject';
+    if (!isZodObjectType || !schema.shape) {
+        const typeName = (0, external_zod_schema_utils_js_namespaceObject.getZodTypeName)(schema);
+        const description = (0, external_zod_schema_utils_js_namespaceObject.getZodDescription)(schema);
+        const paramDesc = description ? `${typeName} - ${description}` : typeName;
+        return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
+    }
+    const paramDescriptions = [];
+    for (const [key, field] of Object.entries(schema.shape))if (field && 'object' == typeof field) {
+        const isFieldOptional = 'function' == typeof field.isOptional && field.isOptional();
+        const typeName = (0, external_zod_schema_utils_js_namespaceObject.getZodTypeName)(field);
+        const description = (0, external_zod_schema_utils_js_namespaceObject.getZodDescription)(field);
+        let paramStr = `${key}${isFieldOptional ? '?' : ''} (${typeName})`;
+        if (description) paramStr += ` - ${description}`;
+        paramDescriptions.push(paramStr);
+    }
+    if (0 === paramDescriptions.length) return `${action.name} action, ${actionDesc}`;
+    return `${action.name} action, ${actionDesc}. Parameters: ${paramDescriptions.join('; ')}`;
+}
+function isZodOptional(value) {
+    return '_def' in value && value._def?.typeName === 'ZodOptional';
+}
+function isZodObject(value) {
+    return '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value;
+}
+function unwrapOptional(value) {
+    if (isZodOptional(value)) return {
+        innerValue: value._def.innerType,
+        isOptional: true
+    };
+    return {
+        innerValue: value,
+        isOptional: false
+    };
+}
+function isLocateField(value) {
+    if (!isZodObject(value)) return false;
+    return 'prompt' in value.shape;
+}
+function makePromptOptional(value, wrapInOptional) {
+    const newShape = {
+        ...value.shape
+    };
+    newShape.prompt = value.shape.prompt.optional();
+    let newSchema = external_zod_namespaceObject.z.object(newShape).passthrough();
+    if (wrapInOptional) newSchema = newSchema.optional();
+    return newSchema;
+}
+function transformSchemaField(key, value) {
+    const { innerValue, isOptional } = unwrapOptional(value);
+    if (isZodObject(innerValue) && isLocateField(innerValue)) return [
+        key,
+        makePromptOptional(innerValue, isOptional)
+    ];
+    return [
+        key,
+        value
+    ];
+}
+function extractActionSchema(paramSchema) {
+    if (!paramSchema) return {};
+    const schema = paramSchema;
+    if (!isZodObject(schema)) return schema;
+    return Object.fromEntries(Object.entries(schema.shape).map(([key, value])=>transformSchemaField(key, value)));
+}
+function serializeArgsToDescription(args) {
+    try {
+        return Object.entries(args).map(([key, value])=>{
+            if ('object' == typeof value && null !== value) try {
+                return `${key}: ${JSON.stringify(value)}`;
+            } catch  {
+                return `${key}: [object]`;
+            }
+            return `${key}: "${value}"`;
+        }).join(', ');
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error serializing args:', errorMessage);
+        return `[args serialization failed: ${errorMessage}]`;
+    }
+}
+function buildActionInstruction(actionName, args) {
+    const locatePrompt = args.locate && 'object' == typeof args.locate ? args.locate.prompt : void 0;
+    switch(actionName){
+        case 'Tap':
+            return locatePrompt ? `Tap on "${locatePrompt}"` : 'Tap';
+        case 'Input':
+            {
+                const value = args.value ?? args.content ?? '';
+                return locatePrompt ? `Input "${value}" into "${locatePrompt}"` : `Input "${value}"`;
+            }
+        case 'Scroll':
+            {
+                const direction = args.direction ?? 'down';
+                return locatePrompt ? `Scroll ${direction} on "${locatePrompt}"` : `Scroll ${direction}`;
+            }
+        case 'Hover':
+            return locatePrompt ? `Hover over "${locatePrompt}"` : 'Hover';
+        case 'KeyboardPress':
+            {
+                const key = args.value ?? args.key ?? '';
+                return `Press key "${key}"`;
+            }
+        default:
+            {
+                const argsDescription = serializeArgsToDescription(args);
+                return argsDescription ? `${actionName}: ${argsDescription}` : actionName;
+            }
+    }
+}
+async function captureScreenshotResult(agent, actionName) {
+    try {
+        const screenshot = await agent.page?.screenshotBase64();
+        if (!screenshot) return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                }
+            ]
+        };
+        const { mimeType, body } = (0, index_js_namespaceObject.parseBase64)(screenshot);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                },
+                {
+                    type: 'image',
+                    data: body,
+                    mimeType
+                }
+            ]
+        };
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error capturing screenshot:', errorMessage);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
+                }
+            ]
+        };
+    }
+}
+function createErrorResult(message) {
+    return {
+        content: [
+            {
+                type: 'text',
+                text: message
+            }
+        ],
+        isError: true
+    };
+}
+async function captureFailureResult(agent, actionName, errorMessage) {
+    const warningText = `Warning: Action "${actionName}" failed: ${errorMessage}. Check the screenshot below for the current page state and decide how to proceed.`;
+    try {
+        const screenshot = await agent.page?.screenshotBase64();
+        if (!screenshot) return {
+            content: [
+                {
+                    type: 'text',
+                    text: warningText
+                }
+            ]
+        };
+        const { mimeType, body } = (0, index_js_namespaceObject.parseBase64)(screenshot);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: warningText
+                },
+                {
+                    type: 'image',
+                    data: body,
+                    mimeType
+                }
+            ]
+        };
+    } catch  {
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: warningText
+                }
+            ]
+        };
+    }
+}
+function generateToolsFromActionSpace(actionSpace, getAgent) {
+    return actionSpace.map((action)=>{
+        const schema = extractActionSchema(action.paramSchema);
+        return {
+            name: action.name,
+            description: describeActionForMCP(action),
+            schema,
+            handler: async (args)=>{
+                try {
+                    const agent = await getAgent();
+                    if (agent.aiAction) {
+                        const instruction = buildActionInstruction(action.name, args);
+                        try {
+                            await agent.aiAction(instruction);
+                        } catch (error) {
+                            const errorMessage = getErrorMessage(error);
+                            console.error(`Error executing action "${action.name}":`, errorMessage);
+                            return await captureFailureResult(agent, action.name, errorMessage);
+                        }
+                    }
+                    return await captureScreenshotResult(agent, action.name);
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error(`Error in handler for "${action.name}":`, errorMessage);
+                    return createErrorResult(`Failed to get agent or execute action "${action.name}": ${errorMessage}`);
+                }
+            }
+        };
+    });
+}
+function generateCommonTools(getAgent) {
+    return [
+        {
+            name: 'take_screenshot',
+            description: 'Capture screenshot of current page/screen',
+            schema: {},
+            handler: async ()=>{
+                try {
+                    const agent = await getAgent();
+                    const screenshot = await agent.page?.screenshotBase64();
+                    if (!screenshot) return createErrorResult('Screenshot not available');
+                    const { mimeType, body } = (0, index_js_namespaceObject.parseBase64)(screenshot);
+                    return {
+                        content: [
+                            {
+                                type: 'image',
+                                data: body,
+                                mimeType
+                            }
+                        ]
+                    };
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error taking screenshot:', errorMessage);
+                    return createErrorResult(`Failed to capture screenshot: ${errorMessage}`);
+                }
+            }
+        },
+        {
+            name: 'act',
+            description: 'Execute a natural language action. The AI will plan and perform multi-step operations in a single invocation, useful for transient UI interactions (e.g., Spotlight, dropdown menus) that disappear between separate commands.',
+            schema: {
+                prompt: external_zod_namespaceObject.z.string().describe('Natural language description of the action to perform, e.g. "press Command+Space, type Safari, press Enter"')
+            },
+            handler: async (args)=>{
+                const prompt = args.prompt;
+                try {
+                    const agent = await getAgent();
+                    if (!agent.aiAction) return createErrorResult('act is not supported by this agent');
+                    const result = await agent.aiAction(prompt, {
+                        deepThink: false
+                    });
+                    const screenshotResult = await captureScreenshotResult(agent, 'act');
+                    if (result) {
+                        const message = 'string' == typeof result ? result : JSON.stringify(result);
+                        screenshotResult.content.unshift({
+                            type: 'text',
+                            text: `Task finished, message: ${message}`
+                        });
+                    }
+                    return screenshotResult;
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error executing act:', errorMessage);
+                    return createErrorResult(`Failed to execute act: ${errorMessage}`);
+                }
+            }
+        }
+    ];
+}
+exports.generateCommonTools = __webpack_exports__.generateCommonTools;
+exports.generateToolsFromActionSpace = __webpack_exports__.generateToolsFromActionSpace;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "generateCommonTools",
+    "generateToolsFromActionSpace"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});

package/dist/lib/mcp/types.js

@@ -0,0 +1,40 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    defaultAppLoadingCheckIntervalMs: ()=>defaultAppLoadingCheckIntervalMs,
+    defaultAppLoadingTimeoutMs: ()=>defaultAppLoadingTimeoutMs
+});
+const defaultAppLoadingTimeoutMs = 10000;
+const defaultAppLoadingCheckIntervalMs = 2000;
+exports.defaultAppLoadingCheckIntervalMs = __webpack_exports__.defaultAppLoadingCheckIntervalMs;
+exports.defaultAppLoadingTimeoutMs = __webpack_exports__.defaultAppLoadingTimeoutMs;
+for(var __rspack_i in __webpack_exports__)if (-1 === [
+    "defaultAppLoadingCheckIntervalMs",
+    "defaultAppLoadingTimeoutMs"
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});