@midscene/shared 1.0.1-beta-20251208031823.0 → 1.0.1-beta-20251208031856.0

package/dist/es/mcp/base-server.mjs

@@ -0,0 +1,241 @@
+import { randomUUID } from "node:crypto";
+import { setIsMcp } from "@midscene/shared/utils";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const CLI_ARGS_CONFIG = {
+    mode: {
+        type: 'string',
+        default: 'stdio'
+    },
+    port: {
+        type: 'string',
+        default: '3000'
+    },
+    host: {
+        type: 'string',
+        default: 'localhost'
+    }
+};
+function launchMCPServer(server, args) {
+    if ('http' === args.mode) return server.launchHttp({
+        port: Number.parseInt(args.port || '3000', 10),
+        host: args.host || 'localhost'
+    });
+    return server.launch();
+}
+const SESSION_TIMEOUT_MS = 1800000;
+const CLEANUP_INTERVAL_MS = 300000;
+const MAX_SESSIONS = 100;
+class BaseMCPServer {
+    async initializeToolsManager() {
+        setIsMcp(true);
+        this.toolsManager = this.createToolsManager();
+        try {
+            await this.toolsManager.initTools();
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            console.error(`Failed to initialize tools: ${message}`);
+            console.error('Tools will be initialized on first use');
+        }
+        this.toolsManager.attachToServer(this.mcpServer);
+    }
+    performCleanup() {
+        console.error(`${this.config.name} closing...`);
+        this.mcpServer.close();
+        this.toolsManager?.closeBrowser?.().catch(console.error);
+    }
+    async launch() {
+        await this.initializeToolsManager();
+        const transport = new StdioServerTransport();
+        try {
+            await this.mcpServer.connect(transport);
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            console.error(`Failed to connect MCP stdio transport: ${message}`);
+            throw new Error(`Failed to initialize MCP stdio transport: ${message}`);
+        }
+        process.on('uncaughtException', (error)=>{
+            console.error(`[${this.config.name}] Uncaught Exception:`, error);
+            console.error('Stack:', error.stack);
+        });
+        process.on('unhandledRejection', (reason)=>{
+            console.error(`[${this.config.name}] Unhandled Rejection:`, reason);
+            if (reason instanceof Error) console.error('Stack:', reason.stack);
+        });
+        process.stdin.on('close', ()=>this.performCleanup());
+        const cleanup = ()=>{
+            console.error(`${this.config.name} shutting down...`);
+            this.performCleanup();
+            process.exit(0);
+        };
+        process.once('SIGINT', cleanup);
+        process.once('SIGTERM', cleanup);
+    }
+    async launchHttp(options) {
+        if (!Number.isInteger(options.port) || options.port < 1 || options.port > 65535) throw new Error(`Invalid port number: ${options.port}. Port must be between 1 and 65535.`);
+        await this.initializeToolsManager();
+        const express = await import("express");
+        const app = express.default();
+        app.use(express.default.json({
+            limit: '10mb'
+        }));
+        const sessions = new Map();
+        app.all('/mcp', async (req, res)=>{
+            const startTime = Date.now();
+            const requestId = randomUUID().substring(0, 8);
+            try {
+                const rawSessionId = req.headers['mcp-session-id'];
+                const sessionId = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;
+                let session = sessionId ? sessions.get(sessionId) : void 0;
+                if (!session && 'POST' === req.method) {
+                    if (sessions.size >= MAX_SESSIONS) {
+                        console.error(`[${new Date().toISOString()}] [${requestId}] Session limit reached: ${sessions.size}/${MAX_SESSIONS}`);
+                        res.status(503).json({
+                            error: 'Too many active sessions',
+                            message: 'Server is at maximum capacity. Please try again later.'
+                        });
+                        return;
+                    }
+                    session = await this.createHttpSession(sessions);
+                    console.log(`[${new Date().toISOString()}] [${requestId}] New session created: ${session.transport.sessionId}`);
+                }
+                if (session) {
+                    session.lastAccessedAt = new Date();
+                    await session.transport.handleRequest(req, res, req.body);
+                    const duration = Date.now() - startTime;
+                    console.log(`[${new Date().toISOString()}] [${requestId}] Request completed in ${duration}ms`);
+                } else {
+                    console.error(`[${new Date().toISOString()}] [${requestId}] Invalid session or GET without session`);
+                    res.status(400).json({
+                        error: 'Invalid session or GET without session'
+                    });
+                }
+            } catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                const duration = Date.now() - startTime;
+                console.error(`[${new Date().toISOString()}] [${requestId}] MCP request error after ${duration}ms: ${message}`);
+                if (!res.headersSent) res.status(500).json({
+                    error: 'Internal server error',
+                    message: 'Failed to process MCP request'
+                });
+            }
+        });
+        const host = options.host || 'localhost';
+        const server = app.listen(options.port, host, ()=>{
+            console.log(`${this.config.name} HTTP server listening on http://${host}:${options.port}/mcp`);
+        }).on('error', (error)=>{
+            if ('EADDRINUSE' === error.code) console.error(`ERROR: Port ${options.port} is already in use.\nPlease try a different port: --port=<number>\nExample: --mode=http --port=${options.port + 1}`);
+            else if ('EACCES' === error.code) console.error(`ERROR: Permission denied to bind to port ${options.port}.\nPorts below 1024 require root/admin privileges.\nPlease use a port above 1024 or run with elevated privileges.`);
+            else console.error(`ERROR: Failed to start HTTP server on ${host}:${options.port}\nReason: ${error.message}\nCode: ${error.code || 'unknown'}`);
+            process.exit(1);
+        });
+        const cleanupInterval = this.startSessionCleanup(sessions);
+        this.setupHttpShutdownHandlers(server, sessions, cleanupInterval);
+    }
+    async createHttpSession(sessions) {
+        const transport = new StreamableHTTPServerTransport({
+            sessionIdGenerator: ()=>randomUUID(),
+            onsessioninitialized: (sid)=>{
+                sessions.set(sid, {
+                    transport,
+                    createdAt: new Date(),
+                    lastAccessedAt: new Date()
+                });
+                console.log(`[${new Date().toISOString()}] Session ${sid} initialized (total: ${sessions.size})`);
+            }
+        });
+        transport.onclose = ()=>{
+            if (transport.sessionId) {
+                sessions.delete(transport.sessionId);
+                console.log(`[${new Date().toISOString()}] Session ${transport.sessionId} closed (remaining: ${sessions.size})`);
+            }
+        };
+        try {
+            await this.mcpServer.connect(transport);
+        } catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            console.error(`[${new Date().toISOString()}] Failed to connect MCP transport: ${message}`);
+            if (transport.sessionId) sessions.delete(transport.sessionId);
+            throw new Error(`Failed to initialize MCP session: ${message}`);
+        }
+        return {
+            transport,
+            createdAt: new Date(),
+            lastAccessedAt: new Date()
+        };
+    }
+    startSessionCleanup(sessions) {
+        return setInterval(()=>{
+            const now = Date.now();
+            for (const [sid, session] of sessions)if (now - session.lastAccessedAt.getTime() > SESSION_TIMEOUT_MS) try {
+                session.transport.close();
+                sessions.delete(sid);
+                console.log(`[${new Date().toISOString()}] Session ${sid} cleaned up due to inactivity (remaining: ${sessions.size})`);
+            } catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                console.error(`[${new Date().toISOString()}] Failed to close session ${sid} during cleanup: ${message}`);
+                sessions.delete(sid);
+            }
+        }, CLEANUP_INTERVAL_MS);
+    }
+    setupHttpShutdownHandlers(server, sessions, cleanupInterval) {
+        const cleanup = ()=>{
+            console.error(`${this.config.name} shutting down...`);
+            clearInterval(cleanupInterval);
+            for (const session of sessions.values())try {
+                session.transport.close();
+            } catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                console.error(`Error closing session during shutdown: ${message}`);
+            }
+            sessions.clear();
+            try {
+                server.close(()=>{
+                    this.performCleanup();
+                    process.exit(0);
+                });
+                setTimeout(()=>{
+                    console.error('Forcefully shutting down after timeout');
+                    this.performCleanup();
+                    process.exit(1);
+                }, 5000);
+            } catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                console.error(`Error closing HTTP server: ${message}`);
+                this.performCleanup();
+                process.exit(1);
+            }
+        };
+        process.once('SIGINT', cleanup);
+        process.once('SIGTERM', cleanup);
+    }
+    getServer() {
+        return this.mcpServer;
+    }
+    getToolsManager() {
+        return this.toolsManager;
+    }
+    constructor(config){
+        _define_property(this, "mcpServer", void 0);
+        _define_property(this, "toolsManager", void 0);
+        _define_property(this, "config", void 0);
+        this.config = config;
+        this.mcpServer = new McpServer({
+            name: config.name,
+            version: config.version,
+            description: config.description
+        });
+    }
+}
+export { BaseMCPServer, CLI_ARGS_CONFIG, launchMCPServer };

package/dist/es/mcp/base-tools.mjs

@@ -0,0 +1,84 @@
+import { parseBase64 } from "@midscene/shared/img";
+import { getDebug } from "@midscene/shared/logger";
+import { generateCommonTools, generateToolsFromActionSpace } from "./tool-generator.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('mcp:base-tools');
+class BaseMidsceneTools {
+    preparePlatformTools() {
+        return [];
+    }
+    async initTools() {
+        this.toolDefinitions = [];
+        const platformTools = this.preparePlatformTools();
+        this.toolDefinitions.push(...platformTools);
+        let actionSpace;
+        try {
+            const agent = await this.ensureAgent();
+            actionSpace = await agent.getActionSpace();
+            debug('Action space from connected agent:', actionSpace.map((a)=>a.name).join(', '));
+        } catch (error) {
+            const errorMessage = error instanceof Error ? error.message : String(error);
+            errorMessage.includes('requires a URL') || errorMessage.includes('web_connect') ? debug('Bridge mode detected - agent will be initialized on first web_connect call') : debug('Agent not available yet, using temporary device for action space');
+            const tempDevice = this.createTemporaryDevice();
+            actionSpace = tempDevice.actionSpace();
+            debug('Action space from temporary device:', actionSpace.map((a)=>a.name).join(', '));
+            await tempDevice.destroy?.();
+        }
+        const actionTools = generateToolsFromActionSpace(actionSpace, ()=>this.ensureAgent());
+        const commonTools = generateCommonTools(()=>this.ensureAgent());
+        this.toolDefinitions.push(...actionTools, ...commonTools);
+        debug('Total tools prepared:', this.toolDefinitions.length);
+    }
+    attachToServer(server) {
+        this.mcpServer = server;
+        if (0 === this.toolDefinitions.length) debug('Warning: No tools to register. Tools may be initialized lazily.');
+        for (const toolDef of this.toolDefinitions)if (toolDef.autoDestroy) this.toolWithAutoDestroy(toolDef.name, toolDef.description, toolDef.schema, toolDef.handler);
+        else this.mcpServer.tool(toolDef.name, toolDef.description, toolDef.schema, toolDef.handler);
+        debug('Registered', this.toolDefinitions.length, 'tools');
+    }
+    toolWithAutoDestroy(name, description, schema, handler) {
+        if (!this.mcpServer) throw new Error('MCP server not attached');
+        this.mcpServer.tool(name, description, schema, async (...args)=>{
+            try {
+                return await handler(...args);
+            } finally{
+                if (!process.env.MIDSCENE_MCP_DISABLE_AGENT_AUTO_DESTROY) {
+                    try {
+                        await this.agent?.destroy?.();
+                    } catch (error) {
+                        debug('Failed to destroy agent during cleanup:', error);
+                    }
+                    this.agent = void 0;
+                }
+            }
+        });
+    }
+    async closeBrowser() {
+        await this.agent?.destroy?.();
+    }
+    buildScreenshotContent(screenshot) {
+        const { mimeType, body } = parseBase64(screenshot);
+        return [
+            {
+                type: 'image',
+                data: body,
+                mimeType
+            }
+        ];
+    }
+    constructor(){
+        _define_property(this, "mcpServer", void 0);
+        _define_property(this, "agent", void 0);
+        _define_property(this, "toolDefinitions", []);
+    }
+}
+export { BaseMidsceneTools };

package/dist/es/mcp/index.mjs

@@ -0,0 +1,4 @@
+export * from "./base-server.mjs";
+export * from "./base-tools.mjs";
+export * from "./tool-generator.mjs";
+export * from "./types.mjs";

package/dist/es/mcp/tool-generator.mjs

@@ -0,0 +1,215 @@
+import { parseBase64 } from "@midscene/shared/img";
+import { z } from "zod";
+function getErrorMessage(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function isZodOptional(value) {
+    return '_def' in value && value._def?.typeName === 'ZodOptional';
+}
+function isZodObject(value) {
+    return '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value;
+}
+function unwrapOptional(value) {
+    if (isZodOptional(value)) return {
+        innerValue: value._def.innerType,
+        isOptional: true
+    };
+    return {
+        innerValue: value,
+        isOptional: false
+    };
+}
+function isLocateField(value) {
+    if (!isZodObject(value)) return false;
+    return 'prompt' in value.shape;
+}
+function makePromptOptional(value, wrapInOptional) {
+    const newShape = {
+        ...value.shape
+    };
+    newShape.prompt = value.shape.prompt.optional();
+    let newSchema = z.object(newShape).passthrough();
+    if (wrapInOptional) newSchema = newSchema.optional();
+    return newSchema;
+}
+function transformSchemaField(key, value) {
+    const { innerValue, isOptional } = unwrapOptional(value);
+    if (isZodObject(innerValue) && isLocateField(innerValue)) return [
+        key,
+        makePromptOptional(innerValue, isOptional)
+    ];
+    return [
+        key,
+        value
+    ];
+}
+function extractActionSchema(paramSchema) {
+    if (!paramSchema) return {};
+    const schema = paramSchema;
+    if (!isZodObject(schema)) return schema;
+    return Object.fromEntries(Object.entries(schema.shape).map(([key, value])=>transformSchemaField(key, value)));
+}
+function serializeArgsToDescription(args) {
+    try {
+        return Object.entries(args).map(([key, value])=>{
+            if ('object' == typeof value && null !== value) try {
+                return `${key}: ${JSON.stringify(value)}`;
+            } catch  {
+                return `${key}: [object]`;
+            }
+            return `${key}: "${value}"`;
+        }).join(', ');
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error serializing args:', errorMessage);
+        return `[args serialization failed: ${errorMessage}]`;
+    }
+}
+function buildActionInstruction(actionName, args) {
+    const argsDescription = serializeArgsToDescription(args);
+    return argsDescription ? `Use the action "${actionName}" with ${argsDescription}` : `Use the action "${actionName}"`;
+}
+async function captureScreenshotResult(agent, actionName) {
+    try {
+        const screenshot = await agent.page?.screenshotBase64();
+        if (!screenshot) return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                }
+            ]
+        };
+        const { mimeType, body } = parseBase64(screenshot);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed.`
+                },
+                {
+                    type: 'image',
+                    data: body,
+                    mimeType
+                }
+            ]
+        };
+    } catch (error) {
+        const errorMessage = getErrorMessage(error);
+        console.error('Error capturing screenshot:', errorMessage);
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`
+                }
+            ]
+        };
+    }
+}
+function createErrorResult(message) {
+    return {
+        content: [
+            {
+                type: 'text',
+                text: message
+            }
+        ],
+        isError: true
+    };
+}
+function generateToolsFromActionSpace(actionSpace, getAgent) {
+    return actionSpace.map((action)=>{
+        const schema = extractActionSchema(action.paramSchema);
+        return {
+            name: action.name,
+            description: action.description || `Execute ${action.name} action`,
+            schema,
+            handler: async (args)=>{
+                try {
+                    const agent = await getAgent();
+                    if (agent.aiAction) {
+                        const instruction = buildActionInstruction(action.name, args);
+                        try {
+                            await agent.aiAction(instruction);
+                        } catch (error) {
+                            const errorMessage = getErrorMessage(error);
+                            console.error(`Error executing action "${action.name}":`, errorMessage);
+                            return createErrorResult(`Failed to execute action "${action.name}": ${errorMessage}`);
+                        }
+                    }
+                    return await captureScreenshotResult(agent, action.name);
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error(`Error in handler for "${action.name}":`, errorMessage);
+                    return createErrorResult(`Failed to get agent or execute action "${action.name}": ${errorMessage}`);
+                }
+            },
+            autoDestroy: true
+        };
+    });
+}
+function generateCommonTools(getAgent) {
+    return [
+        {
+            name: 'take_screenshot',
+            description: 'Capture screenshot of current page/screen',
+            schema: {},
+            handler: async ()=>{
+                try {
+                    const agent = await getAgent();
+                    const screenshot = await agent.page?.screenshotBase64();
+                    if (!screenshot) return createErrorResult('Screenshot not available');
+                    const { mimeType, body } = parseBase64(screenshot);
+                    return {
+                        content: [
+                            {
+                                type: 'image',
+                                data: body,
+                                mimeType
+                            }
+                        ]
+                    };
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error taking screenshot:', errorMessage);
+                    return createErrorResult(`Failed to capture screenshot: ${errorMessage}`);
+                }
+            },
+            autoDestroy: true
+        },
+        {
+            name: 'wait_for',
+            description: 'Wait until condition becomes true',
+            schema: {
+                assertion: z.string().describe('Condition to wait for'),
+                timeoutMs: z.number().optional().default(15000),
+                checkIntervalMs: z.number().optional().default(3000)
+            },
+            handler: async (args)=>{
+                try {
+                    const agent = await getAgent();
+                    const { assertion, timeoutMs, checkIntervalMs } = args;
+                    if (agent.aiWaitFor) await agent.aiWaitFor(assertion, {
+                        timeoutMs,
+                        checkIntervalMs
+                    });
+                    return {
+                        content: [
+                            {
+                                type: 'text',
+                                text: `Condition met: "${assertion}"`
+                            }
+                        ]
+                    };
+                } catch (error) {
+                    const errorMessage = getErrorMessage(error);
+                    console.error('Error in wait_for:', errorMessage);
+                    return createErrorResult(`Wait condition failed: ${errorMessage}`);
+                }
+            },
+            autoDestroy: true
+        }
+    ];
+}
+export { generateCommonTools, generateToolsFromActionSpace };

package/dist/es/mcp/types.mjs

@@ -0,0 +1,3 @@
+const defaultAppLoadingTimeoutMs = 10000;
+const defaultAppLoadingCheckIntervalMs = 2000;
+export { defaultAppLoadingCheckIntervalMs, defaultAppLoadingTimeoutMs };