@midscene/shared 0.30.10 → 1.0.0

package/src/mcp/base-tools.ts

@@ -0,0 +1,196 @@
+import { parseBase64 } from '@midscene/shared/img';
+import { getDebug } from '@midscene/shared/logger';
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import {
+  generateCommonTools,
+  generateToolsFromActionSpace,
+} from './tool-generator';
+import type {
+  ActionSpaceItem,
+  BaseAgent,
+  BaseDevice,
+  IMidsceneTools,
+  ToolDefinition,
+} from './types';
+
+const debug = getDebug('mcp:base-tools');
+
+/**
+ * Base class for platform-specific MCP tools
+ * Generic type TAgent allows subclasses to use their specific agent types
+ */
+export abstract class BaseMidsceneTools<TAgent extends BaseAgent = BaseAgent>
+  implements IMidsceneTools
+{
+  protected mcpServer?: McpServer;
+  protected agent?: TAgent;
+  protected toolDefinitions: ToolDefinition[] = [];
+
+  /**
+   * Ensure agent is initialized and ready for use.
+   * Must be implemented by subclasses to create platform-specific agent.
+   * @param initParam Optional initialization parameter (platform-specific, e.g., URL, device ID)
+   * @returns Promise resolving to initialized agent instance
+   * @throws Error if agent initialization fails
+   */
+  protected abstract ensureAgent(initParam?: string): Promise<TAgent>;
+
+  /**
+   * Optional: prepare platform-specific tools (e.g., device connection)
+   */
+  protected preparePlatformTools(): ToolDefinition[] {
+    return [];
+  }
+
+  /**
+   * Must be implemented by subclasses to create a temporary device instance
+   * This allows getting real actionSpace without connecting to device
+   */
+  protected abstract createTemporaryDevice(): BaseDevice;
+
+  /**
+   * Initialize all tools by querying actionSpace
+   * Uses two-layer fallback strategy:
+   * 1. Try to get actionSpace from connected agent (if available)
+   * 2. Create temporary device instance to read actionSpace (always succeeds)
+   */
+  public async initTools(): Promise<void> {
+    this.toolDefinitions = [];
+
+    // 1. Add platform-specific tools first (device connection, etc.)
+    // These don't require an agent and should always be available
+    const platformTools = this.preparePlatformTools();
+    this.toolDefinitions.push(...platformTools);
+
+    // 2. Try to get agent and its action space (two-layer fallback)
+    let actionSpace: ActionSpaceItem[];
+    try {
+      // Layer 1: Try to use connected agent
+      const agent = await this.ensureAgent();
+      actionSpace = await agent.getActionSpace();
+      debug(
+        'Action space from connected agent:',
+        actionSpace.map((a) => a.name).join(', '),
+      );
+    } catch (error) {
+      // Layer 2: Create temporary device instance to read actionSpace
+      // This is expected behavior for bridge mode without URL or unconnected devices
+      const errorMessage =
+        error instanceof Error ? error.message : String(error);
+      if (
+        errorMessage.includes('requires a URL') ||
+        errorMessage.includes('web_connect')
+      ) {
+        debug(
+          'Bridge mode detected - agent will be initialized on first web_connect call',
+        );
+      } else {
+        debug(
+          'Agent not available yet, using temporary device for action space',
+        );
+      }
+      const tempDevice = this.createTemporaryDevice();
+      actionSpace = tempDevice.actionSpace();
+      debug(
+        'Action space from temporary device:',
+        actionSpace.map((a) => a.name).join(', '),
+      );
+
+      // Destroy temporary instance using optional chaining
+      await tempDevice.destroy?.();
+    }
+
+    // 3. Generate tools from action space (core innovation)
+    const actionTools = generateToolsFromActionSpace(actionSpace, () =>
+      this.ensureAgent(),
+    );
+
+    // 4. Add common tools (screenshot, waitFor)
+    const commonTools = generateCommonTools(() => this.ensureAgent());
+
+    this.toolDefinitions.push(...actionTools, ...commonTools);
+
+    debug('Total tools prepared:', this.toolDefinitions.length);
+  }
+
+  /**
+   * Attach to MCP server and register all tools
+   */
+  public attachToServer(server: McpServer): void {
+    this.mcpServer = server;
+
+    if (this.toolDefinitions.length === 0) {
+      debug('Warning: No tools to register. Tools may be initialized lazily.');
+    }
+
+    for (const toolDef of this.toolDefinitions) {
+      if (toolDef.autoDestroy) {
+        this.toolWithAutoDestroy(
+          toolDef.name,
+          toolDef.description,
+          toolDef.schema,
+          toolDef.handler,
+        );
+      } else {
+        this.mcpServer.tool(
+          toolDef.name,
+          toolDef.description,
+          toolDef.schema,
+          toolDef.handler,
+        );
+      }
+    }
+
+    debug('Registered', this.toolDefinitions.length, 'tools');
+  }
+
+  /**
+   * Wrapper for auto-destroy behavior
+   */
+  private toolWithAutoDestroy(
+    name: string,
+    description: string,
+    schema: any,
+    handler: (...args: any[]) => Promise<any>,
+  ): void {
+    if (!this.mcpServer) {
+      throw new Error('MCP server not attached');
+    }
+
+    this.mcpServer.tool(name, description, schema, async (...args: any[]) => {
+      try {
+        return await handler(...args);
+      } finally {
+        if (!process.env.MIDSCENE_MCP_DISABLE_AGENT_AUTO_DESTROY) {
+          try {
+            await this.agent?.destroy?.();
+          } catch (error) {
+            debug('Failed to destroy agent during cleanup:', error);
+          }
+          this.agent = undefined;
+        }
+      }
+    });
+  }
+
+  /**
+   * Cleanup method - destroy agent and release resources
+   */
+  public async closeBrowser(): Promise<void> {
+    await this.agent?.destroy?.();
+  }
+
+  /**
+   * Helper: Convert base64 screenshot to image content array
+   */
+  protected buildScreenshotContent(screenshot: string) {
+    const { mimeType, body } = parseBase64(screenshot);
+    return [
+      {
+        type: 'image' as const,
+        data: body,
+        mimeType,
+      },
+    ];
+  }
+}

package/src/mcp/index.ts

@@ -0,0 +1,5 @@
+export * from './base-server';
+export * from './base-tools';
+export * from './tool-generator';
+export * from './types';
+export * from './inject-report-html-plugin';

package/src/mcp/inject-report-html-plugin.ts

@@ -0,0 +1,119 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+const MAGIC_STRING = 'REPLACE_ME_WITH_REPORT_HTML';
+const REPLACED_MARK = '/*REPORT_HTML_REPLACED*/';
+const REG_EXP_FOR_REPLACE = /\/\*REPORT_HTML_REPLACED\*\/.*/;
+
+interface RslibPluginApi {
+  onAfterBuild: (callback: () => void) => void;
+}
+
+/**
+ * Rslib plugin to inject report HTML from @midscene/core dist into MCP bundle.
+ * This runs after build and reads the already-injected HTML from core.
+ *
+ * Prerequisites:
+ * - @midscene/report must be in devDependencies to ensure correct build order
+ * - @midscene/core dist must exist with injected HTML
+ *
+ * @param packageDir - The directory of the MCP package (use __dirname)
+ */
+export function injectReportHtmlFromCore(packageDir: string) {
+  return {
+    name: 'inject-report-html-from-core',
+    setup(api: RslibPluginApi) {
+      api.onAfterBuild(() => {
+        const coreUtilsPath = path.resolve(
+          packageDir,
+          '..',
+          'core',
+          'dist',
+          'lib',
+          'utils.js',
+        );
+
+        if (!fs.existsSync(coreUtilsPath)) {
+          console.warn(
+            '[inject-report-html] @midscene/core dist not found, skipping',
+          );
+          return;
+        }
+
+        const coreContent = fs.readFileSync(coreUtilsPath, 'utf-8');
+        if (!coreContent.includes(REPLACED_MARK)) {
+          console.warn(
+            '[inject-report-html] HTML not found in core dist. Ensure report builds first.',
+          );
+          return;
+        }
+
+        // Extract the JSON string after the marker
+        // JSON strings can contain escaped quotes, so we need to properly parse it
+        const markerIndex = coreContent.indexOf(REPLACED_MARK);
+        const jsonStart = markerIndex + REPLACED_MARK.length;
+
+        // Find the end of the JSON string by tracking quote escaping
+        let jsonEnd = jsonStart;
+        if (coreContent[jsonStart] === '"') {
+          jsonEnd = jsonStart + 1;
+          while (jsonEnd < coreContent.length) {
+            if (coreContent[jsonEnd] === '\\') {
+              jsonEnd += 2; // Skip escaped character
+            } else if (coreContent[jsonEnd] === '"') {
+              jsonEnd += 1; // Include closing quote
+              break;
+            } else {
+              jsonEnd += 1;
+            }
+          }
+        }
+
+        const jsonString = coreContent.slice(jsonStart, jsonEnd);
+        if (!jsonString || jsonString.length < 10) {
+          console.warn('[inject-report-html] Failed to extract HTML from core');
+          return;
+        }
+
+        const finalContent = `${REPLACED_MARK}${jsonString}`;
+        const distDir = path.join(packageDir, 'dist');
+
+        if (!fs.existsSync(distDir)) return;
+
+        const jsFiles = fs
+          .readdirSync(distDir)
+          .filter((f) => f.endsWith('.js'));
+        let injectedCount = 0;
+
+        for (const file of jsFiles) {
+          const filePath = path.join(distDir, file);
+          const content = fs.readFileSync(filePath, 'utf-8');
+
+          if (content.includes(REPLACED_MARK)) {
+            if (REG_EXP_FOR_REPLACE.test(content)) {
+              fs.writeFileSync(
+                filePath,
+                content.replace(REG_EXP_FOR_REPLACE, () => finalContent),
+              );
+              console.log(`[inject-report-html] Updated: ${file}`);
+              injectedCount++;
+            }
+          } else if (content.includes(`'${MAGIC_STRING}'`)) {
+            fs.writeFileSync(
+              filePath,
+              content.replace(`'${MAGIC_STRING}'`, () => finalContent),
+            );
+            console.log(`[inject-report-html] Injected: ${file}`);
+            injectedCount++;
+          }
+        }
+
+        if (injectedCount > 0) {
+          console.log(
+            `[inject-report-html] Completed: ${injectedCount} file(s)`,
+          );
+        }
+      });
+    },
+  };
+}

package/src/mcp/tool-generator.ts

@@ -0,0 +1,330 @@
+import { parseBase64 } from '@midscene/shared/img';
+import { z } from 'zod';
+import { getZodDescription, getZodTypeName } from '../zod-schema-utils';
+import type {
+  ActionSpaceItem,
+  BaseAgent,
+  ToolDefinition,
+  ToolResult,
+} from './types';
+
+/**
+ * Extract error message from unknown error type
+ */
+function getErrorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/**
+ * Generate MCP tool description from ActionSpaceItem
+ * Format: "actionName action, description. Parameters: param1 (type) - desc; param2 (type) - desc"
+ */
+function describeActionForMCP(action: ActionSpaceItem): string {
+  const actionDesc = action.description || `Execute ${action.name} action`;
+
+  if (!action.paramSchema) {
+    return `${action.name} action, ${actionDesc}`;
+  }
+
+  const schema = action.paramSchema as {
+    _def?: { typeName?: string };
+    shape?: Record<string, unknown>;
+  };
+  const isZodObjectType = schema._def?.typeName === 'ZodObject';
+
+  if (!isZodObjectType || !schema.shape) {
+    // Simple type schema
+    const typeName = getZodTypeName(schema);
+    const description = getZodDescription(schema as z.ZodTypeAny);
+    const paramDesc = description ? `${typeName} - ${description}` : typeName;
+    return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
+  }
+
+  // Object schema with multiple fields
+  const paramDescriptions: string[] = [];
+  for (const [key, field] of Object.entries(schema.shape)) {
+    if (field && typeof field === 'object') {
+      const isFieldOptional =
+        typeof (field as { isOptional?: () => boolean }).isOptional ===
+          'function' && (field as { isOptional: () => boolean }).isOptional();
+      const typeName = getZodTypeName(field);
+      const description = getZodDescription(field as z.ZodTypeAny);
+
+      let paramStr = `${key}${isFieldOptional ? '?' : ''} (${typeName})`;
+      if (description) {
+        paramStr += ` - ${description}`;
+      }
+      paramDescriptions.push(paramStr);
+    }
+  }
+
+  if (paramDescriptions.length === 0) {
+    return `${action.name} action, ${actionDesc}`;
+  }
+
+  return `${action.name} action, ${actionDesc}. Parameters: ${paramDescriptions.join('; ')}`;
+}
+
+/**
+ * Type guard: check if a Zod type is ZodOptional
+ */
+function isZodOptional(
+  value: z.ZodTypeAny,
+): value is z.ZodOptional<z.ZodTypeAny> {
+  return '_def' in value && value._def?.typeName === 'ZodOptional';
+}
+
+/**
+ * Type guard: check if a Zod type is ZodObject
+ */
+function isZodObject(value: z.ZodTypeAny): value is z.ZodObject<z.ZodRawShape> {
+  return (
+    '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value
+  );
+}
+
+/**
+ * Unwrap ZodOptional to get inner type
+ */
+function unwrapOptional(value: z.ZodTypeAny): {
+  innerValue: z.ZodTypeAny;
+  isOptional: boolean;
+} {
+  if (isZodOptional(value)) {
+    return { innerValue: value._def.innerType, isOptional: true };
+  }
+  return { innerValue: value, isOptional: false };
+}
+
+/**
+ * Check if a Zod object schema contains a 'prompt' field (locate field pattern)
+ */
+function isLocateField(value: z.ZodTypeAny): boolean {
+  if (!isZodObject(value)) {
+    return false;
+  }
+  return 'prompt' in value.shape;
+}
+
+/**
+ * Transform a locate field schema to make its 'prompt' field optional
+ */
+function makePromptOptional(
+  value: z.ZodObject<z.ZodRawShape>,
+  wrapInOptional: boolean,
+): z.ZodTypeAny {
+  const newShape = { ...value.shape };
+  newShape.prompt = value.shape.prompt.optional();
+
+  let newSchema: z.ZodTypeAny = z.object(newShape).passthrough();
+  if (wrapInOptional) {
+    newSchema = newSchema.optional();
+  }
+  return newSchema;
+}
+
+/**
+ * Transform schema field to make locate.prompt optional if applicable
+ */
+function transformSchemaField(
+  key: string,
+  value: z.ZodTypeAny,
+): [string, z.ZodTypeAny] {
+  const { innerValue, isOptional } = unwrapOptional(value);
+
+  if (isZodObject(innerValue) && isLocateField(innerValue)) {
+    return [key, makePromptOptional(innerValue, isOptional)];
+  }
+  return [key, value];
+}
+
+/**
+ * Extract and transform schema from action's paramSchema
+ */
+function extractActionSchema(
+  paramSchema: z.ZodTypeAny | undefined,
+): Record<string, z.ZodTypeAny> {
+  if (!paramSchema) {
+    return {};
+  }
+
+  const schema = paramSchema as z.ZodTypeAny;
+  if (!isZodObject(schema)) {
+    return schema as unknown as Record<string, z.ZodTypeAny>;
+  }
+
+  return Object.fromEntries(
+    Object.entries(schema.shape).map(([key, value]) =>
+      transformSchemaField(key, value as z.ZodTypeAny),
+    ),
+  );
+}
+
+/**
+ * Serialize args to human-readable description for AI action
+ */
+function serializeArgsToDescription(args: Record<string, unknown>): string {
+  try {
+    return Object.entries(args)
+      .map(([key, value]) => {
+        if (typeof value === 'object' && value !== null) {
+          try {
+            return `${key}: ${JSON.stringify(value)}`;
+          } catch {
+            // Circular reference or non-serializable object
+            return `${key}: [object]`;
+          }
+        }
+        return `${key}: "${value}"`;
+      })
+      .join(', ');
+  } catch (error: unknown) {
+    const errorMessage = getErrorMessage(error);
+    console.error('Error serializing args:', errorMessage);
+    return `[args serialization failed: ${errorMessage}]`;
+  }
+}
+
+/**
+ * Build action instruction string from action name and args
+ */
+function buildActionInstruction(
+  actionName: string,
+  args: Record<string, unknown>,
+): string {
+  const argsDescription = serializeArgsToDescription(args);
+  return argsDescription
+    ? `Use the action "${actionName}" with ${argsDescription}`
+    : `Use the action "${actionName}"`;
+}
+
+/**
+ * Capture screenshot and return as tool result
+ */
+async function captureScreenshotResult(
+  agent: BaseAgent,
+  actionName: string,
+): Promise<ToolResult> {
+  try {
+    const screenshot = await agent.page?.screenshotBase64();
+    if (!screenshot) {
+      return {
+        content: [{ type: 'text', text: `Action "${actionName}" completed.` }],
+      };
+    }
+
+    const { mimeType, body } = parseBase64(screenshot);
+    return {
+      content: [
+        { type: 'text', text: `Action "${actionName}" completed.` },
+        { type: 'image', data: body, mimeType },
+      ],
+    };
+  } catch (error: unknown) {
+    const errorMessage = getErrorMessage(error);
+    console.error('Error capturing screenshot:', errorMessage);
+    return {
+      content: [
+        {
+          type: 'text',
+          text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`,
+        },
+      ],
+    };
+  }
+}
+
+/**
+ * Create error result for tool handler
+ */
+function createErrorResult(message: string): ToolResult {
+  return {
+    content: [{ type: 'text', text: message }],
+    isError: true,
+  };
+}
+
+/**
+ * Converts DeviceAction from actionSpace into MCP ToolDefinition
+ * This is the core logic that removes need for hardcoded tool definitions
+ */
+export function generateToolsFromActionSpace(
+  actionSpace: ActionSpaceItem[],
+  getAgent: () => Promise<BaseAgent>,
+): ToolDefinition[] {
+  return actionSpace.map((action) => {
+    const schema = extractActionSchema(action.paramSchema as z.ZodTypeAny);
+
+    return {
+      name: action.name,
+      description: describeActionForMCP(action),
+      schema,
+      handler: async (args: Record<string, unknown>) => {
+        try {
+          const agent = await getAgent();
+
+          if (agent.aiAction) {
+            const instruction = buildActionInstruction(action.name, args);
+            try {
+              await agent.aiAction(instruction);
+            } catch (error: unknown) {
+              const errorMessage = getErrorMessage(error);
+              console.error(
+                `Error executing action "${action.name}":`,
+                errorMessage,
+              );
+              return createErrorResult(
+                `Failed to execute action "${action.name}": ${errorMessage}`,
+              );
+            }
+          }
+
+          return await captureScreenshotResult(agent, action.name);
+        } catch (error: unknown) {
+          const errorMessage = getErrorMessage(error);
+          console.error(`Error in handler for "${action.name}":`, errorMessage);
+          return createErrorResult(
+            `Failed to get agent or execute action "${action.name}": ${errorMessage}`,
+          );
+        }
+      },
+      autoDestroy: true,
+    };
+  });
+}
+
+/**
+ * Generate common tools (screenshot, waitFor)
+ * SIMPLIFIED: Only keep essential helper tools, removed assert
+ */
+export function generateCommonTools(
+  getAgent: () => Promise<BaseAgent>,
+): ToolDefinition[] {
+  return [
+    {
+      name: 'take_screenshot',
+      description: 'Capture screenshot of current page/screen',
+      schema: {},
+      handler: async (): Promise<ToolResult> => {
+        try {
+          const agent = await getAgent();
+          const screenshot = await agent.page?.screenshotBase64();
+          if (!screenshot) {
+            return createErrorResult('Screenshot not available');
+          }
+          const { mimeType, body } = parseBase64(screenshot);
+          return {
+            content: [{ type: 'image', data: body, mimeType }],
+          };
+        } catch (error: unknown) {
+          const errorMessage = getErrorMessage(error);
+          console.error('Error taking screenshot:', errorMessage);
+          return createErrorResult(
+            `Failed to capture screenshot: ${errorMessage}`,
+          );
+        }
+      },
+      autoDestroy: true,
+    },
+  ];
+}