@midscene/shared 1.0.1-beta-20251208112226.0 → 1.0.1-beta-20251209024153.0

package/src/mcp/inject-report-html-plugin.ts

@@ -0,0 +1,119 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+const MAGIC_STRING = 'REPLACE_ME_WITH_REPORT_HTML';
+const REPLACED_MARK = '/*REPORT_HTML_REPLACED*/';
+const REG_EXP_FOR_REPLACE = /\/\*REPORT_HTML_REPLACED\*\/.*/;
+
+interface RslibPluginApi {
+  onAfterBuild: (callback: () => void) => void;
+}
+
+/**
+ * Rslib plugin to inject report HTML from @midscene/core dist into MCP bundle.
+ * This runs after build and reads the already-injected HTML from core.
+ *
+ * Prerequisites:
+ * - @midscene/report must be in devDependencies to ensure correct build order
+ * - @midscene/core dist must exist with injected HTML
+ *
+ * @param packageDir - The directory of the MCP package (use __dirname)
+ */
+export function injectReportHtmlFromCore(packageDir: string) {
+  return {
+    name: 'inject-report-html-from-core',
+    setup(api: RslibPluginApi) {
+      api.onAfterBuild(() => {
+        const coreUtilsPath = path.resolve(
+          packageDir,
+          '..',
+          'core',
+          'dist',
+          'lib',
+          'utils.js',
+        );
+
+        if (!fs.existsSync(coreUtilsPath)) {
+          console.warn(
+            '[inject-report-html] @midscene/core dist not found, skipping',
+          );
+          return;
+        }
+
+        const coreContent = fs.readFileSync(coreUtilsPath, 'utf-8');
+        if (!coreContent.includes(REPLACED_MARK)) {
+          console.warn(
+            '[inject-report-html] HTML not found in core dist. Ensure report builds first.',
+          );
+          return;
+        }
+
+        // Extract the JSON string after the marker
+        // JSON strings can contain escaped quotes, so we need to properly parse it
+        const markerIndex = coreContent.indexOf(REPLACED_MARK);
+        const jsonStart = markerIndex + REPLACED_MARK.length;
+
+        // Find the end of the JSON string by tracking quote escaping
+        let jsonEnd = jsonStart;
+        if (coreContent[jsonStart] === '"') {
+          jsonEnd = jsonStart + 1;
+          while (jsonEnd < coreContent.length) {
+            if (coreContent[jsonEnd] === '\\') {
+              jsonEnd += 2; // Skip escaped character
+            } else if (coreContent[jsonEnd] === '"') {
+              jsonEnd += 1; // Include closing quote
+              break;
+            } else {
+              jsonEnd += 1;
+            }
+          }
+        }
+
+        const jsonString = coreContent.slice(jsonStart, jsonEnd);
+        if (!jsonString || jsonString.length < 10) {
+          console.warn('[inject-report-html] Failed to extract HTML from core');
+          return;
+        }
+
+        const finalContent = `${REPLACED_MARK}${jsonString}`;
+        const distDir = path.join(packageDir, 'dist');
+
+        if (!fs.existsSync(distDir)) return;
+
+        const jsFiles = fs
+          .readdirSync(distDir)
+          .filter((f) => f.endsWith('.js'));
+        let injectedCount = 0;
+
+        for (const file of jsFiles) {
+          const filePath = path.join(distDir, file);
+          const content = fs.readFileSync(filePath, 'utf-8');
+
+          if (content.includes(REPLACED_MARK)) {
+            if (REG_EXP_FOR_REPLACE.test(content)) {
+              fs.writeFileSync(
+                filePath,
+                content.replace(REG_EXP_FOR_REPLACE, () => finalContent),
+              );
+              console.log(`[inject-report-html] Updated: ${file}`);
+              injectedCount++;
+            }
+          } else if (content.includes(`'${MAGIC_STRING}'`)) {
+            fs.writeFileSync(
+              filePath,
+              content.replace(`'${MAGIC_STRING}'`, () => finalContent),
+            );
+            console.log(`[inject-report-html] Injected: ${file}`);
+            injectedCount++;
+          }
+        }
+
+        if (injectedCount > 0) {
+          console.log(
+            `[inject-report-html] Completed: ${injectedCount} file(s)`,
+          );
+        }
+      });
+    },
+  };
+}

package/src/mcp/tool-generator.ts

@@ -0,0 +1,330 @@
+import { parseBase64 } from '@midscene/shared/img';
+import { z } from 'zod';
+import { getZodDescription, getZodTypeName } from '../zod-schema-utils';
+import type {
+  ActionSpaceItem,
+  BaseAgent,
+  ToolDefinition,
+  ToolResult,
+} from './types';
+
+/**
+ * Extract error message from unknown error type
+ */
+function getErrorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/**
+ * Generate MCP tool description from ActionSpaceItem
+ * Format: "actionName action, description. Parameters: param1 (type) - desc; param2 (type) - desc"
+ */
+function describeActionForMCP(action: ActionSpaceItem): string {
+  const actionDesc = action.description || `Execute ${action.name} action`;
+
+  if (!action.paramSchema) {
+    return `${action.name} action, ${actionDesc}`;
+  }
+
+  const schema = action.paramSchema as {
+    _def?: { typeName?: string };
+    shape?: Record<string, unknown>;
+  };
+  const isZodObjectType = schema._def?.typeName === 'ZodObject';
+
+  if (!isZodObjectType || !schema.shape) {
+    // Simple type schema
+    const typeName = getZodTypeName(schema);
+    const description = getZodDescription(schema as z.ZodTypeAny);
+    const paramDesc = description ? `${typeName} - ${description}` : typeName;
+    return `${action.name} action, ${actionDesc}. Parameter: ${paramDesc}`;
+  }
+
+  // Object schema with multiple fields
+  const paramDescriptions: string[] = [];
+  for (const [key, field] of Object.entries(schema.shape)) {
+    if (field && typeof field === 'object') {
+      const isFieldOptional =
+        typeof (field as { isOptional?: () => boolean }).isOptional ===
+          'function' && (field as { isOptional: () => boolean }).isOptional();
+      const typeName = getZodTypeName(field);
+      const description = getZodDescription(field as z.ZodTypeAny);
+
+      let paramStr = `${key}${isFieldOptional ? '?' : ''} (${typeName})`;
+      if (description) {
+        paramStr += ` - ${description}`;
+      }
+      paramDescriptions.push(paramStr);
+    }
+  }
+
+  if (paramDescriptions.length === 0) {
+    return `${action.name} action, ${actionDesc}`;
+  }
+
+  return `${action.name} action, ${actionDesc}. Parameters: ${paramDescriptions.join('; ')}`;
+}
+
+/**
+ * Type guard: check if a Zod type is ZodOptional
+ */
+function isZodOptional(
+  value: z.ZodTypeAny,
+): value is z.ZodOptional<z.ZodTypeAny> {
+  return '_def' in value && value._def?.typeName === 'ZodOptional';
+}
+
+/**
+ * Type guard: check if a Zod type is ZodObject
+ */
+function isZodObject(value: z.ZodTypeAny): value is z.ZodObject<z.ZodRawShape> {
+  return (
+    '_def' in value && value._def?.typeName === 'ZodObject' && 'shape' in value
+  );
+}
+
+/**
+ * Unwrap ZodOptional to get inner type
+ */
+function unwrapOptional(value: z.ZodTypeAny): {
+  innerValue: z.ZodTypeAny;
+  isOptional: boolean;
+} {
+  if (isZodOptional(value)) {
+    return { innerValue: value._def.innerType, isOptional: true };
+  }
+  return { innerValue: value, isOptional: false };
+}
+
+/**
+ * Check if a Zod object schema contains a 'prompt' field (locate field pattern)
+ */
+function isLocateField(value: z.ZodTypeAny): boolean {
+  if (!isZodObject(value)) {
+    return false;
+  }
+  return 'prompt' in value.shape;
+}
+
+/**
+ * Transform a locate field schema to make its 'prompt' field optional
+ */
+function makePromptOptional(
+  value: z.ZodObject<z.ZodRawShape>,
+  wrapInOptional: boolean,
+): z.ZodTypeAny {
+  const newShape = { ...value.shape };
+  newShape.prompt = value.shape.prompt.optional();
+
+  let newSchema: z.ZodTypeAny = z.object(newShape).passthrough();
+  if (wrapInOptional) {
+    newSchema = newSchema.optional();
+  }
+  return newSchema;
+}
+
+/**
+ * Transform schema field to make locate.prompt optional if applicable
+ */
+function transformSchemaField(
+  key: string,
+  value: z.ZodTypeAny,
+): [string, z.ZodTypeAny] {
+  const { innerValue, isOptional } = unwrapOptional(value);
+
+  if (isZodObject(innerValue) && isLocateField(innerValue)) {
+    return [key, makePromptOptional(innerValue, isOptional)];
+  }
+  return [key, value];
+}
+
+/**
+ * Extract and transform schema from action's paramSchema
+ */
+function extractActionSchema(
+  paramSchema: z.ZodTypeAny | undefined,
+): Record<string, z.ZodTypeAny> {
+  if (!paramSchema) {
+    return {};
+  }
+
+  const schema = paramSchema as z.ZodTypeAny;
+  if (!isZodObject(schema)) {
+    return schema as unknown as Record<string, z.ZodTypeAny>;
+  }
+
+  return Object.fromEntries(
+    Object.entries(schema.shape).map(([key, value]) =>
+      transformSchemaField(key, value as z.ZodTypeAny),
+    ),
+  );
+}
+
+/**
+ * Serialize args to human-readable description for AI action
+ */
+function serializeArgsToDescription(args: Record<string, unknown>): string {
+  try {
+    return Object.entries(args)
+      .map(([key, value]) => {
+        if (typeof value === 'object' && value !== null) {
+          try {
+            return `${key}: ${JSON.stringify(value)}`;
+          } catch {
+            // Circular reference or non-serializable object
+            return `${key}: [object]`;
+          }
+        }
+        return `${key}: "${value}"`;
+      })
+      .join(', ');
+  } catch (error: unknown) {
+    const errorMessage = getErrorMessage(error);
+    console.error('Error serializing args:', errorMessage);
+    return `[args serialization failed: ${errorMessage}]`;
+  }
+}
+
+/**
+ * Build action instruction string from action name and args
+ */
+function buildActionInstruction(
+  actionName: string,
+  args: Record<string, unknown>,
+): string {
+  const argsDescription = serializeArgsToDescription(args);
+  return argsDescription
+    ? `Use the action "${actionName}" with ${argsDescription}`
+    : `Use the action "${actionName}"`;
+}
+
+/**
+ * Capture screenshot and return as tool result
+ */
+async function captureScreenshotResult(
+  agent: BaseAgent,
+  actionName: string,
+): Promise<ToolResult> {
+  try {
+    const screenshot = await agent.page?.screenshotBase64();
+    if (!screenshot) {
+      return {
+        content: [{ type: 'text', text: `Action "${actionName}" completed.` }],
+      };
+    }
+
+    const { mimeType, body } = parseBase64(screenshot);
+    return {
+      content: [
+        { type: 'text', text: `Action "${actionName}" completed.` },
+        { type: 'image', data: body, mimeType },
+      ],
+    };
+  } catch (error: unknown) {
+    const errorMessage = getErrorMessage(error);
+    console.error('Error capturing screenshot:', errorMessage);
+    return {
+      content: [
+        {
+          type: 'text',
+          text: `Action "${actionName}" completed (screenshot unavailable: ${errorMessage})`,
+        },
+      ],
+    };
+  }
+}
+
+/**
+ * Create error result for tool handler
+ */
+function createErrorResult(message: string): ToolResult {
+  return {
+    content: [{ type: 'text', text: message }],
+    isError: true,
+  };
+}
+
+/**
+ * Converts DeviceAction from actionSpace into MCP ToolDefinition
+ * This is the core logic that removes need for hardcoded tool definitions
+ */
+export function generateToolsFromActionSpace(
+  actionSpace: ActionSpaceItem[],
+  getAgent: () => Promise<BaseAgent>,
+): ToolDefinition[] {
+  return actionSpace.map((action) => {
+    const schema = extractActionSchema(action.paramSchema as z.ZodTypeAny);
+
+    return {
+      name: action.name,
+      description: describeActionForMCP(action),
+      schema,
+      handler: async (args: Record<string, unknown>) => {
+        try {
+          const agent = await getAgent();
+
+          if (agent.aiAction) {
+            const instruction = buildActionInstruction(action.name, args);
+            try {
+              await agent.aiAction(instruction);
+            } catch (error: unknown) {
+              const errorMessage = getErrorMessage(error);
+              console.error(
+                `Error executing action "${action.name}":`,
+                errorMessage,
+              );
+              return createErrorResult(
+                `Failed to execute action "${action.name}": ${errorMessage}`,
+              );
+            }
+          }
+
+          return await captureScreenshotResult(agent, action.name);
+        } catch (error: unknown) {
+          const errorMessage = getErrorMessage(error);
+          console.error(`Error in handler for "${action.name}":`, errorMessage);
+          return createErrorResult(
+            `Failed to get agent or execute action "${action.name}": ${errorMessage}`,
+          );
+        }
+      },
+      autoDestroy: true,
+    };
+  });
+}
+
+/**
+ * Generate common tools (screenshot, waitFor)
+ * SIMPLIFIED: Only keep essential helper tools, removed assert
+ */
+export function generateCommonTools(
+  getAgent: () => Promise<BaseAgent>,
+): ToolDefinition[] {
+  return [
+    {
+      name: 'take_screenshot',
+      description: 'Capture screenshot of current page/screen',
+      schema: {},
+      handler: async (): Promise<ToolResult> => {
+        try {
+          const agent = await getAgent();
+          const screenshot = await agent.page?.screenshotBase64();
+          if (!screenshot) {
+            return createErrorResult('Screenshot not available');
+          }
+          const { mimeType, body } = parseBase64(screenshot);
+          return {
+            content: [{ type: 'image', data: body, mimeType }],
+          };
+        } catch (error: unknown) {
+          const errorMessage = getErrorMessage(error);
+          console.error('Error taking screenshot:', errorMessage);
+          return createErrorResult(
+            `Failed to capture screenshot: ${errorMessage}`,
+          );
+        }
+      },
+      autoDestroy: true,
+    },
+  ];
+}

package/src/mcp/types.ts

@@ -0,0 +1,108 @@
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import type { z } from 'zod';
+
+// Avoid circular dependency: don't import from @midscene/core
+// Instead, use generic types that will be provided by implementation
+
+/**
+ * Default timeout constants for app loading verification
+ */
+export const defaultAppLoadingTimeoutMs = 10000;
+export const defaultAppLoadingCheckIntervalMs = 2000;
+
+/**
+ * Content item types for tool results (MCP compatible)
+ */
+export type ToolResultContent =
+  | { type: 'text'; text: string }
+  | { type: 'image'; data: string; mimeType: string }
+  | { type: 'audio'; data: string; mimeType: string }
+  | {
+      type: 'resource';
+      resource:
+        | { text: string; uri: string; mimeType?: string }
+        | { uri: string; blob: string; mimeType?: string };
+    };
+
+/**
+ * Result type for tool execution (MCP compatible)
+ */
+export interface ToolResult {
+  [x: string]: unknown;
+  content: ToolResultContent[];
+  isError?: boolean;
+  _meta?: Record<string, unknown>;
+}
+
+/**
+ * Tool handler function type
+ * Takes parsed arguments and returns a tool result
+ */
+export type ToolHandler<T = Record<string, unknown>> = (
+  args: T,
+) => Promise<ToolResult>;
+
+/**
+ * Tool schema type using Zod
+ */
+export type ToolSchema = Record<string, z.ZodTypeAny>;
+
+/**
+ * Tool definition for MCP server
+ */
+export interface ToolDefinition<T = Record<string, unknown>> {
+  name: string;
+  description: string;
+  schema: ToolSchema;
+  handler: ToolHandler<T>;
+  autoDestroy?: boolean;
+}
+
+/**
+ * Action space item definition
+ * Note: Intentionally no index signature to maintain compatibility with DeviceAction
+ */
+export interface ActionSpaceItem {
+  name: string;
+  description?: string;
+  args?: Record<string, unknown>;
+  paramSchema?: z.ZodTypeAny;
+}
+
+/**
+ * Base agent interface
+ * Represents a platform-specific agent (Android, iOS, Web)
+ * Note: Return types use `unknown` for compatibility with platform-specific implementations
+ */
+export interface BaseAgent {
+  getActionSpace(): Promise<ActionSpaceItem[]>;
+  destroy?(): Promise<void>;
+  page?: {
+    screenshotBase64(): Promise<string>;
+  };
+  aiAction?: (
+    description: string,
+    params?: Record<string, unknown>,
+  ) => Promise<unknown>;
+  aiWaitFor?: (
+    assertion: string,
+    options: Record<string, unknown>,
+  ) => Promise<unknown>;
+}
+
+/**
+ * Base device interface for temporary device instances
+ */
+export interface BaseDevice {
+  actionSpace(): ActionSpaceItem[];
+  destroy?(): Promise<void>;
+}
+
+/**
+ * Interface for platform-specific MCP tools manager
+ */
+export interface IMidsceneTools {
+  attachToServer(server: McpServer): void;
+  initTools(): Promise<void>;
+  closeBrowser?(): Promise<void>;
+}

package/src/zod-schema-utils.ts

@@ -0,0 +1,133 @@
+import type { z } from 'zod';
+
+/**
+ * Recursively unwrap optional, nullable, default, and effects wrapper types
+ * to get the actual inner Zod type
+ */
+export function unwrapZodField(field: unknown): unknown {
+  const f = field as {
+    _def?: { typeName?: string; innerType?: unknown; schema?: unknown };
+  };
+  if (!f._def) return f;
+
+  const typeName = f._def.typeName;
+
+  // Handle wrapper types that have innerType
+  if (
+    typeName === 'ZodOptional' ||
+    typeName === 'ZodNullable' ||
+    typeName === 'ZodDefault'
+  ) {
+    return unwrapZodField(f._def.innerType);
+  }
+
+  // Handle ZodEffects (transformations, refinements, preprocessors)
+  if (typeName === 'ZodEffects') {
+    if (f._def.schema) {
+      return unwrapZodField(f._def.schema);
+    }
+  }
+
+  return f;
+}
+
+/**
+ * Check if a field is a Midscene locator field
+ * Checks for either:
+ * 1. midscene_location_field_flag in shape (result schema)
+ * 2. prompt field in shape (input schema)
+ */
+export function isMidsceneLocatorField(field: unknown): boolean {
+  const actualField = unwrapZodField(field) as {
+    _def?: { typeName?: string; shape?: () => Record<string, unknown> };
+  };
+
+  if (actualField._def?.typeName === 'ZodObject') {
+    const shape = actualField._def.shape?.();
+    if (shape) {
+      // Method 1: Check for the location field flag (for result schema)
+      if ('midscene_location_field_flag' in shape) {
+        return true;
+      }
+      // Method 2: Check if it's the input schema by checking for 'prompt' field
+      if ('prompt' in shape && shape.prompt) {
+        return true;
+      }
+    }
+  }
+  return false;
+}
+
+/**
+ * Get type name string from a Zod schema field
+ * @param field - Zod schema field
+ * @param locatorTypeDescription - Optional description for MidsceneLocation fields (used by core)
+ */
+export function getZodTypeName(
+  field: unknown,
+  locatorTypeDescription?: string,
+): string {
+  const actualField = unwrapZodField(field) as {
+    _def?: { typeName?: string; values?: unknown[]; options?: unknown[] };
+  };
+  const fieldTypeName = actualField._def?.typeName;
+
+  if (fieldTypeName === 'ZodString') return 'string';
+  if (fieldTypeName === 'ZodNumber') return 'number';
+  if (fieldTypeName === 'ZodBoolean') return 'boolean';
+  if (fieldTypeName === 'ZodArray') return 'array';
+  if (fieldTypeName === 'ZodObject') {
+    // Check if this is a Midscene locator field
+    if (isMidsceneLocatorField(actualField)) {
+      return locatorTypeDescription || 'object';
+    }
+    return 'object';
+  }
+  if (fieldTypeName === 'ZodEnum') {
+    const values =
+      (actualField._def?.values as unknown[] | undefined)
+        ?.map((option: unknown) => String(`'${option}'`))
+        .join(', ') ?? 'enum';
+    return `enum(${values})`;
+  }
+  // Handle ZodUnion by listing all option types
+  if (fieldTypeName === 'ZodUnion') {
+    const options = actualField._def?.options as unknown[] | undefined;
+    if (options && options.length > 0) {
+      const types = options.map((opt: unknown) =>
+        getZodTypeName(opt, locatorTypeDescription),
+      );
+      return types.join(' | ');
+    }
+    return 'union';
+  }
+
+  return 'unknown';
+}
+
+/**
+ * Get description from a Zod schema field
+ */
+export function getZodDescription(field: z.ZodTypeAny): string | null {
+  // Check for direct description on the original field (wrapper may have description)
+  if ('description' in field) {
+    return (field as { description?: string }).description || null;
+  }
+
+  const actualField = unwrapZodField(field) as {
+    description?: string;
+    _def?: { typeName?: string; shape?: () => Record<string, unknown> };
+  };
+
+  // Check for description on the unwrapped field
+  if ('description' in actualField) {
+    return actualField.description || null;
+  }
+
+  // Check for MidsceneLocation fields and add description
+  if (isMidsceneLocatorField(actualField)) {
+    return 'Location information for the target element';
+  }
+
+  return null;
+}