openuispec 0.2.3 → 0.2.4

package/mcp-server/index.ts

@@ -26,6 +26,8 @@ import { readFileSync as fsReadFileSync, existsSync, readdirSync } from "node:fs
 import { relative, resolve } from "node:path";
 import YAML from "yaml";
 import { takeScreenshot } from "./screenshot.js";
+import { takeAndroidScreenshot } from "./screenshot-android.js";
+import { takeIOSScreenshot } from "./screenshot-ios.js";
 
 // ── resolve project cwd ──────────────────────────────────────────────
 
@@ -149,6 +151,12 @@ VISUAL VERIFICATION:
 - openuispec_screenshot(route, viewport?, theme?) — screenshot the generated web app at a route.
   Starts the dev server automatically. Use after generation to visually verify UI matches the spec.
   Requires puppeteer (npm install -g puppeteer).
+- openuispec_screenshot_android(screen?, theme?, wait_for?) — screenshot the generated Android app.
+  Builds APK, installs on emulator, and captures via adb screencap.
+  Shows the real app with navigation, images, and themes. Requires a running emulator.
+- openuispec_screenshot_ios(screen?, device?, nav?, theme?, wait_for?) — screenshot the generated iOS app.
+  Builds with xcodebuild, installs on simulator, and captures via xcrun simctl.
+  Shows the real app with navigation, images, and themes. Requires Xcode.
 
 Skip these tools ONLY when the request is purely non-UI (API logic, database, infrastructure, etc.)
 or explicitly platform-specific polish that doesn't affect shared UI semantics.`,
@@ -696,9 +704,10 @@ server.registerTool(
       wait_for: z.number().optional().default(1000).describe("Milliseconds to wait after page load before screenshotting (default 1000)"),
       full_page: z.boolean().optional().default(false).describe("Capture the full scrollable page instead of just the viewport"),
       selector: z.string().optional().describe("CSS selector to screenshot a specific element instead of the full page"),
+      output_dir: z.string().optional().describe("Directory to save the screenshot PNG (relative to web app root). E.g. 'screenshots'. If omitted, only returns base64 in response."),
     },
   },
-  async ({ route, viewport, theme, wait_for, full_page, selector }) => {
+  async ({ route, viewport, theme, wait_for, full_page, selector, output_dir }) => {
     try {
       return await takeScreenshot(projectCwd, {
         route,
@@ -707,6 +716,7 @@ server.registerTool(
         wait_for,
         full_page,
         selector,
+        output_dir,
       });
     } catch (err) {
       return toolError(err);
@@ -714,6 +724,59 @@ server.registerTool(
   }
 );
 
+// ── tool: openuispec_screenshot_android ───────────────────────────────
+
+server.registerTool(
+  "openuispec_screenshot_android",
+  {
+    description: "Take a screenshot of an Android app on an emulator. Builds the APK, installs it, launches the app, and captures via adb screencap. Shows the real app with navigation, images, and themes. Requires a running Android emulator. Works with any Android project — use project_dir to point directly at a project, or uses OpenUISpec manifest discovery if available.",
+    inputSchema: {
+      screen: z.string().optional().describe("Screen name for metadata and filename (e.g. 'home_feed')."),
+      route: z.string().optional().describe("Deep link URI to navigate to a specific screen (e.g. 'myapp://profile/123'). If omitted, launches the main activity."),
+      nav: z.array(z.string()).optional().describe("UI navigation steps — tap elements by visible text in order (e.g. ['Profile', 'Settings']). Executed after the app launches and loads."),
+      theme: z.enum(["light", "dark"]).optional().describe("Force light or dark mode via system UI mode"),
+      wait_for: z.number().optional().default(3000).describe("Milliseconds to wait after launch for content to load (default 3000)"),
+      output_dir: z.string().optional().describe("Directory to save the screenshot PNG (relative to Android project root). E.g. 'screenshots'. If omitted, only returns base64 in response."),
+      project_dir: z.string().optional().describe("Direct path to the Android project root (containing gradlew). Skips OpenUISpec manifest lookup. Use this for standalone Android projects."),
+      module: z.string().optional().describe("App module name (e.g. 'app', 'mobile'). If omitted, auto-detects by scanning settings.gradle for the module with com.android.application plugin."),
+    },
+  },
+  async ({ screen, route, nav, theme, wait_for, output_dir, project_dir, module }) => {
+    try {
+      return await takeAndroidScreenshot(projectCwd, { screen, route, nav, theme, wait_for, output_dir, project_dir, module });
+    } catch (err) {
+      return toolError(err);
+    }
+  }
+);
+
+// ── tool: openuispec_screenshot_ios ───────────────────────────────────
+
+server.registerTool(
+  "openuispec_screenshot_ios",
+  {
+    description: "Take a screenshot of an iOS app on a Simulator. Builds with xcodebuild, installs on simulator, launches the app, and captures via xcrun simctl. Shows the real app with navigation, images, and themes. Requires Xcode. Works with any iOS project — use project_dir to point directly at a project, or uses OpenUISpec manifest discovery if available.",
+    inputSchema: {
+      screen: z.string().optional().describe("Screen name for metadata and filename (e.g. 'settings')."),
+      device: z.string().optional().describe("Simulator device name (e.g. 'iPhone 15', 'iPad Pro 11-inch (M4)'). If omitted, uses any booted iPhone or the first available one."),
+      nav: z.array(z.string()).optional().describe("UI navigation steps — tap elements by visible text in order (e.g. ['Profile', 'Settings']). Executed after the app launches and loads."),
+      theme: z.enum(["light", "dark"]).optional().describe("Force light or dark appearance on the simulator"),
+      wait_for: z.number().optional().default(3000).describe("Milliseconds to wait after launch for content to load (default 3000)"),
+      output_dir: z.string().optional().describe("Directory to save the screenshot PNG (relative to iOS project root). E.g. 'screenshots'. If omitted, only returns base64 in response."),
+      project_dir: z.string().optional().describe("Direct path to the iOS project root (containing .xcodeproj/.xcworkspace). Skips OpenUISpec manifest lookup. Use this for standalone iOS projects."),
+      scheme: z.string().optional().describe("Xcode scheme name to build. If omitted, auto-detects from xcshareddata/xcschemes or uses the project name."),
+      bundle_id: z.string().optional().describe("App bundle identifier (e.g. 'com.example.myapp'). If omitted, auto-detects from project.pbxproj."),
+    },
+  },
+  async ({ screen, device, nav, theme, wait_for, output_dir, project_dir, scheme, bundle_id }) => {
+    try {
+      return await takeIOSScreenshot(projectCwd, { screen, device, nav, theme, wait_for, output_dir, project_dir, scheme, bundle_id });
+    } catch (err) {
+      return toolError(err);
+    }
+  }
+);
+
 // ── start server ─────────────────────────────────────────────────────
 
 export async function startMcpServer() {

package/mcp-server/screenshot-android.ts

@@ -0,0 +1,462 @@
+/**
+ * Android screenshot tool — builds the app, installs on an emulator,
+ * and captures real screenshots via adb screencap.
+ *
+ * Gives pixel-perfect results with real navigation, images, themes,
+ * and all runtime behavior. Requires a running Android emulator.
+ */
+
+import { exec as execCb, execSync } from "node:child_process";
+import { promisify } from "node:util";
+import { existsSync, readFileSync, unlinkSync, mkdirSync, copyFileSync } from "node:fs";
+import { join, resolve } from "node:path";
+import {
+  type ScreenshotResult,
+  findPlatformAppDir,
+  buildScreenshotResponse,
+} from "./screenshot-shared.js";
+
+const exec = promisify(execCb);
+
+// ── types ───────────────────────────────────────────────────────────
+
+export interface AndroidScreenshotOptions {
+  screen?: string;
+  route?: string;
+  nav?: string[];
+  theme?: "light" | "dark";
+  wait_for?: number;
+  output_dir?: string;
+  project_dir?: string;
+  module?: string;
+}
+
+// ── constants ───────────────────────────────────────────────────────
+
+const ADB_SCREENSHOT_PATH = "/sdcard/openuispec_screenshot.png";
+
+// ── Android app directory discovery ─────────────────────────────────
+
+function isAndroidProject(dir: string): boolean {
+  return existsSync(join(dir, "gradlew")) ||
+    existsSync(join(dir, "app", "build.gradle.kts")) ||
+    existsSync(join(dir, "app", "build.gradle"));
+}
+
+export function findAndroidAppDir(projectCwd: string, directDir?: string): string {
+  return findPlatformAppDir(projectCwd, "android", isAndroidProject, directDir);
+}
+
+// ── app module auto-detection ────────────────────────────────────────
+
+export function detectAppModule(androidDir: string): string {
+  // Read settings.gradle.kts or settings.gradle to find included modules
+  for (const settingsFile of ["settings.gradle.kts", "settings.gradle"]) {
+    const settingsPath = join(androidDir, settingsFile);
+    if (!existsSync(settingsPath)) continue;
+
+    try {
+      const content = readFileSync(settingsPath, "utf-8");
+      // Match include(":app"), include(":module1", ":module2"), include ":app"
+      const modules: string[] = [];
+      const includeMatches = content.matchAll(/include\s*\(?([^)\n]+)\)?/g);
+      for (const m of includeMatches) {
+        const args = m[1];
+        const moduleNames = args.matchAll(/[":]+([^",:)\s]+)/g);
+        for (const mn of moduleNames) {
+          modules.push(mn[1]);
+        }
+      }
+
+      // For each module, check if its build.gradle.kts/.gradle has com.android.application
+      for (const mod of modules) {
+        const modDir = join(androidDir, mod);
+        for (const buildFile of ["build.gradle.kts", "build.gradle"]) {
+          const buildPath = join(modDir, buildFile);
+          if (!existsSync(buildPath)) continue;
+          try {
+            const buildContent = readFileSync(buildPath, "utf-8");
+            if (buildContent.includes("com.android.application")) {
+              return mod;
+            }
+          } catch { /* skip */ }
+        }
+      }
+    } catch { /* skip */ }
+  }
+
+  return "app"; // fallback
+}
+
+// ── app package / activity extraction ───────────────────────────────
+
+export interface AppInfo {
+  applicationId: string;
+  launchActivity: string;
+  moduleName: string;
+}
+
+function readBuildFile(androidDir: string, moduleName: string): string | null {
+  for (const filename of ["build.gradle.kts", "build.gradle"]) {
+    try { return readFileSync(join(androidDir, moduleName, filename), "utf-8"); } catch { /* skip */ }
+  }
+  return null;
+}
+
+export function extractAppInfo(androidDir: string, moduleOverride?: string): AppInfo {
+  const moduleName = moduleOverride ?? detectAppModule(androidDir);
+
+  // Try to get applicationId from build.gradle.kts or build.gradle
+  let applicationId = "";
+  const buildContent = readBuildFile(androidDir, moduleName);
+  if (buildContent) {
+    // Kotlin DSL: applicationId = "..."
+    const appIdMatch = buildContent.match(/applicationId\s*=\s*"([^"]+)"/);
+    if (appIdMatch) applicationId = appIdMatch[1];
+    if (!applicationId) {
+      // Groovy DSL: applicationId "..." or applicationId '...'
+      const groovyMatch = buildContent.match(/applicationId\s+['"]([^'"]+)['"]/);
+      if (groovyMatch) applicationId = groovyMatch[1];
+    }
+    if (!applicationId) {
+      const nsMatch = buildContent.match(/namespace\s*=?\s*['"]([^'"]+)['"]/);
+      if (nsMatch) applicationId = nsMatch[1];
+    }
+  }
+
+  // Get launch activity from AndroidManifest.xml
+  let launchActivity = ".MainActivity";
+  const manifestPath = join(androidDir, moduleName, "src", "main", "AndroidManifest.xml");
+  try {
+    const manifest = readFileSync(manifestPath, "utf-8");
+    // Find activity with MAIN/LAUNCHER intent filter
+    const activityMatch = manifest.match(
+      /<activity[^>]*android:name="([^"]+)"[^]*?action android:name="android\.intent\.action\.MAIN"/,
+    );
+    if (activityMatch) launchActivity = activityMatch[1];
+
+    // If applicationId still empty, try manifest package
+    if (!applicationId) {
+      const pkgMatch = manifest.match(/package="([^"]+)"/);
+      if (pkgMatch) applicationId = pkgMatch[1];
+    }
+  } catch { /* use defaults */ }
+
+  if (!applicationId) {
+    throw new Error(
+      `Could not determine applicationId from ${moduleName}/build.gradle.kts (or .gradle) or AndroidManifest.xml`,
+    );
+  }
+
+  // Resolve relative activity name
+  const fullActivity = launchActivity.startsWith(".")
+    ? `${applicationId}${launchActivity}`
+    : launchActivity;
+
+  return { applicationId, launchActivity: fullActivity, moduleName };
+}
+
+// ── adb helpers ─────────────────────────────────────────────────────
+
+export function findAdb(): string {
+  // Check ANDROID_HOME first
+  const androidHome = process.env.ANDROID_HOME || process.env.ANDROID_SDK_ROOT;
+  if (androidHome) {
+    const adbPath = join(androidHome, "platform-tools", "adb");
+    if (existsSync(adbPath)) return adbPath;
+  }
+  // Fall back to PATH
+  try {
+    execSync("which adb", { stdio: "pipe" });
+    return "adb";
+  } catch {
+    throw new Error(
+      "adb not found. Set ANDROID_HOME or add platform-tools to PATH.\n" +
+      "Install via Android Studio or: sdkmanager 'platform-tools'",
+    );
+  }
+}
+
+export async function getConnectedEmulator(adb: string): Promise<string> {
+  try {
+    const { stdout } = await exec(`${adb} devices`);
+    const lines = stdout.trim().split("\n").slice(1); // skip header
+    for (const line of lines) {
+      const [serial, state] = line.split("\t");
+      if (state === "device" && (serial.startsWith("emulator-") || serial.includes(":"))) {
+        return serial;
+      }
+    }
+    // Also accept physical devices if no emulator
+    for (const line of lines) {
+      const [serial, state] = line.split("\t");
+      if (state === "device") return serial;
+    }
+  } catch { /* fall through */ }
+
+  throw new Error(
+    "No connected Android device or emulator found.\n" +
+    "Start an emulator from Android Studio or run: emulator -avd <avd_name>",
+  );
+}
+
+export async function adbShell(adb: string, serial: string, cmd: string): Promise<string> {
+  const { stdout } = await exec(`${adb} -s ${serial} shell ${cmd}`, { timeout: 30_000 });
+  return stdout.trim();
+}
+
+export async function adbExec(adb: string, serial: string, args: string): Promise<string> {
+  const { stdout } = await exec(`${adb} -s ${serial} ${args}`, { timeout: 60_000 });
+  return stdout.trim();
+}
+
+// ── build APK ───────────────────────────────────────────────────────
+
+export async function buildApk(androidDir: string, moduleName: string): Promise<string> {
+  if (!existsSync(join(androidDir, "gradlew"))) {
+    throw new Error(`No gradlew found in ${androidDir}. Initialize the Gradle wrapper first.`);
+  }
+
+  try {
+    await exec(`./gradlew :${moduleName}:assembleDebug`, {
+      cwd: androidDir,
+      timeout: 300_000,
+      env: { ...process.env, JAVA_OPTS: "-Xmx2g" },
+    });
+  } catch (err: any) {
+    const output = ((err.stderr ?? "") + "\n" + (err.stdout ?? "")).slice(-500);
+    throw new Error(`APK build failed:\n${output}`);
+  }
+
+  // Find the built APK
+  const apkCandidates = [
+    join(androidDir, moduleName, "build", "outputs", "apk", "debug", `${moduleName}-debug.apk`),
+    join(androidDir, moduleName, "build", "outputs", "apk", "debug", `${moduleName}-debug-unsigned.apk`),
+    // Common fallback names
+    join(androidDir, moduleName, "build", "outputs", "apk", "debug", "app-debug.apk"),
+    join(androidDir, moduleName, "build", "outputs", "apk", "debug", "app-debug-unsigned.apk"),
+  ];
+  for (const apk of apkCandidates) {
+    if (existsSync(apk)) return apk;
+  }
+
+  throw new Error(`APK not found after build in ${moduleName}/build/outputs/. Check Gradle output.`);
+}
+
+// ── install & launch ────────────────────────────────────────────────
+
+export async function installAndLaunch(
+  adb: string,
+  serial: string,
+  apkPath: string,
+  appInfo: AppInfo,
+  route?: string,
+): Promise<void> {
+  // Install (replace existing)
+  await adbExec(adb, serial, `install -r "${apkPath}"`);
+
+  // Force-stop any existing instance
+  await adbShell(adb, serial, `am force-stop ${appInfo.applicationId}`);
+
+  if (route) {
+    // Navigate via deep link
+    await adbShell(adb, serial, `am start -a android.intent.action.VIEW -d "${route}" ${appInfo.applicationId}`);
+  } else {
+    // Launch the main activity
+    await adbShell(adb, serial, `am start -n ${appInfo.applicationId}/${appInfo.launchActivity}`);
+  }
+}
+
+// ── theme control ───────────────────────────────────────────────────
+
+export async function setTheme(adb: string, serial: string, theme: "light" | "dark"): Promise<void> {
+  const mode = theme === "dark" ? "yes" : "no";
+  await adbShell(adb, serial, `cmd uimode night ${mode}`);
+}
+
+// ── UI navigation via tap ───────────────────────────────────────────
+
+export async function tapByText(adb: string, serial: string, text: string): Promise<void> {
+  // Dump UI hierarchy
+  await adbShell(adb, serial, `uiautomator dump /sdcard/ui_dump.xml`);
+  const xml = await adbShell(adb, serial, `cat /sdcard/ui_dump.xml`);
+  await adbShell(adb, serial, `rm /sdcard/ui_dump.xml`);
+
+  const lowerText = text.toLowerCase();
+
+  // Parse each <node .../> extracting text, content-desc, and bounds
+  const nodeRegex = /<node\s[^>]+>/g;
+  interface UiNode { text: string; desc: string; cx: number; cy: number }
+  const nodes: UiNode[] = [];
+
+  let nodeMatch;
+  while ((nodeMatch = nodeRegex.exec(xml)) !== null) {
+    const attrs = nodeMatch[0];
+    const textMatch = attrs.match(/\btext="([^"]*)"/);
+    const descMatch = attrs.match(/\bcontent-desc="([^"]*)"/);
+    const boundsMatch = attrs.match(/\bbounds="\[(\d+),(\d+)\]\[(\d+),(\d+)\]"/);
+    if (!boundsMatch) continue;
+
+    nodes.push({
+      text: textMatch?.[1] ?? "",
+      desc: descMatch?.[1] ?? "",
+      cx: Math.round((parseInt(boundsMatch[1]) + parseInt(boundsMatch[3])) / 2),
+      cy: Math.round((parseInt(boundsMatch[2]) + parseInt(boundsMatch[4])) / 2),
+    });
+  }
+
+  // Exact match on text or content-desc
+  for (const node of nodes) {
+    if (node.text.toLowerCase() === lowerText || node.desc.toLowerCase() === lowerText) {
+      await adbShell(adb, serial, `input tap ${node.cx} ${node.cy}`);
+      return;
+    }
+  }
+
+  // Partial/contains match
+  for (const node of nodes) {
+    if (node.text.toLowerCase().includes(lowerText) || node.desc.toLowerCase().includes(lowerText)) {
+      await adbShell(adb, serial, `input tap ${node.cx} ${node.cy}`);
+      return;
+    }
+  }
+
+  throw new Error(`UI element with text "${text}" not found on screen.`);
+}
+
+export async function navigateByTaps(
+  adb: string,
+  serial: string,
+  steps: string[],
+): Promise<void> {
+  for (const step of steps) {
+    await tapByText(adb, serial, step);
+    // Wait for navigation animation
+    await new Promise(r => setTimeout(r, 800));
+  }
+}
+
+// ── screenshot capture ──────────────────────────────────────────────
+
+export async function captureScreenshot(
+  adb: string,
+  serial: string,
+  localPath: string,
+): Promise<void> {
+  await adbShell(adb, serial, `screencap -p ${ADB_SCREENSHOT_PATH}`);
+  await adbExec(adb, serial, `pull ${ADB_SCREENSHOT_PATH} "${localPath}"`);
+  await adbShell(adb, serial, `rm ${ADB_SCREENSHOT_PATH}`);
+}
+
+// ── wait for app ready ──────────────────────────────────────────────
+
+async function waitForAppReady(
+  adb: string,
+  serial: string,
+  applicationId: string,
+  waitMs: number,
+): Promise<void> {
+  // Wait for the activity to be in resumed state
+  const startTime = Date.now();
+  const timeout = Math.min(waitMs, 15_000);
+
+  while (Date.now() - startTime < timeout) {
+    try {
+      const output = await adbShell(adb, serial,
+        `dumpsys activity activities | grep -E "mResumedActivity|topResumedActivity"`,
+      );
+      if (output.includes(applicationId)) {
+        // App is in foreground, wait the remaining time for content to load
+        const elapsed = Date.now() - startTime;
+        const remaining = Math.max(waitMs - elapsed, 500);
+        await new Promise(r => setTimeout(r, remaining));
+        return;
+      }
+    } catch { /* activity not ready yet */ }
+    await new Promise(r => setTimeout(r, 500));
+  }
+
+  // Fallback: just wait the full duration
+  await new Promise(r => setTimeout(r, waitMs));
+}
+
+// ── main entry point ────────────────────────────────────────────────
+
+export async function takeAndroidScreenshot(
+  projectCwd: string,
+  options: AndroidScreenshotOptions,
+): Promise<ScreenshotResult> {
+  const {
+    screen,
+    route,
+    nav,
+    theme,
+    wait_for = 3000,
+    output_dir,
+    project_dir,
+    module,
+  } = options;
+
+  // 1. Find Android project
+  const androidDir = findAndroidAppDir(projectCwd, project_dir);
+  const appInfo = extractAppInfo(androidDir, module);
+
+  // 2. Find adb and emulator
+  const adb = findAdb();
+  const serial = await getConnectedEmulator(adb);
+
+  // 3. Build APK
+  const apkPath = await buildApk(androidDir, appInfo.moduleName);
+
+  // 4. Set theme if requested
+  if (theme) {
+    await setTheme(adb, serial, theme);
+  }
+
+  // 5. Install and launch
+  await installAndLaunch(adb, serial, apkPath, appInfo, route);
+
+  // 6. Wait for app to be ready and content to load
+  await waitForAppReady(adb, serial, appInfo.applicationId, wait_for);
+
+  // 7. Navigate via UI taps if specified
+  if (nav && nav.length > 0) {
+    await navigateByTaps(adb, serial, nav);
+  }
+
+  // 8. Capture screenshot
+  const screenLabel = screen ?? "main";
+  const themeLabel = theme ?? "default";
+  const filename = `${screenLabel}_${themeLabel}.png`;
+  const tmpPath = join(androidDir, ".openuispec-screenshot.png");
+  await captureScreenshot(adb, serial, tmpPath);
+
+  // 9. Save to output_dir if specified
+  let savedPath: string | undefined;
+  if (output_dir) {
+    const outDir = resolve(androidDir, output_dir);
+    mkdirSync(outDir, { recursive: true });
+    savedPath = join(outDir, filename);
+    copyFileSync(tmpPath, savedPath);
+  }
+
+  // 10. Read and return
+  try {
+    const data = readFileSync(tmpPath).toString("base64");
+    const snapshots = [{
+      screen: screenLabel,
+      path: savedPath ?? filename,
+      data,
+    }];
+
+    return buildScreenshotResponse(snapshots, (s) => ({
+      screen: s.screen,
+      path: savedPath ?? null,
+      emulator: serial,
+      theme: themeLabel,
+      applicationId: appInfo.applicationId,
+    }));
+  } finally {
+    try { unlinkSync(tmpPath); } catch { /* ignore */ }
+  }
+}