@sensaiorg/adapter-android 0.1.0

package/src/tools/smart-actions.ts

@@ -0,0 +1,593 @@
+/**
+ * Smart Actions - Higher-level compound tools that reduce round-trips.
+ *
+ * Provides:
+ * - wait_for_text: Poll until text appears on screen (navigation waits)
+ * - scroll_to_text: Scroll until an element with matching text is visible
+ * - fill_form: Batch-fill multiple form fields in one call
+ * - tap_and_wait: Tap element then wait for expected text
+ * - assert_screen: Quick screen assertions (contains / notContains)
+ * - open_deep_link: Open app via deep link URL scheme
+ * - take_screenshot: Capture screen as base64 PNG image
+ */
+
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import type { ConnectionManager } from "../transport/connection-manager.js";
+import { flattenTree, type UiNode } from "../util/ui-tree-parser.js";
+import { extractTextStrings } from "../util/text-extractor.js";
+import { getCachedTree } from "./ui-tree.js";
+import { readFile, unlink } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+/** Sleep helper */
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Find an element by text/resourceId/contentDescription in the tree.
+ */
+function findElement(
+  flat: UiNode[],
+  selector: { text?: string; resourceId?: string; contentDescription?: string },
+): UiNode | null {
+  for (const node of flat) {
+    if (selector.text && node.text.toLowerCase().includes(selector.text.toLowerCase())) {
+      return node;
+    }
+    if (selector.resourceId && node.resourceId.includes(selector.resourceId)) {
+      return node;
+    }
+    if (
+      selector.contentDescription &&
+      node.contentDescription.toLowerCase().includes(selector.contentDescription.toLowerCase())
+    ) {
+      return node;
+    }
+  }
+  return null;
+}
+
+export function registerSmartActionTools(server: McpServer, cm: ConnectionManager): void {
+  /**
+   * wait_for_text - Poll until specific text appears on screen.
+   */
+  server.tool(
+    "wait_for_text",
+    "Wait until specific text appears on the screen. Polls the UI tree at intervals until the text is found or timeout is reached. Essential after navigation, screen transitions, or async operations.",
+    {
+      text: z.string().describe("Text to wait for (case-insensitive substring match)"),
+      timeoutMs: z.number().optional().describe("Maximum wait time in ms (default: 10000)"),
+      pollIntervalMs: z.number().optional().describe("Polling interval in ms (default: 1000)"),
+    },
+    async (params) => {
+      const timeout = params.timeoutMs ?? 10_000;
+      const interval = params.pollIntervalMs ?? 1_000;
+      const startTime = Date.now();
+      const searchLower = params.text.toLowerCase();
+      let attempts = 0;
+
+      while (Date.now() - startTime < timeout) {
+        attempts++;
+        // Force fresh dump each poll
+        cm.uiCache.invalidate();
+
+        try {
+          const tree = await getCachedTree(cm, { visibleOnly: true, includeSystemUI: false });
+          const texts = extractTextStrings(tree);
+          const found = texts.some((t) => t.toLowerCase().includes(searchLower));
+
+          if (found) {
+            return {
+              content: [
+                {
+                  type: "text" as const,
+                  text: JSON.stringify({
+                    success: true,
+                    found: true,
+                    text: params.text,
+                    elapsedMs: Date.now() - startTime,
+                    attempts,
+                  }),
+                },
+              ],
+            };
+          }
+        } catch {
+          // UI dump failed, retry
+        }
+
+        if (Date.now() - startTime + interval < timeout) {
+          await sleep(interval);
+        }
+      }
+
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: JSON.stringify({
+              success: false,
+              found: false,
+              text: params.text,
+              elapsedMs: Date.now() - startTime,
+              attempts,
+              hint: "Text did not appear within timeout. Check if the expected screen transition occurred.",
+            }),
+          },
+        ],
+      };
+    },
+  );
+
+  /**
+   * scroll_to_text - Scroll until element with text is visible.
+   */
+  server.tool(
+    "scroll_to_text",
+    "Scroll the screen until an element containing the specified text becomes visible. Useful for finding elements in long scrollable lists or forms.",
+    {
+      text: z.string().describe("Text to scroll to (case-insensitive substring match)"),
+      direction: z.enum(["up", "down"]).optional().describe("Scroll direction (default: down)"),
+      maxScrolls: z.number().optional().describe("Maximum scroll attempts (default: 10)"),
+    },
+    async (params) => {
+      const direction = params.direction ?? "down";
+      const maxScrolls = params.maxScrolls ?? 10;
+      const searchLower = params.text.toLowerCase();
+
+      // Screen dimensions for scroll coordinates (assume 1080x2340 default, adjust from first tree)
+      let screenWidth = 1080;
+      let screenHeight = 2340;
+
+      for (let i = 0; i < maxScrolls; i++) {
+        cm.uiCache.invalidate();
+
+        try {
+          const tree = await getCachedTree(cm, { visibleOnly: true, includeSystemUI: false });
+          const flat = flattenTree(tree);
+
+          // Detect screen size from root node bounds
+          if (flat.length > 0 && flat[0].bounds) {
+            screenWidth = flat[0].bounds.right;
+            screenHeight = flat[0].bounds.bottom;
+          }
+
+          // Check if target text is visible
+          const match = flat.find(
+            (n) =>
+              n.text.toLowerCase().includes(searchLower) ||
+              n.contentDescription.toLowerCase().includes(searchLower),
+          );
+
+          if (match && match.bounds) {
+            return {
+              content: [
+                {
+                  type: "text" as const,
+                  text: JSON.stringify({
+                    success: true,
+                    found: true,
+                    text: match.text || match.contentDescription,
+                    center: {
+                      x: Math.round((match.bounds.left + match.bounds.right) / 2),
+                      y: Math.round((match.bounds.top + match.bounds.bottom) / 2),
+                    },
+                    scrollsNeeded: i,
+                  }),
+                },
+              ],
+            };
+          }
+        } catch {
+          // Continue scrolling
+        }
+
+        // Perform scroll
+        const centerX = Math.round(screenWidth / 2);
+        const startY = direction === "down"
+          ? Math.round(screenHeight * 0.7)
+          : Math.round(screenHeight * 0.3);
+        const endY = direction === "down"
+          ? Math.round(screenHeight * 0.3)
+          : Math.round(screenHeight * 0.7);
+
+        await cm.adb.shell(`input swipe ${centerX} ${startY} ${centerX} ${endY} 300`);
+        // Brief pause for scroll animation to settle
+        await sleep(500);
+      }
+
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: JSON.stringify({
+              success: false,
+              found: false,
+              text: params.text,
+              scrollAttempts: maxScrolls,
+              hint: `Text "${params.text}" not found after ${maxScrolls} scroll attempts.`,
+            }),
+          },
+        ],
+      };
+    },
+  );
+
+  /**
+   * fill_form - Fill multiple form fields in one call.
+   */
+  server.tool(
+    "fill_form",
+    "Fill multiple form fields in a single call. Each field is identified by text, resourceId, or contentDescription. Taps the field, clears it, types the value. Much more efficient than separate tap+type calls for each field.",
+    {
+      fields: z
+        .array(
+          z.object({
+            selector: z.object({
+              text: z.string().optional().describe("Find field by visible text/label"),
+              resourceId: z.string().optional().describe("Find field by resource ID"),
+              contentDescription: z.string().optional().describe("Find field by content description"),
+            }).describe("How to find the field"),
+            value: z.string().describe("Text to enter in the field"),
+            clearFirst: z.boolean().optional().describe("Clear existing text before typing (default: true)"),
+          }),
+        )
+        .describe("Array of field selectors and values to fill"),
+    },
+    async (params) => {
+      const results: Array<{ field: string; success: boolean; error?: string }> = [];
+
+      for (const field of params.fields) {
+        const clearFirst = field.clearFirst ?? true;
+        const fieldId = field.selector.text || field.selector.resourceId || field.selector.contentDescription || "unknown";
+
+        try {
+          // Force fresh tree for each field (screen changes after typing)
+          cm.uiCache.invalidate();
+          const tree = await getCachedTree(cm, { visibleOnly: true, includeSystemUI: false });
+          const flat = flattenTree(tree);
+          const node = findElement(flat, field.selector);
+
+          if (!node || !node.bounds) {
+            results.push({ field: fieldId, success: false, error: "Element not found" });
+            continue;
+          }
+
+          const x = Math.round((node.bounds.left + node.bounds.right) / 2);
+          const y = Math.round((node.bounds.top + node.bounds.bottom) / 2);
+
+          // Tap the field
+          await cm.adb.shell(`input tap ${x} ${y}`);
+          await sleep(300); // Wait for field to focus
+
+          // Clear if needed
+          if (clearFirst) {
+            await cm.adb.shell("input keyevent 123"); // MOVE_END
+            await cm.adb.shell("input keyevent " + Array(50).fill("67").join(" "));
+          }
+
+          // Type the value
+          const escaped = field.value
+            .replace(/\\/g, "\\\\")
+            .replace(/ /g, "%s")
+            .replace(/'/g, "\\'")
+            .replace(/"/g, '\\"')
+            .replace(/&/g, "\\&")
+            .replace(/</g, "\\<")
+            .replace(/>/g, "\\>")
+            .replace(/\|/g, "\\|")
+            .replace(/;/g, "\\;")
+            .replace(/\(/g, "\\(")
+            .replace(/\)/g, "\\)")
+            .replace(/\$/g, "\\$")
+            .replace(/`/g, "\\`");
+
+          await cm.adb.shell(`input text ${escaped}`);
+
+          // Dismiss keyboard with ESCAPE to prevent focus issues with next field
+          await cm.adb.shell("input keyevent 111"); // ESCAPE
+
+          results.push({ field: fieldId, success: true });
+        } catch (err) {
+          results.push({
+            field: fieldId,
+            success: false,
+            error: err instanceof Error ? err.message : String(err),
+          });
+        }
+      }
+
+      const allSuccess = results.every((r) => r.success);
+
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: JSON.stringify({
+              success: allSuccess,
+              filledCount: results.filter((r) => r.success).length,
+              totalFields: params.fields.length,
+              results,
+            }),
+          },
+        ],
+      };
+    },
+  );
+
+  /**
+   * tap_and_wait - Tap an element then wait for expected text to appear.
+   */
+  server.tool(
+    "tap_and_wait",
+    "Tap an element and wait for expected text to appear on screen. Combines tap + wait_for_text in a single call to reduce round-trips.",
+    {
+      text: z.string().optional().describe("Visible text to tap"),
+      resourceId: z.string().optional().describe("Android resource ID to tap"),
+      contentDescription: z.string().optional().describe("Content description to tap"),
+      x: z.number().optional().describe("X coordinate to tap (fallback)"),
+      y: z.number().optional().describe("Y coordinate to tap (fallback)"),
+      waitForText: z.string().describe("Text to wait for after tap"),
+      timeoutMs: z.number().optional().describe("Max wait time in ms (default: 10000)"),
+    },
+    async (params) => {
+      try {
+        // Step 1: Find and tap the element
+        cm.uiCache.invalidate();
+        let tapX: number | undefined;
+        let tapY: number | undefined;
+        let matchedBy = "";
+
+        if (params.text || params.resourceId || params.contentDescription) {
+          const tree = await getCachedTree(cm, { visibleOnly: true, includeSystemUI: false });
+          const flat = flattenTree(tree);
+          const match = findElement(flat, {
+            text: params.text,
+            resourceId: params.resourceId,
+            contentDescription: params.contentDescription,
+          });
+
+          if (match?.bounds) {
+            tapX = Math.round((match.bounds.left + match.bounds.right) / 2);
+            tapY = Math.round((match.bounds.top + match.bounds.bottom) / 2);
+            matchedBy = params.text
+              ? `text="${match.text}"`
+              : params.resourceId
+                ? `resourceId="${match.resourceId}"`
+                : `contentDescription="${match.contentDescription}"`;
+          }
+        }
+
+        if (tapX === undefined || tapY === undefined) {
+          if (params.x !== undefined && params.y !== undefined) {
+            tapX = params.x;
+            tapY = params.y;
+            matchedBy = `coordinates (${tapX}, ${tapY})`;
+          } else {
+            return {
+              content: [
+                {
+                  type: "text" as const,
+                  text: JSON.stringify({
+                    success: false,
+                    error: "Element not found",
+                    hint: "Use get_screen_text to see available elements.",
+                  }),
+                },
+              ],
+            };
+          }
+        }
+
+        cm.uiCache.invalidate();
+        await cm.adb.shell(`input tap ${tapX} ${tapY}`);
+
+        // Step 2: Wait for text to appear
+        const timeout = params.timeoutMs ?? 10_000;
+        const interval = 500;
+        const startTime = Date.now();
+        const searchLower = params.waitForText.toLowerCase();
+
+        while (Date.now() - startTime < timeout) {
+          await sleep(interval);
+          cm.uiCache.invalidate();
+
+          try {
+            const tree = await getCachedTree(cm, { visibleOnly: true, includeSystemUI: false });
+            const texts = extractTextStrings(tree);
+            const found = texts.some((t) => t.toLowerCase().includes(searchLower));
+
+            if (found) {
+              return {
+                content: [
+                  {
+                    type: "text" as const,
+                    text: JSON.stringify({
+                      success: true,
+                      tappedBy: matchedBy,
+                      tapped: { x: tapX, y: tapY },
+                      waitedFor: params.waitForText,
+                      elapsedMs: Date.now() - startTime,
+                    }),
+                  },
+                ],
+              };
+            }
+          } catch {
+            // UI dump failed, retry
+          }
+        }
+
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: JSON.stringify({
+                success: false,
+                tappedBy: matchedBy,
+                tapped: { x: tapX, y: tapY },
+                waitedFor: params.waitForText,
+                timedOut: true,
+                elapsedMs: Date.now() - startTime,
+                hint: "Tap succeeded but expected text did not appear within timeout.",
+              }),
+            },
+          ],
+        };
+      } catch (err) {
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: `tap_and_wait failed: ${err instanceof Error ? err.message : String(err)}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  );
+
+  /**
+   * assert_screen - Quick screen assertions without returning the full tree.
+   */
+  server.tool(
+    "assert_screen",
+    "Quickly verify screen contains (or doesn't contain) expected text. Returns pass/fail without the full UI tree, saving tokens.",
+    {
+      contains: z.array(z.string()).optional().describe("Text strings that MUST be on screen"),
+      notContains: z.array(z.string()).optional().describe("Text strings that must NOT be on screen"),
+    },
+    async (params) => {
+      try {
+        cm.uiCache.invalidate();
+        const tree = await getCachedTree(cm, { visibleOnly: true, includeSystemUI: false });
+        const allText = extractTextStrings(tree).map((t) => t.toLowerCase());
+
+        const missing: string[] = [];
+        const unexpected: string[] = [];
+
+        for (const expected of params.contains ?? []) {
+          if (!allText.some((t) => t.includes(expected.toLowerCase()))) {
+            missing.push(expected);
+          }
+        }
+
+        for (const banned of params.notContains ?? []) {
+          if (allText.some((t) => t.includes(banned.toLowerCase()))) {
+            unexpected.push(banned);
+          }
+        }
+
+        const pass = missing.length === 0 && unexpected.length === 0;
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: JSON.stringify({
+                pass,
+                ...(missing.length > 0 ? { missing } : {}),
+                ...(unexpected.length > 0 ? { unexpected } : {}),
+              }),
+            },
+          ],
+        };
+      } catch (err) {
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: `assert_screen failed: ${err instanceof Error ? err.message : String(err)}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  );
+
+  /**
+   * open_deep_link - Open app via deep link URL scheme.
+   */
+  server.tool(
+    "open_deep_link",
+    "Open the app via a deep link URL scheme. Navigates directly to a specific screen without tapping through the UI.",
+    {
+      url: z.string().describe("Deep link URL (e.g., myapp://screen/detail?id=123 or https://example.com/path)"),
+    },
+    async (params) => {
+      try {
+        cm.uiCache.invalidate();
+        await cm.adb.shell(`am start -a android.intent.action.VIEW -d "${params.url}"`);
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: JSON.stringify({ success: true, url: params.url }),
+            },
+          ],
+        };
+      } catch (err) {
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: `open_deep_link failed: ${err instanceof Error ? err.message : String(err)}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  );
+
+  /**
+   * take_screenshot - Capture the screen as a base64 PNG image.
+   */
+  server.tool(
+    "take_screenshot",
+    "Capture a screenshot of the current screen and return it as a base64-encoded PNG image. Useful for visual verification of the app state.",
+    {},
+    async () => {
+      try {
+        const localPath = join(tmpdir(), `emudebug-screenshot-${Date.now()}.png`);
+
+        // Capture screenshot on device and pull to local
+        await cm.adb.shell("screencap -p /sdcard/emudebug_screenshot.png");
+        await cm.adb.exec(["pull", "/sdcard/emudebug_screenshot.png", localPath]);
+        // Clean up device file (fire and forget)
+        cm.adb.shell("rm -f /sdcard/emudebug_screenshot.png").catch(() => {});
+
+        // Read the local file as base64
+        const imageBuffer = await readFile(localPath);
+        const base64 = imageBuffer.toString("base64");
+
+        // Clean up local file
+        await unlink(localPath).catch(() => {});
+
+        return {
+          content: [
+            {
+              type: "image" as const,
+              data: base64,
+              mimeType: "image/png",
+            },
+          ],
+        };
+      } catch (err) {
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: `Error taking screenshot: ${err instanceof Error ? err.message : String(err)}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  );
+}