@sensaiorg/adapter-android 0.1.0

package/src/tools/ui-tree.ts

@@ -0,0 +1,258 @@
+/**
+ * UI Tree Tools - Inspect the Android view hierarchy.
+ *
+ * Provides three tools:
+ * - get_ui_tree: Full UI hierarchy as JSON
+ * - get_screen_text: Extract all visible text in reading order
+ * - get_element_details: Deep inspection of a specific element
+ */
+
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { z } from "zod";
+import type { ConnectionManager } from "../transport/connection-manager.js";
+import { parseUiTree, flattenTree, type UiNode } from "../util/ui-tree-parser.js";
+import { extractText, extractTextStrings, findByText } from "../util/text-extractor.js";
+
+/**
+ * Dump the UI hierarchy via ADB uiautomator (raw XML).
+ * Uses a unique filename per dump to prevent race conditions on concurrent requests.
+ */
+async function dumpUiTreeRaw(cm: ConnectionManager): Promise<string> {
+  const dumpPath = `/sdcard/sensai_dump_${Date.now()}.xml`;
+  const dumpOutput = await cm.adb.shell(`uiautomator dump ${dumpPath}`);
+  if (dumpOutput.includes("ERROR") || !dumpOutput.includes("dumped")) {
+    throw new Error(`uiautomator dump failed: ${dumpOutput.trim()}`);
+  }
+  const xml = await cm.adb.shell(`cat ${dumpPath}`);
+  // Clean up the dump file (fire and forget)
+  cm.adb.shell(`rm -f ${dumpPath}`).catch(() => {});
+  if (!xml.includes("<hierarchy")) {
+    throw new Error("UI dump produced invalid XML");
+  }
+  return xml;
+}
+
+/**
+ * Get the UI tree with caching support. Shared by all tools that need UI data.
+ * Exported so other tool modules can reuse without redundant dumps.
+ */
+export async function getCachedTree(
+  cm: ConnectionManager,
+  options: { maxDepth?: number; visibleOnly?: boolean; includeSystemUI?: boolean; forceRefresh?: boolean },
+): Promise<UiNode[]> {
+  // Phase 2: prefer agent for richer data
+  if (cm.agent.isConnected()) {
+    try {
+      const result = await cm.agent.call("getUiTree", options);
+      return result as UiNode[];
+    } catch {
+      // Fall through to ADB
+    }
+  }
+
+  // Check cache unless force refresh requested
+  if (!options.forceRefresh) {
+    const cached = cm.uiCache.get();
+    if (cached) {
+      // Re-parse with current options from cached XML
+      return parseUiTree(cached.xml, options);
+    }
+  }
+
+  // Phase 1: ADB uiautomator dump
+  const xml = await dumpUiTreeRaw(cm);
+  const tree = parseUiTree(xml, options);
+
+  // Cache the raw XML and default-options tree
+  cm.uiCache.set(tree, xml);
+
+  return tree;
+}
+
+export function registerUiTreeTools(server: McpServer, cm: ConnectionManager): void {
+  /**
+   * get_ui_tree - Returns the full UI hierarchy as structured JSON.
+   */
+  server.tool(
+    "get_ui_tree",
+    "Get the Android UI view hierarchy as structured JSON. Returns class names, resource IDs, text, bounds, and interactive state for every view element.",
+    {
+      maxDepth: z.number().optional().describe("Maximum depth to traverse (0 = unlimited)"),
+      visibleOnly: z.boolean().optional().describe("Only include visible nodes"),
+      includeSystemUI: z.boolean().optional().describe("Include status bar, nav bar, etc."),
+      interactableOnly: z.boolean().optional().describe("Only return nodes that are clickable, focusable, or have text content. Reduces noise significantly."),
+      format: z.enum(["full", "compact"]).optional().describe("Output format: 'full' (default) returns UiNode objects, 'compact' returns [text, resourceId, bounds, clickable] tuples for ~60% token savings"),
+    },
+    async (params) => {
+      try {
+        const tree = await getCachedTree(cm, {
+          maxDepth: params.maxDepth,
+          visibleOnly: params.visibleOnly,
+          includeSystemUI: params.includeSystemUI,
+        });
+
+        let flat = flattenTree(tree);
+        const totalNodes = flat.length;
+
+        // Filter to interactable nodes only
+        if (params.interactableOnly) {
+          flat = flat.filter((n) => n.clickable || n.focusable || !!n.text);
+        }
+
+        const summary = {
+          totalNodes,
+          returnedNodes: flat.length,
+          clickableNodes: flat.filter((n) => n.clickable).length,
+          textNodes: flat.filter((n) => n.text).length,
+          filtered: !!params.interactableOnly,
+          format: params.format ?? "full",
+        };
+
+        let treeData: unknown;
+        if (params.format === "compact") {
+          // Compact format: [text, resourceId, boundsStr, clickable] tuples
+          // Saves ~60% tokens vs full UiNode objects
+          treeData = flat.map((n) => [
+            n.text || n.contentDescription || "",
+            n.resourceId || "",
+            n.bounds ? `[${n.bounds.left},${n.bounds.top}][${n.bounds.right},${n.bounds.bottom}]` : "",
+            n.clickable,
+          ]);
+        } else {
+          // Full format: return tree if not filtered, flat array if filtered
+          treeData = params.interactableOnly ? flat : tree;
+        }
+
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: JSON.stringify({ summary, tree: treeData }),
+            },
+          ],
+        };
+      } catch (err) {
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: `Error getting UI tree: ${err instanceof Error ? err.message : String(err)}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  );
+
+  /**
+   * get_screen_text - Extract all visible text from the screen in reading order.
+   */
+  server.tool(
+    "get_screen_text",
+    "Extract all visible text from the current screen, ordered top-to-bottom left-to-right (reading order). Useful for understanding what the user sees.",
+    {},
+    async () => {
+      try {
+        const tree = await getCachedTree(cm, { visibleOnly: true, includeSystemUI: false });
+        const textEntries = extractText(tree);
+        const textStrings = extractTextStrings(tree);
+
+        // Deduplicate: trim whitespace, remove empty strings, remove duplicates
+        const deduped = [...new Set(
+          textStrings
+            .map((s) => s.trim())
+            .filter((s) => s.length > 0),
+        )];
+
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: JSON.stringify({
+                  screenText: deduped,
+                  detailed: textEntries,
+                }),
+            },
+          ],
+        };
+      } catch (err) {
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: `Error extracting screen text: ${err instanceof Error ? err.message : String(err)}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  );
+
+  /**
+   * get_element_details - Deep inspection of elements matching a search query.
+   */
+  server.tool(
+    "get_element_details",
+    "Find and inspect UI elements matching a text query. Returns full details including bounds, class, interactive state, and nearby elements.",
+    {
+      query: z.string().describe("Text to search for (case-insensitive substring match)"),
+      includeParent: z.boolean().optional().describe("Include parent context for matched elements"),
+    },
+    async (params) => {
+      try {
+        const tree = await getCachedTree(cm, { visibleOnly: true, includeSystemUI: false });
+        const matches = findByText(tree, params.query);
+
+        if (matches.length === 0) {
+          return {
+            content: [
+              {
+                type: "text" as const,
+                text: JSON.stringify({
+                  found: 0,
+                  message: `No elements found matching "${params.query}"`,
+                  hint: "Try using get_screen_text to see all visible text first.",
+                }),
+              },
+            ],
+          };
+        }
+
+        const details = matches.map((node) => ({
+          text: node.text,
+          contentDescription: node.contentDescription,
+          className: node.className,
+          resourceId: node.resourceId,
+          bounds: node.bounds,
+          clickable: node.clickable,
+          enabled: node.enabled,
+          focusable: node.focusable,
+          scrollable: node.scrollable,
+          centerX: node.bounds ? Math.round((node.bounds.left + node.bounds.right) / 2) : null,
+          centerY: node.bounds ? Math.round((node.bounds.top + node.bounds.bottom) / 2) : null,
+        }));
+
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: JSON.stringify({ found: matches.length, elements: details }),
+            },
+          ],
+        };
+      } catch (err) {
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: `Error inspecting elements: ${err instanceof Error ? err.message : String(err)}`,
+            },
+          ],
+          isError: true,
+        };
+      }
+    },
+  );
+}

package/src/transport/adb-client.test.ts

@@ -0,0 +1,228 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+// vi.hoisted runs before vi.mock hoisting, so the fn is available in mock factories
+const mockExecFileAsync = vi.hoisted(() => vi.fn());
+
+vi.mock("node:child_process", () => ({
+  execFile: vi.fn(),
+}));
+
+vi.mock("node:util", () => ({
+  promisify: () => mockExecFileAsync,
+}));
+
+import { AdbClient } from "./adb-client.js";
+
+describe("AdbClient", () => {
+  let client: AdbClient;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    client = new AdbClient("/usr/bin/adb", "emulator-5554");
+  });
+
+  describe("exec()", () => {
+    it("runs adb with correct device serial and args", async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "ok\n" });
+
+      await client.exec(["shell", "ls"]);
+
+      expect(mockExecFileAsync).toHaveBeenCalledWith(
+        "/usr/bin/adb",
+        ["-s", "emulator-5554", "shell", "ls"],
+        { timeout: 30_000, maxBuffer: 10 * 1024 * 1024 },
+      );
+    });
+
+    it("returns stdout from the command", async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "file1.txt\nfile2.txt\n" });
+
+      const result = await client.exec(["shell", "ls"]);
+
+      expect(result).toBe("file1.txt\nfile2.txt\n");
+    });
+
+    it("throws with descriptive error on failure", async () => {
+      mockExecFileAsync.mockRejectedValue(new Error("command not found"));
+
+      await expect(client.exec(["shell", "badcmd"])).rejects.toThrow(
+        "ADB command failed [adb -s emulator-5554 shell badcmd]: command not found",
+      );
+    });
+
+    it("sets connected=true on success", async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "ok" });
+
+      expect(client.isDeviceConnected).toBe(false);
+      await client.exec(["get-state"]);
+      expect(client.isDeviceConnected).toBe(true);
+    });
+
+    it('sets connected=false on "device not found" error', async () => {
+      // First succeed to set connected=true
+      mockExecFileAsync.mockResolvedValueOnce({ stdout: "ok" });
+      await client.exec(["get-state"]);
+      expect(client.isDeviceConnected).toBe(true);
+
+      // Then fail with device not found
+      mockExecFileAsync.mockRejectedValue(
+        new Error("error: device 'emulator-5554' not found"),
+      );
+      await expect(client.exec(["get-state"])).rejects.toThrow();
+      expect(client.isDeviceConnected).toBe(false);
+    });
+
+    it("respects custom timeout", async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "" });
+
+      await client.exec(["shell", "slow-cmd"], 5_000);
+
+      expect(mockExecFileAsync).toHaveBeenCalledWith(
+        "/usr/bin/adb",
+        ["-s", "emulator-5554", "shell", "slow-cmd"],
+        { timeout: 5_000, maxBuffer: 10 * 1024 * 1024 },
+      );
+    });
+  });
+
+  describe("shell()", () => {
+    it('delegates to exec with "shell" prefix', async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "result\n" });
+
+      const result = await client.shell("getprop ro.product.model");
+
+      expect(mockExecFileAsync).toHaveBeenCalledWith(
+        "/usr/bin/adb",
+        ["-s", "emulator-5554", "shell", "getprop ro.product.model"],
+        expect.objectContaining({ timeout: 30_000 }),
+      );
+      expect(result).toBe("result\n");
+    });
+  });
+
+  describe("isConnected()", () => {
+    it('returns true when device responds "device"', async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "device\n" });
+
+      const result = await client.isConnected();
+
+      expect(result).toBe(true);
+      expect(client.isDeviceConnected).toBe(true);
+    });
+
+    it("returns false on timeout/error", async () => {
+      mockExecFileAsync.mockRejectedValue(new Error("timeout"));
+
+      const result = await client.isConnected();
+
+      expect(result).toBe(false);
+      expect(client.isDeviceConnected).toBe(false);
+    });
+
+    it("returns false when device state is not 'device'", async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "offline\n" });
+
+      const result = await client.isConnected();
+
+      expect(result).toBe(false);
+    });
+
+    it("calls get-state with 5s timeout", async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "device\n" });
+
+      await client.isConnected();
+
+      expect(mockExecFileAsync).toHaveBeenCalledWith(
+        "/usr/bin/adb",
+        ["-s", "emulator-5554", "get-state"],
+        expect.objectContaining({ timeout: 5_000 }),
+      );
+    });
+  });
+
+  describe("getDeviceInfo()", () => {
+    it("returns structured device info", async () => {
+      mockExecFileAsync.mockImplementation((_path: string, args: string[]) => {
+        const cmd = args.join(" ");
+        if (cmd.includes("getprop ro.product.model"))
+          return Promise.resolve({ stdout: "Pixel 7\n" });
+        if (cmd.includes("getprop ro.build.version.release"))
+          return Promise.resolve({ stdout: "14\n" });
+        if (cmd.includes("getprop ro.build.version.sdk"))
+          return Promise.resolve({ stdout: "34\n" });
+        if (cmd.includes("wm size"))
+          return Promise.resolve({ stdout: "Physical size: 1080x2400\n" });
+        if (cmd.includes("wm density"))
+          return Promise.resolve({ stdout: "Physical density: 420\n" });
+        return Promise.resolve({ stdout: "" });
+      });
+
+      const info = await client.getDeviceInfo();
+
+      expect(info).toEqual({
+        serial: "emulator-5554",
+        model: "Pixel 7",
+        androidVersion: "14",
+        sdkVersion: "34",
+        screenSize: "1080x2400",
+        density: "420",
+      });
+    });
+
+    it('returns "unknown" for failed property reads', async () => {
+      mockExecFileAsync.mockRejectedValue(new Error("device offline"));
+
+      const info = await client.getDeviceInfo();
+
+      expect(info).toEqual({
+        serial: "emulator-5554",
+        model: "unknown",
+        androidVersion: "unknown",
+        sdkVersion: "unknown",
+        screenSize: "unknown",
+        density: "unknown",
+      });
+    });
+  });
+
+  describe("pull()", () => {
+    it("delegates to exec with correct args", async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "1 file pulled.\n" });
+
+      const result = await client.pull("/sdcard/screenshot.png", "/tmp/shot.png");
+
+      expect(mockExecFileAsync).toHaveBeenCalledWith(
+        "/usr/bin/adb",
+        ["-s", "emulator-5554", "pull", "/sdcard/screenshot.png", "/tmp/shot.png"],
+        expect.objectContaining({ timeout: 30_000 }),
+      );
+      expect(result).toBe("1 file pulled.\n");
+    });
+  });
+
+  describe("forward()", () => {
+    it("delegates to exec with tcp format", async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "" });
+
+      await client.forward(8081, 8081);
+
+      expect(mockExecFileAsync).toHaveBeenCalledWith(
+        "/usr/bin/adb",
+        ["-s", "emulator-5554", "forward", "tcp:8081", "tcp:8081"],
+        expect.objectContaining({ timeout: 30_000 }),
+      );
+    });
+  });
+
+  describe("isDeviceConnected getter", () => {
+    it("returns cached state (false initially)", () => {
+      expect(client.isDeviceConnected).toBe(false);
+    });
+
+    it("returns true after a successful exec", async () => {
+      mockExecFileAsync.mockResolvedValue({ stdout: "ok" });
+      await client.exec(["version"]);
+      expect(client.isDeviceConnected).toBe(true);
+    });
+  });
+});

package/src/transport/adb-client.ts

@@ -0,0 +1,139 @@
+/**
+ * ADB Client - Executes ADB commands against Android devices/emulators.
+ *
+ * Provides a typed interface around the `adb` CLI for shell commands,
+ * device info queries, and general command execution.
+ */
+
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+
+const execFileAsync = promisify(execFile);
+
+/** Maximum time (ms) to wait for an ADB command before killing it. */
+const DEFAULT_TIMEOUT_MS = 30_000;
+
+/** Information about the connected Android device. */
+export interface DeviceInfo {
+  serial: string;
+  model: string;
+  androidVersion: string;
+  sdkVersion: string;
+  screenSize: string;
+  density: string;
+}
+
+export class AdbClient {
+  private readonly adbPath: string;
+  private readonly device: string;
+  private connected = false;
+
+  constructor(adbPath: string, device: string) {
+    this.adbPath = adbPath;
+    this.device = device;
+  }
+
+  /**
+   * Execute an arbitrary ADB command (not a shell command).
+   * @param args - Arguments passed directly to `adb -s <device> ...args`.
+   * @param timeoutMs - Optional timeout override.
+   * @returns stdout of the command.
+   */
+  async exec(args: string[], timeoutMs = DEFAULT_TIMEOUT_MS): Promise<string> {
+    const fullArgs = ["-s", this.device, ...args];
+    try {
+      const { stdout } = await execFileAsync(this.adbPath, fullArgs, {
+        timeout: timeoutMs,
+        maxBuffer: 10 * 1024 * 1024, // 10 MB
+      });
+      this.connected = true;
+      return stdout;
+    } catch (error: unknown) {
+      const msg = error instanceof Error ? error.message : String(error);
+      if (msg.includes("device") && msg.includes("not found")) {
+        this.connected = false;
+      }
+      throw new Error(`ADB command failed [adb ${fullArgs.join(" ")}]: ${msg}`);
+    }
+  }
+
+  /**
+   * Execute a shell command on the device.
+   * @param command - Shell command string to run via `adb shell`.
+   * @param timeoutMs - Optional timeout override.
+   */
+  async shell(command: string, timeoutMs = DEFAULT_TIMEOUT_MS): Promise<string> {
+    return this.exec(["shell", command], timeoutMs);
+  }
+
+  /**
+   * Check whether the device is reachable.
+   */
+  async isConnected(): Promise<boolean> {
+    try {
+      const output = await this.exec(["get-state"], 5_000);
+      this.connected = output.trim() === "device";
+      return this.connected;
+    } catch {
+      this.connected = false;
+      return false;
+    }
+  }
+
+  /**
+   * Retrieve structured information about the connected device.
+   */
+  async getDeviceInfo(): Promise<DeviceInfo> {
+    const prop = async (key: string): Promise<string> => {
+      try {
+        return (await this.shell(`getprop ${key}`)).trim();
+      } catch {
+        return "unknown";
+      }
+    };
+
+    const screenSize = await (async () => {
+      try {
+        const raw = await this.shell("wm size");
+        const match = raw.match(/(\d+x\d+)/);
+        return match ? match[1] : "unknown";
+      } catch {
+        return "unknown";
+      }
+    })();
+
+    const density = await (async () => {
+      try {
+        const raw = await this.shell("wm density");
+        const match = raw.match(/(\d+)/);
+        return match ? match[1] : "unknown";
+      } catch {
+        return "unknown";
+      }
+    })();
+
+    return {
+      serial: this.device,
+      model: await prop("ro.product.model"),
+      androidVersion: await prop("ro.build.version.release"),
+      sdkVersion: await prop("ro.build.version.sdk"),
+      screenSize,
+      density,
+    };
+  }
+
+  /** Pull a file from the device to a local path. */
+  async pull(remotePath: string, localPath: string): Promise<string> {
+    return this.exec(["pull", remotePath, localPath]);
+  }
+
+  /** Forward a TCP port from host to device. */
+  async forward(hostPort: number, devicePort: number): Promise<void> {
+    await this.exec(["forward", `tcp:${hostPort}`, `tcp:${devicePort}`]);
+  }
+
+  /** Get the cached connection state (does not ping the device). */
+  get isDeviceConnected(): boolean {
+    return this.connected;
+  }
+}