devlens-mcp 0.3.0

package/src/snapshot/ref-registry.ts

@@ -0,0 +1,230 @@
+import type { SnapshotNode, Bounds } from "../platform/device.js";
+
+/**
+ * RefRegistry — the Playwright-style ref assignment system.
+ *
+ * Assigns monotonically increasing refs (e1, e2, ...) to every interactive
+ * or meaningful node in the accessibility tree. Maintains a map from ref
+ * to node info (bounds, center coords) so interaction tools can resolve
+ * "tap ref=e5" to actual screen coordinates.
+ */
+
+export interface RefEntry {
+  ref: string;
+  node: SnapshotNode;
+  centerX: number;
+  centerY: number;
+  bounds: Bounds;
+}
+
+export interface ValidationIssue {
+  type: "zero-size" | "invisible" | "empty-container";
+  nodeType: string;
+  text?: string;
+  label?: string;
+  resourceId?: string;
+  bounds: Bounds;
+  detail: string;
+}
+
+export interface ValidationReport {
+  issues: ValidationIssue[];
+  totalNodes: number;
+  visibleNodes: number;
+  zeroSizeNodes: number;
+  interactiveNodes: number;
+}
+
+export class RefRegistry {
+  private refMap: Map<string, RefEntry> = new Map();
+  private counter: number = 0;
+
+  /** Clear all refs and reset counter (call before each new snapshot) */
+  reset(): void {
+    this.refMap.clear();
+    this.counter = 0;
+  }
+
+  /**
+   * Walk the snapshot tree and assign refs to all interactive or content-bearing nodes.
+   * Returns a new tree with refs attached (as annotations in the formatter output).
+   */
+  assignRefs(root: SnapshotNode): Map<SnapshotNode, string> {
+    this.reset();
+    const nodeToRef = new Map<SnapshotNode, string>();
+    this.walkAndAssign(root, nodeToRef);
+    return nodeToRef;
+  }
+
+  private walkAndAssign(
+    node: SnapshotNode,
+    nodeToRef: Map<SnapshotNode, string>
+  ): void {
+    if (this.shouldAssignRef(node)) {
+      const ref = `e${++this.counter}`;
+      const centerX = node.bounds.x + node.bounds.width / 2;
+      const centerY = node.bounds.y + node.bounds.height / 2;
+
+      this.refMap.set(ref, {
+        ref,
+        node,
+        centerX,
+        centerY,
+        bounds: node.bounds,
+      });
+      nodeToRef.set(node, ref);
+    }
+
+    for (const child of node.children) {
+      this.walkAndAssign(child, nodeToRef);
+    }
+  }
+
+  /** Determine if a node should get a ref */
+  private shouldAssignRef(node: SnapshotNode): boolean {
+    // Skip invisible/zero-size elements
+    if (node.bounds.width === 0 || node.bounds.height === 0) return false;
+
+    // Always assign refs to interactive elements
+    if (node.interactive) return true;
+
+    // Assign refs to elements with meaningful text content
+    if (node.text && node.text.trim().length > 0) return true;
+
+    // Assign refs to elements with accessibility labels
+    if (node.label && node.label.trim().length > 0) return true;
+
+    // Skip purely structural containers
+    return false;
+  }
+
+  /** Resolve a ref string to its entry (coordinates + node info) */
+  resolve(ref: string): RefEntry | undefined {
+    return this.refMap.get(ref);
+  }
+
+  /** Get all current ref entries */
+  getAllRefs(): RefEntry[] {
+    return Array.from(this.refMap.values());
+  }
+
+  /** Get the total number of assigned refs */
+  get size(): number {
+    return this.refMap.size;
+  }
+
+  /**
+   * Validate the snapshot tree for visibility issues.
+   * Flags zero-size elements that should be visible, explicitly invisible elements,
+   * and empty containers that might indicate missing content.
+   */
+  validateTree(root: SnapshotNode): ValidationReport {
+    const issues: ValidationIssue[] = [];
+    let totalNodes = 0;
+    let visibleNodes = 0;
+    let interactiveNodes = 0;
+    let zeroSizeNodes = 0;
+
+    const walk = (node: SnapshotNode): void => {
+      totalNodes++;
+
+      const isZeroSize =
+        node.bounds.width === 0 || node.bounds.height === 0;
+
+      if (isZeroSize) {
+        zeroSizeNodes++;
+        // Only flag if the node has content that SHOULD be visible
+        if (node.text || node.label || node.interactive) {
+          issues.push({
+            type: "zero-size",
+            nodeType: node.type,
+            text: node.text,
+            label: node.label,
+            resourceId: node.resourceId,
+            bounds: node.bounds,
+            detail:
+              `${node.type}${node.text ? ` "${node.text}"` : ""}${node.label ? ` label="${node.label}"` : ""}` +
+              ` has zero size (${node.bounds.width}x${node.bounds.height}) — renders as invisible`,
+          });
+        }
+      } else {
+        visibleNodes++;
+      }
+
+      if (node.visible === false) {
+        issues.push({
+          type: "invisible",
+          nodeType: node.type,
+          text: node.text,
+          label: node.label,
+          resourceId: node.resourceId,
+          bounds: node.bounds,
+          detail:
+            `${node.type}${node.text ? ` "${node.text}"` : ""} has visible=false`,
+        });
+      }
+
+      if (node.interactive) interactiveNodes++;
+
+      // Empty container with a resource ID — likely an intentional component
+      // that should have content (e.g., missing icon, empty image)
+      if (
+        node.children.length === 0 &&
+        node.resourceId &&
+        !node.text &&
+        !node.label &&
+        !isZeroSize
+      ) {
+        issues.push({
+          type: "empty-container",
+          nodeType: node.type,
+          resourceId: node.resourceId,
+          bounds: node.bounds,
+          detail:
+            `${node.type} id="${node.resourceId}" is an empty container (no children, no text) — may be a missing icon or placeholder`,
+        });
+      }
+
+      for (const child of node.children) {
+        walk(child);
+      }
+    };
+
+    walk(root);
+    return { issues, totalNodes, visibleNodes, zeroSizeNodes, interactiveNodes };
+  }
+
+  /**
+   * Search the full snapshot tree (including zero-size nodes) for text/label matches.
+   * Used by find_element to detect invisible elements that the refMap skips.
+   */
+  findInTree(
+    root: SnapshotNode,
+    query: { text?: string; label?: string }
+  ): SnapshotNode[] {
+    const matches: SnapshotNode[] = [];
+    const walk = (node: SnapshotNode): void => {
+      const isZeroSize =
+        node.bounds.width === 0 || node.bounds.height === 0;
+      if (isZeroSize) {
+        let matched = false;
+        if (query.text && node.text) {
+          matched = node.text
+            .toLowerCase()
+            .includes(query.text.toLowerCase());
+        }
+        if (query.label && node.label) {
+          matched =
+            matched ||
+            node.label.toLowerCase().includes(query.label.toLowerCase());
+        }
+        if (matched) matches.push(node);
+      }
+      for (const child of node.children) {
+        walk(child);
+      }
+    };
+    walk(root);
+    return matches;
+  }
+}

package/src/snapshot/snapshot-differ.ts

@@ -0,0 +1,220 @@
+import type { SnapshotNode } from "../platform/device.js";
+
+/**
+ * Incremental snapshot differ — compares two snapshot trees and returns
+ * only the changed portions. Saves 60-80% tokens in iterative workflows.
+ */
+
+export interface SnapshotDiff {
+  /** Nodes that were added since the last snapshot */
+  added: SnapshotNode[];
+  /** Nodes that were removed since the last snapshot */
+  removed: SnapshotNode[];
+  /** Nodes whose properties changed (text, value, focus, etc.) */
+  changed: Array<{
+    node: SnapshotNode;
+    changes: string[];
+  }>;
+  /** Elements that are the same content but moved to a different position */
+  moved: Array<{
+    node: SnapshotNode;
+    from: { x: number; y: number };
+    to: { x: number; y: number };
+    deltaX: number;
+    deltaY: number;
+  }>;
+  /** Elements that kept position but changed size */
+  resized: Array<{
+    node: SnapshotNode;
+    from: { width: number; height: number };
+    to: { width: number; height: number };
+  }>;
+  /** Whether the tree structure changed significantly */
+  structuralChange: boolean;
+}
+
+/**
+ * Content-based signature — used for element identity.
+ * Intentionally excludes position/size so moved elements are matched
+ * to their previous version rather than treated as add+remove.
+ */
+function contentSignature(node: SnapshotNode): string {
+  return [node.type, node.resourceId || "", node.text || "", node.label || ""].join("|");
+}
+
+/**
+ * Compute a diff between two snapshot trees.
+ *
+ * Identity matching: uses content signature (type + resourceId + text + label).
+ * Elements with matching content but different position are reported as "moved".
+ * Elements with matching content+position but different size are reported as "resized".
+ * Elements present in current but not previous are "added".
+ * Elements present in previous but not current are "removed".
+ */
+export function diffSnapshots(
+  previous: SnapshotNode,
+  current: SnapshotNode
+): SnapshotDiff {
+  const prevNodes = flattenTree(previous);
+  const currNodes = flattenTree(current);
+
+  const prevByContent = new Map<string, SnapshotNode>();
+  for (const node of prevNodes) {
+    prevByContent.set(contentSignature(node), node);
+  }
+
+  const currByContent = new Map<string, SnapshotNode>();
+  for (const node of currNodes) {
+    currByContent.set(contentSignature(node), node);
+  }
+
+  const added: SnapshotNode[] = [];
+  const removed: SnapshotNode[] = [];
+  const changed: Array<{ node: SnapshotNode; changes: string[] }> = [];
+  const moved: SnapshotDiff["moved"] = [];
+  const resized: SnapshotDiff["resized"] = [];
+
+  // Pass 1: find added, moved, resized, changed nodes
+  for (const [sig, currNode] of currByContent) {
+    const prevNode = prevByContent.get(sig);
+
+    if (!prevNode) {
+      added.push(currNode);
+      continue;
+    }
+
+    const posChanged =
+      prevNode.bounds.x !== currNode.bounds.x ||
+      prevNode.bounds.y !== currNode.bounds.y;
+
+    const sizeChanged =
+      prevNode.bounds.width !== currNode.bounds.width ||
+      prevNode.bounds.height !== currNode.bounds.height;
+
+    if (posChanged) {
+      moved.push({
+        node: currNode,
+        from: { x: prevNode.bounds.x, y: prevNode.bounds.y },
+        to: { x: currNode.bounds.x, y: currNode.bounds.y },
+        deltaX: currNode.bounds.x - prevNode.bounds.x,
+        deltaY: currNode.bounds.y - prevNode.bounds.y,
+      });
+    }
+
+    if (sizeChanged) {
+      resized.push({
+        node: currNode,
+        from: { width: prevNode.bounds.width, height: prevNode.bounds.height },
+        to: { width: currNode.bounds.width, height: currNode.bounds.height },
+      });
+    }
+
+    const changes = detectChanges(prevNode, currNode);
+    if (changes.length > 0) {
+      changed.push({ node: currNode, changes });
+    }
+  }
+
+  // Pass 2: find removed nodes
+  for (const [sig, node] of prevByContent) {
+    if (!currByContent.has(sig)) {
+      removed.push(node);
+    }
+  }
+
+  const structuralChange =
+    added.length > prevNodes.length * 0.3 ||
+    removed.length > prevNodes.length * 0.3;
+
+  return { added, removed, changed, moved, resized, structuralChange };
+}
+
+/** Detect property-level changes between two nodes with the same identity */
+function detectChanges(prev: SnapshotNode, curr: SnapshotNode): string[] {
+  const changes: string[] = [];
+
+  if (prev.text !== curr.text) {
+    changes.push(`text: "${prev.text}" → "${curr.text}"`);
+  }
+  if (prev.value !== curr.value) {
+    changes.push(`value: "${prev.value}" → "${curr.value}"`);
+  }
+  if (prev.focused !== curr.focused) {
+    changes.push(`focused: ${prev.focused} → ${curr.focused}`);
+  }
+  if (prev.enabled !== curr.enabled) {
+    changes.push(`enabled: ${prev.enabled} → ${curr.enabled}`);
+  }
+  if (prev.label !== curr.label) {
+    changes.push(`label: "${prev.label}" → "${curr.label}"`);
+  }
+
+  return changes;
+}
+
+/** Flatten a tree into a list of all nodes */
+function flattenTree(node: SnapshotNode): SnapshotNode[] {
+  const result: SnapshotNode[] = [node];
+  for (const child of node.children) {
+    result.push(...flattenTree(child));
+  }
+  return result;
+}
+
+/** Format a diff for LLM consumption */
+export function formatDiff(diff: SnapshotDiff): string {
+  if (diff.structuralChange) {
+    return "[Major structural change — use full snapshot]";
+  }
+
+  const lines: string[] = [];
+
+  if (diff.added.length > 0) {
+    lines.push("ADDED:");
+    for (const node of diff.added) {
+      lines.push(`  + ${node.type} ${node.text ? `"${node.text}"` : ""}`);
+    }
+  }
+
+  if (diff.removed.length > 0) {
+    lines.push("REMOVED:");
+    for (const node of diff.removed) {
+      lines.push(`  - ${node.type} ${node.text ? `"${node.text}"` : ""}`);
+    }
+  }
+
+  if (diff.moved.length > 0) {
+    lines.push("MOVED:");
+    for (const m of diff.moved) {
+      const label = m.node.text ? `"${m.node.text}"` : m.node.type;
+      const dx = m.deltaX > 0 ? `+${m.deltaX}` : `${m.deltaX}`;
+      const dy = m.deltaY > 0 ? `+${m.deltaY}` : `${m.deltaY}`;
+      lines.push(`  -> ${label} moved by (${dx}, ${dy})px`);
+    }
+  }
+
+  if (diff.resized.length > 0) {
+    lines.push("RESIZED:");
+    for (const r of diff.resized) {
+      const label = r.node.text ? `"${r.node.text}"` : r.node.type;
+      lines.push(
+        `  <> ${label}: ${r.from.width}x${r.from.height} → ${r.to.width}x${r.to.height}`
+      );
+    }
+  }
+
+  if (diff.changed.length > 0) {
+    lines.push("CHANGED:");
+    for (const { node, changes } of diff.changed) {
+      lines.push(
+        `  ~ ${node.type} ${node.text ? `"${node.text}"` : ""}: ${changes.join(", ")}`
+      );
+    }
+  }
+
+  if (lines.length === 0) {
+    return "[No changes detected]";
+  }
+
+  return lines.join("\n");
+}

package/src/tools/app-tools.ts

@@ -0,0 +1,111 @@
+import { z } from "zod";
+import type { DeviceManager } from "../platform/device-manager.js";
+
+export const appToolSchemas = {
+  devlens_launch_app: {
+    description:
+      "Launch an app on the device by its bundle ID (iOS) or package name (Android). The app must already be installed.",
+    parameters: z.object({
+      appId: z
+        .string()
+        .describe(
+          "Bundle ID (iOS, e.g., com.example.app) or package name (Android, e.g., com.example.app)"
+        ),
+    }),
+  },
+  devlens_terminate_app: {
+    description: "Force stop a running app on the device.",
+    parameters: z.object({
+      appId: z.string().describe("Bundle ID or package name of the app to stop"),
+    }),
+  },
+  devlens_install_app: {
+    description:
+      "Install an app from a local file path. Supports .apk (Android), .app/.ipa (iOS).",
+    parameters: z.object({
+      path: z
+        .string()
+        .describe("Local file path to the app binary (.apk, .app, or .ipa)"),
+    }),
+  },
+  devlens_list_apps: {
+    description:
+      "List all third-party apps installed on the device. Returns app IDs and names.",
+    parameters: z.object({}),
+  },
+};
+
+export function createAppToolHandlers(deviceManager: DeviceManager) {
+  return {
+    devlens_launch_app: async (params: { appId: string }) => {
+      const device = await deviceManager.getDevice();
+      await device.launchApp(params.appId);
+
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: `Launched app: ${params.appId}`,
+          },
+        ],
+      };
+    },
+
+    devlens_terminate_app: async (params: { appId: string }) => {
+      const device = await deviceManager.getDevice();
+      await device.terminateApp(params.appId);
+
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: `Terminated app: ${params.appId}`,
+          },
+        ],
+      };
+    },
+
+    devlens_install_app: async (params: { path: string }) => {
+      const device = await deviceManager.getDevice();
+      await device.installApp(params.path);
+
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: `Installed app from: ${params.path}`,
+          },
+        ],
+      };
+    },
+
+    devlens_list_apps: async () => {
+      const device = await deviceManager.getDevice();
+      const apps = await device.listApps();
+
+      if (apps.length === 0) {
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: "No third-party apps found on the device.",
+            },
+          ],
+        };
+      }
+
+      const lines = apps.map(
+        (app) => `${app.name} — ${app.appId}${app.isRunning ? " (running)" : ""}`
+      );
+
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: `Installed apps (${apps.length}):\n\n${lines.join("\n")}`,
+          },
+        ],
+      };
+    },
+  };
+}

package/src/tools/device-tools.ts

@@ -0,0 +1,96 @@
+import { z } from "zod";
+import type { DeviceManager } from "../platform/device-manager.js";
+
+export const deviceToolSchemas = {
+  devlens_list_devices: {
+    description:
+      "List all running iOS Simulators and Android Emulators. Returns device IDs, names, platforms, and OS versions.",
+    parameters: z.object({}),
+  },
+  devlens_device_info: {
+    description:
+      "Get detailed information about a specific device: screen size, OS version, orientation. If no deviceId is provided, uses the first available device.",
+    parameters: z.object({
+      deviceId: z
+        .string()
+        .optional()
+        .describe("Device ID from devlens_list_devices. Optional."),
+    }),
+  },
+  devlens_set_orientation: {
+    description: "Set the device orientation to portrait or landscape.",
+    parameters: z.object({
+      orientation: z
+        .enum(["portrait", "landscape"])
+        .describe("Target orientation"),
+    }),
+  },
+};
+
+export function createDeviceToolHandlers(deviceManager: DeviceManager) {
+  return {
+    devlens_list_devices: async () => {
+      const devices = await deviceManager.discoverDevices();
+      if (devices.length === 0) {
+        return {
+          content: [
+            {
+              type: "text" as const,
+              text: "No running simulators or emulators found.\n\nTo start a device:\n- Android: Open Android Studio → Device Manager → Start an emulator\n- iOS: Open Xcode → Window → Devices and Simulators → Boot a simulator",
+            },
+          ],
+        };
+      }
+
+      const lines = devices.map(
+        (d) =>
+          `${d.isBooted ? "●" : "○"} ${d.name} (${d.platform}, ${d.osVersion}) — id: ${d.id}`
+      );
+
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: `Found ${devices.length} device(s):\n\n${lines.join("\n")}`,
+          },
+        ],
+      };
+    },
+
+    devlens_device_info: async (params: { deviceId?: string }) => {
+      const device = await deviceManager.getDevice(params.deviceId);
+      const info = await device.getInfo();
+
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: [
+              `Device: ${info.name}`,
+              `Platform: ${info.platform}`,
+              `OS Version: ${info.osVersion}`,
+              `Screen: ${info.screenWidth}x${info.screenHeight}`,
+              `Orientation: ${info.orientation}`,
+              `ID: ${info.id}`,
+            ].join("\n"),
+          },
+        ],
+      };
+    },
+
+    devlens_set_orientation: async (params: {
+      orientation: "portrait" | "landscape";
+    }) => {
+      const device = await deviceManager.getDevice();
+      await device.setOrientation(params.orientation);
+      return {
+        content: [
+          {
+            type: "text" as const,
+            text: `Orientation set to ${params.orientation}.`,
+          },
+        ],
+      };
+    },
+  };
+}