agent-device 0.1.8 → 0.1.9

package/skills/agent-device/SKILL.md

@@ -45,7 +45,7 @@ agent-device snapshot -c              # Compact output
 agent-device snapshot -d 3            # Limit depth
 agent-device snapshot -s "Camera"      # Scope to label/identifier
 agent-device snapshot --raw           # Raw node output
-agent-device snapshot --backend hybrid # Default: best speed vs correctness trade-off (AX fast, XCTest complete)
+agent-device snapshot --backend xctest # Default: full XCTest snapshot (most complete)
 agent-device snapshot --backend ax    # macOS Accessibility tree (fast, needs permissions)
 agent-device snapshot --backend xctest # XCTest snapshot (slow, no permissions)
 ```
@@ -53,7 +53,7 @@ agent-device snapshot --backend xctest # XCTest snapshot (slow, no permissions)
 Hybrid will automatically fill empty containers (e.g. `group`, `tab bar`) by scoping XCTest to the container label.
 It is recommended because AX is fast but can miss UI details, while XCTest is slower but more complete.
 If you want explicit control or AX is unavailable, use `--backend xctest`.
-In practice, if AX returns a `Tab Bar` group with no children, hybrid will run a scoped XCTest snapshot for `Tab Bar` and insert those nodes under the group.
+Use `--backend ax` when you need faster snapshots and can tolerate missing details.
 
 ### Find (semantic)
 
@@ -139,7 +139,9 @@ agent-device apps --platform android --user-installed
 
 - Always snapshot right before interactions; refs invalidate on UI changes.
 - Prefer `snapshot -i` to reduce output size.
-- On iOS, hybrid is the default and uses AX first, so Accessibility permission is still required.
+- On iOS, `xctest` is the default and does not require Accessibility permission.
+- If XCTest returns 0 nodes (foreground app changed), agent-device falls back to AX when available.
+- `open <app>` can be used within an existing session to switch apps and update the session bundle id.
 - If AX returns the Simulator window or empty tree, restart Simulator or use `--backend xctest`.
 - Use `--session <name>` for parallel sessions; avoid device contention.
 

package/skills/agent-device/references/snapshot-refs.md

@@ -46,4 +46,4 @@ agent-device snapshot -i -s @e3 --platform ios
 
 - Ref not found: re-snapshot.
 - AX returns Simulator window: restart Simulator and re-run.
-- AX empty: verify Accessibility permission or use `--backend xctest` (hybrid is recommended because AX is fast but can miss UI details, while XCTest is slower but more complete).
+- AX empty: verify Accessibility permission or use `--backend xctest` (XCTest is slower but more complete).

package/src/core/dispatch.ts

@@ -28,7 +28,7 @@ export type CommandFlags = {
   snapshotDepth?: number;
   snapshotScope?: string;
   snapshotRaw?: boolean;
-  snapshotBackend?: 'ax' | 'xctest' | 'hybrid';
+  snapshotBackend?: 'ax' | 'xctest';
   noRecord?: boolean;
   recordJson?: boolean;
   appsFilter?: 'launchable' | 'user-installed' | 'all';
@@ -83,7 +83,7 @@ export async function dispatchCommand(
     snapshotDepth?: number;
     snapshotScope?: string;
     snapshotRaw?: boolean;
-    snapshotBackend?: 'ax' | 'xctest' | 'hybrid';
+    snapshotBackend?: 'ax' | 'xctest';
   },
 ): Promise<Record<string, unknown> | void> {
   const interactor = getInteractor(device);
@@ -283,7 +283,7 @@ export async function dispatchCommand(
       return { setting, state };
     }
     case 'snapshot': {
-      const backend = context?.snapshotBackend ?? 'hybrid';
+      const backend = context?.snapshotBackend ?? 'xctest';
       if (device.platform === 'ios') {
         if (device.kind !== 'simulator') {
           throw new AppError(
@@ -295,25 +295,6 @@ export async function dispatchCommand(
           const ax = await snapshotAx(device, { traceLogPath: context?.traceLogPath });
           return { nodes: ax.nodes ?? [], truncated: false, backend: 'ax' };
         }
-        if (backend === 'hybrid') {
-          const ax = await snapshotAx(device, { traceLogPath: context?.traceLogPath });
-          const axNodes = ax.nodes ?? [];
-          const containers = findHybridContainers(axNodes);
-          if (containers.length === 0) {
-            return { nodes: axNodes, truncated: false, backend: 'hybrid' };
-          }
-          const merged = await fillHybridContainers(device, axNodes, containers, {
-            appBundleId: context?.appBundleId,
-            interactiveOnly: context?.snapshotInteractiveOnly,
-            compact: context?.snapshotCompact,
-            depth: context?.snapshotDepth,
-            raw: context?.snapshotRaw,
-            verbose: context?.verbose,
-            logPath: context?.logPath,
-            traceLogPath: context?.traceLogPath,
-          });
-          return { nodes: merged.nodes, truncated: merged.truncated, backend: 'hybrid' };
-        }
         const result = (await runIosRunnerCommand(
           device,
           {
@@ -327,7 +308,16 @@ export async function dispatchCommand(
           },
           { verbose: context?.verbose, logPath: context?.logPath, traceLogPath: context?.traceLogPath },
         )) as { nodes?: RawSnapshotNode[]; truncated?: boolean };
-        return { nodes: result.nodes ?? [], truncated: result.truncated ?? false, backend: 'xctest' };
+        const nodes = result.nodes ?? [];
+        if (nodes.length === 0) {
+          try {
+            const ax = await snapshotAx(device, { traceLogPath: context?.traceLogPath });
+            return { nodes: ax.nodes ?? [], truncated: false, backend: 'ax' };
+          } catch {
+            // keep the empty XCTest snapshot if AX is unavailable
+          }
+        }
+        return { nodes, truncated: result.truncated ?? false, backend: 'xctest' };
       }
       const androidResult = await snapshotAndroid(device, {
         interactiveOnly: context?.snapshotInteractiveOnly,
@@ -343,116 +333,6 @@ export async function dispatchCommand(
   }
 }
 
-type HybridContainer = {
-  index: number;
-  depth: number;
-  label?: string;
-  identifier?: string;
-  type?: string;
-};
-
-const hybridContainerTypes = new Set(['tabbar', 'toolbar', 'group']);
-
-function findHybridContainers(nodes: RawSnapshotNode[]): HybridContainer[] {
-  const containers: HybridContainer[] = [];
-  for (let i = 0; i < nodes.length; i += 1) {
-    const node = nodes[i];
-    const depth = node.depth ?? 0;
-    const nextDepth = nodes[i + 1]?.depth ?? -1;
-    if (nextDepth > depth) continue;
-    const normalized = normalizeSnapshotType(node.type);
-    if (!hybridContainerTypes.has(normalized)) continue;
-    containers.push({
-      index: i,
-      depth,
-      label: node.label,
-      identifier: node.identifier,
-      type: node.type,
-    });
-  }
-  return containers;
-}
-
-async function fillHybridContainers(
-  device: DeviceInfo,
-  axNodes: RawSnapshotNode[],
-  containers: HybridContainer[],
-  options: {
-    appBundleId?: string;
-    interactiveOnly?: boolean;
-    compact?: boolean;
-    depth?: number;
-    raw?: boolean;
-    verbose?: boolean;
-    logPath?: string;
-    traceLogPath?: string;
-  },
-): Promise<{ nodes: RawSnapshotNode[]; truncated: boolean }> {
-  let merged = [...axNodes];
-  let truncated = false;
-  let offset = 0;
-  for (const container of containers) {
-    const scope = resolveContainerScope(container);
-    if (!scope) continue;
-    const result = (await runIosRunnerCommand(
-      device,
-      {
-        command: 'snapshot',
-        appBundleId: options.appBundleId,
-        interactiveOnly: options.interactiveOnly,
-        compact: options.compact,
-        depth: options.depth,
-        scope,
-        raw: options.raw,
-      },
-      { verbose: options.verbose, logPath: options.logPath, traceLogPath: options.traceLogPath },
-    )) as { nodes?: RawSnapshotNode[]; truncated?: boolean };
-    if (result.truncated) truncated = true;
-    const filtered = (result.nodes ?? []).filter((node) => {
-      const normalized = normalizeSnapshotType(node.type);
-      return normalized !== 'application' && normalized !== 'window';
-    });
-    if (filtered.length === 0) continue;
-    const adjusted = adjustDepths(filtered, container.depth + 1);
-    merged.splice(container.index + 1 + offset, 0, ...adjusted);
-    offset += adjusted.length;
-  }
-  merged = merged.map((node, index) => ({ ...node, index }));
-  return { nodes: merged, truncated };
-}
-
-function adjustDepths(nodes: RawSnapshotNode[], baseDepth: number): RawSnapshotNode[] {
-  let minDepth = Number.POSITIVE_INFINITY;
-  for (const node of nodes) {
-    const depth = node.depth ?? 0;
-    if (depth < minDepth) minDepth = depth;
-  }
-  if (!Number.isFinite(minDepth)) minDepth = 0;
-  return nodes.map((node) => ({
-    ...node,
-    depth: baseDepth + (node.depth ?? 0) - minDepth,
-  }));
-}
-
-function normalizeSnapshotType(type?: string): string {
-  if (!type) return '';
-  let value = type.replace(/XCUIElementType/gi, '').toLowerCase();
-  if (value.startsWith('ax')) {
-    value = value.replace(/^ax/, '');
-  }
-  return value;
-}
-
-function resolveContainerScope(container: HybridContainer): string | null {
-  const candidates = [container.label, container.identifier];
-  for (const candidate of candidates) {
-    if (!candidate) continue;
-    const value = candidate.trim();
-    if (value) return value;
-  }
-  return null;
-}
-
 function invertScrollDirection(direction: 'up' | 'down' | 'left' | 'right'): 'up' | 'down' | 'left' | 'right' {
   switch (direction) {
     case 'up':

package/src/daemon.ts

@@ -63,7 +63,7 @@ type SessionAction = {
     snapshotDepth?: number;
     snapshotScope?: string;
     snapshotRaw?: boolean;
-    snapshotBackend?: 'ax' | 'xctest' | 'hybrid';
+    snapshotBackend?: 'ax' | 'xctest';
     noRecord?: boolean;
     recordJson?: boolean;
   };
@@ -91,7 +91,7 @@ function contextFromFlags(
   snapshotCompact?: boolean;
   snapshotDepth?: number;
   snapshotScope?: string;
-  snapshotBackend?: 'ax' | 'xctest' | 'hybrid';
+  snapshotBackend?: 'ax' | 'xctest';
   snapshotRaw?: boolean;
 } {
   return {
@@ -244,13 +244,43 @@ async function handleRequest(req: DaemonRequest): Promise<DaemonResponse> {
 
   if (command === 'open') {
     if (sessions.has(sessionName)) {
-      return {
-        ok: false,
-        error: {
-          code: 'INVALID_ARGS',
-          message: 'Session already active. Close it first or pass a new --session name.',
-        },
+      const session = sessions.get(sessionName);
+      const appName = req.positionals?.[0];
+      if (!session || !appName) {
+        return {
+          ok: false,
+          error: {
+            code: 'INVALID_ARGS',
+            message: 'Session already active. Close it first or pass a new --session name.',
+          },
+        };
+      }
+      let appBundleId: string | undefined;
+      if (session.device.platform === 'ios') {
+        try {
+          const { resolveIosApp } = await import('./platforms/ios/index.ts');
+          appBundleId = await resolveIosApp(session.device, appName);
+        } catch {
+          appBundleId = undefined;
+        }
+      }
+      await dispatchCommand(session.device, 'open', req.positionals ?? [], req.flags?.out, {
+        ...contextFromFlags(req.flags, appBundleId),
+      });
+      const nextSession: SessionState = {
+        ...session,
+        appBundleId,
+        appName,
+        snapshot: undefined,
       };
+      recordAction(nextSession, {
+        command,
+        positionals: req.positionals ?? [],
+        flags: req.flags ?? {},
+        result: { session: sessionName, appName, appBundleId },
+      });
+      sessions.set(sessionName, nextSession);
+      return { ok: true, data: { session: sessionName, appName, appBundleId } };
     }
     const device = await resolveTargetDevice(req.flags ?? {});
     await ensureDeviceReady(device);
@@ -377,7 +407,7 @@ async function handleRequest(req: DaemonRequest): Promise<DaemonResponse> {
     })) as {
       nodes?: RawSnapshotNode[];
       truncated?: boolean;
-      backend?: 'ax' | 'xctest' | 'hybrid' | 'android';
+      backend?: 'ax' | 'xctest' | 'android';
     };
     const rawNodes = data?.nodes ?? [];
     const nodes = attachRefs(req.flags?.snapshotRaw ? rawNodes : pruneGroupNodes(rawNodes));
@@ -767,7 +797,7 @@ async function handleRequest(req: DaemonRequest): Promise<DaemonResponse> {
       })) as {
         nodes?: RawSnapshotNode[];
         truncated?: boolean;
-        backend?: 'ax' | 'xctest' | 'hybrid' | 'android';
+        backend?: 'ax' | 'xctest' | 'android';
       };
       const rawNodes = data?.nodes ?? [];
       const nodes = attachRefs(req.flags?.snapshotRaw ? rawNodes : pruneGroupNodes(rawNodes));
@@ -965,26 +995,6 @@ async function handleRequest(req: DaemonRequest): Promise<DaemonResponse> {
       return { ok: false, error: { code: 'COMMAND_FAILED', message: `Ref ${refInput} not found or has no bounds` } };
     }
     const refLabel = resolveRefLabel(node, session.snapshot.nodes);
-    const label = node.label?.trim();
-    if (
-      session.device.platform === 'ios' &&
-      session.device.kind === 'simulator' &&
-      label &&
-      isLabelUnique(session.snapshot.nodes, label)
-    ) {
-      await runIosRunnerCommand(
-        session.device,
-        { command: 'tap', text: label, appBundleId: session.appBundleId },
-        { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath },
-      );
-      recordAction(session, {
-        command,
-        positionals: req.positionals ?? [],
-        flags: req.flags ?? {},
-        result: { ref, refLabel: label, mode: 'text' },
-      });
-      return { ok: true, data: { ref, mode: 'text' } };
-    }
     const { x, y } = centerOfRect(node.rect);
     await dispatchCommand(session.device, 'press', [String(x), String(y)], req.flags?.out, {
       ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath),
@@ -1022,45 +1032,29 @@ async function handleRequest(req: DaemonRequest): Promise<DaemonResponse> {
       }
       const refLabel = resolveRefLabel(node, session.snapshot.nodes);
       const label = node.label?.trim();
-      if (session.device.platform === 'ios' && session.device.kind === 'simulator') {
-        if (refLabel && isTextInputType(node.type)) {
-          await runIosRunnerCommand(
-            session.device,
-            { command: 'tap', text: refLabel, appBundleId: session.appBundleId },
-            { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath },
-          );
-          await runIosRunnerCommand(
-            session.device,
-            { command: 'type', text, appBundleId: session.appBundleId },
-            { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath },
-          );
-          recordAction(session, {
-            command,
-            positionals: req.positionals ?? [],
-            flags: req.flags ?? {},
-            result: { ref, refLabel, mode: 'text' },
-          });
-          return { ok: true, data: { ref, mode: 'text' } };
-        }
-        if (label && isLabelUnique(session.snapshot.nodes, label)) {
-          await runIosRunnerCommand(
-            session.device,
-            { command: 'tap', text: label, appBundleId: session.appBundleId },
-            { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath },
-          );
-          await runIosRunnerCommand(
-            session.device,
-            { command: 'type', text, appBundleId: session.appBundleId },
-            { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath },
-          );
-          recordAction(session, {
-            command,
-            positionals: req.positionals ?? [],
-            flags: req.flags ?? {},
-            result: { ref, refLabel: label, mode: 'text' },
-          });
-          return { ok: true, data: { ref, mode: 'text' } };
+      if (session.device.platform === 'ios' && session.device.kind === 'simulator' && isTextInputType(node.type)) {
+        const coords = node.rect ? centerOfRect(node.rect) : null;
+        if (!coords) {
+          return {
+            ok: false,
+            error: { code: 'COMMAND_FAILED', message: `Ref ${req.positionals[0]} not found or has no bounds` },
+          };
         }
+        await dispatchCommand(session.device, 'focus', [String(coords.x), String(coords.y)], req.flags?.out, {
+          ...contextFromFlags(req.flags, session.appBundleId, session.trace?.outPath),
+        });
+        await runIosRunnerCommand(
+          session.device,
+          { command: 'type', text, appBundleId: session.appBundleId },
+          { verbose: req.flags?.verbose, logPath, traceLogPath: session?.trace?.outPath },
+        );
+        recordAction(session, {
+          command,
+          positionals: req.positionals ?? [],
+          flags: req.flags ?? {},
+          result: { ref, refLabel: refLabel ?? label, action: 'fill', text },
+        });
+        return { ok: true, data: { ref } };
       }
       const { x, y } = centerOfRect(node.rect);
       const data = await dispatchCommand(

package/src/utils/args.ts

@@ -17,7 +17,7 @@ export type ParsedArgs = {
     snapshotDepth?: number;
     snapshotScope?: string;
     snapshotRaw?: boolean;
-    snapshotBackend?: 'ax' | 'xctest' | 'hybrid';
+    snapshotBackend?: 'ax' | 'xctest';
     appsFilter?: 'launchable' | 'user-installed' | 'all';
     appsMetadata?: boolean;
     noRecord?: boolean;
@@ -81,7 +81,7 @@ export function parseArgs(argv: string[]): ParsedArgs {
         ? arg.split('=')[1]
         : argv[i + 1];
       if (!arg.includes('=')) i += 1;
-      if (value !== 'ax' && value !== 'xctest' && value !== 'hybrid') {
+      if (value !== 'ax' && value !== 'xctest') {
         throw new AppError('INVALID_ARGS', `Invalid backend: ${value}`);
       }
       flags.snapshotBackend = value;
@@ -161,16 +161,15 @@ CLI to control iOS and Android devices for AI agents.
 Commands:
   open [app]                                 Boot device/simulator; optionally launch app
   close [app]                                Close app or just end session
-  snapshot [-i] [-c] [-d <depth>] [-s <scope>] [--raw] [--backend ax|xctest|hybrid]
+  snapshot [-i] [-c] [-d <depth>] [-s <scope>] [--raw] [--backend ax|xctest]
                                              Capture accessibility tree
     -i                                       Interactive elements only
     -c                                       Compact output (drop empty structure)
     -d <depth>                               Limit snapshot depth
     -s <scope>                               Scope snapshot to label/identifier
     --raw                                    Raw node output
-    --backend ax|xctest|hybrid               hybrid: default; AX snapshot with XCTest fill for empty containers
+    --backend ax|xctest                      xctest: default; XCTest snapshot (slower, no permissions)
                                              ax: macOS Accessibility tree (fast, needs permissions)
-                                             xctest: XCTest snapshot (slower, no permissions)
   devices                                   List available devices
   apps [--user-installed|--all|--metadata]  List installed apps (Android launchable by default, iOS simulator)
   appstate                                  Show foreground app/activity

package/src/utils/snapshot.ts

@@ -34,7 +34,7 @@ export type SnapshotState = {
   nodes: SnapshotNode[];
   createdAt: number;
   truncated?: boolean;
-  backend?: 'ax' | 'xctest' | 'hybrid' | 'android';
+  backend?: 'ax' | 'xctest' | 'android';
 };
 
 export function attachRefs(nodes: RawSnapshotNode[]): SnapshotNode[] {