agent-device 0.2.4 → 0.2.5

package/src/platforms/ios/index.ts

@@ -83,68 +83,6 @@ export async function closeIosApp(device: DeviceInfo, app: string): Promise<void
   ]);
 }
 
-export async function pressIos(device: DeviceInfo, x: number, y: number): Promise<void> {
-  ensureSimulator(device, 'press');
-  throw new AppError(
-    'UNSUPPORTED_OPERATION',
-    'simctl io tap is not available; use the XCTest runner for input',
-  );
-}
-
-export async function longPressIos(
-  device: DeviceInfo,
-  x: number,
-  y: number,
-  durationMs = 800,
-): Promise<void> {
-  ensureSimulator(device, 'long-press');
-  throw new AppError(
-    'UNSUPPORTED_OPERATION',
-    'long-press is not supported on iOS simulators without XCTest runner support',
-  );
-}
-
-export async function focusIos(device: DeviceInfo, x: number, y: number): Promise<void> {
-  await pressIos(device, x, y);
-}
-
-export async function typeIos(device: DeviceInfo, text: string): Promise<void> {
-  ensureSimulator(device, 'type');
-  throw new AppError(
-    'UNSUPPORTED_OPERATION',
-    'simctl io keyboard is not available; use the XCTest runner for input',
-  );
-}
-
-export async function fillIos(
-  device: DeviceInfo,
-  x: number,
-  y: number,
-  text: string,
-): Promise<void> {
-  await focusIos(device, x, y);
-  await typeIos(device, text);
-}
-
-export async function scrollIos(
-  device: DeviceInfo,
-  direction: string,
-  amount = 0.6,
-): Promise<void> {
-  ensureSimulator(device, 'scroll');
-  throw new AppError(
-    'UNSUPPORTED_OPERATION',
-    'simctl io swipe is not available; use the XCTest runner for input',
-  );
-}
-
-export async function scrollIntoViewIos(text: string): Promise<void> {
-  throw new AppError(
-    'UNSUPPORTED_OPERATION',
-    `scrollintoview is not supported on iOS without UI automation (${text})`,
-  );
-}
-
 export async function screenshotIos(device: DeviceInfo, outPath: string): Promise<void> {
   if (device.kind === 'simulator') {
     await ensureBootedSimulator(device);

package/src/platforms/ios/runner-client.ts

@@ -11,6 +11,7 @@ import net from 'node:net';
 export type RunnerCommand = {
   command:
     | 'tap'
+    | 'longPress'
     | 'type'
     | 'swipe'
     | 'findText'
@@ -27,6 +28,7 @@ export type RunnerCommand = {
   action?: 'get' | 'accept' | 'dismiss';
   x?: number;
   y?: number;
+  durationMs?: number;
   direction?: 'up' | 'down' | 'left' | 'right';
   scale?: number;
   interactiveOnly?: boolean;

package/src/utils/args.ts

@@ -21,8 +21,9 @@ export type ParsedArgs = {
     appsFilter?: 'launchable' | 'user-installed' | 'all';
     appsMetadata?: boolean;
     activity?: string;
+    saveScript?: boolean;
     noRecord?: boolean;
-    recordJson?: boolean;
+    replayUpdate?: boolean;
     help: boolean;
   };
 };
@@ -61,8 +62,12 @@ export function parseArgs(argv: string[]): ParsedArgs {
       flags.noRecord = true;
       continue;
     }
-    if (arg === '--record-json') {
-      flags.recordJson = true;
+    if (arg === '--save-script') {
+      flags.saveScript = true;
+      continue;
+    }
+    if (arg === '--update' || arg === '-u') {
+      flags.replayUpdate = true;
       continue;
     }
     if (arg === '--user-installed') {
@@ -180,17 +185,19 @@ Commands:
   back                                      Navigate back (where supported)
   home                                      Go to home screen (where supported)
   app-switcher                              Open app switcher (where supported)
-  wait <ms>|text <text>|@ref [timeoutMs]     Wait for duration or text to appear
+  wait <ms>|text <text>|@ref|<selector> [timeoutMs]
+                                             Wait for duration, text, ref, or selector to appear
   alert [get|accept|dismiss|wait] [timeout] Inspect or handle alert (iOS simulator)
-  click <@ref>                               Click element by snapshot ref
-  get text <@ref>                            Return element text by ref
-  get attrs <@ref>                           Return element attributes by ref
-  replay <path>                              Replay a recorded session
+  click <@ref|selector>                      Click element by snapshot ref or selector
+  get text <@ref|selector>                   Return element text by ref or selector
+  get attrs <@ref|selector>                  Return element attributes by ref or selector
+  replay <path> [--update|-u]                Replay a recorded session
   press <x> <y>                              Tap at coordinates
   long-press <x> <y> [durationMs]            Long press (where supported)
   focus <x> <y>                              Focus input at coordinates
   type <text>                                Type text in focused field
-  fill <x> <y> <text> | fill <@ref> <text>   Tap then type
+  fill <x> <y> <text> | fill <@ref|selector> <text>
+                                             Tap then type
   scroll <direction> [amount]                Scroll in direction (0-1 amount)
   scrollintoview <text>                      Scroll until text appears (Android only)
   screenshot [path]                          Capture screenshot
@@ -204,6 +211,7 @@ Commands:
   find value <value> <action> [value]        Find by value
   find role <role> <action> [value]          Find by role/type
   find id <id> <action> [value]              Find by identifier/resource-id
+  is <predicate> <selector> [value]          Assert UI state (visible|hidden|exists|editable|selected|text)
   settings <wifi|airplane|location> <on|off> Toggle OS settings (simulators)
   session list                               List active sessions
 
@@ -216,8 +224,9 @@ Flags:
   --session <name>                           Named session
   --verbose                                  Stream daemon/runner logs
   --json                                     JSON output
+  --save-script                             Save session script (.ad) on close
   --no-record                                Do not record this action
-  --record-json                              Record JSON session log
+  --update, -u                               Replay: update selectors and rewrite replay file in place
   --user-installed                           Apps: list user-installed packages (Android only)
   --all                                      Apps: list all packages (Android only)
 `;

package/src/utils/interactors.ts

@@ -15,17 +15,18 @@ import {
 } from '../platforms/android/index.ts';
 import {
   closeIosApp,
-  fillIos,
-  focusIos,
-  longPressIos,
   openIosApp,
   openIosDevice,
-  pressIos,
-  scrollIos,
-  scrollIntoViewIos,
   screenshotIos,
-  typeIos,
 } from '../platforms/ios/index.ts';
+import { runIosRunnerCommand } from '../platforms/ios/runner-client.ts';
+
+export type RunnerContext = {
+  appBundleId?: string;
+  verbose?: boolean;
+  logPath?: string;
+  traceLogPath?: string;
+};
 
 export type Interactor = {
   open(app: string, options?: { activity?: string }): Promise<void>;
@@ -37,11 +38,11 @@ export type Interactor = {
   type(text: string): Promise<void>;
   fill(x: number, y: number, text: string): Promise<void>;
   scroll(direction: string, amount?: number): Promise<void>;
-  scrollIntoView(text: string): Promise<void>;
+  scrollIntoView(text: string): Promise<{ attempts?: number } | void>;
   screenshot(outPath: string): Promise<void>;
 };
 
-export function getInteractor(device: DeviceInfo): Interactor {
+export function getInteractor(device: DeviceInfo, runnerContext: RunnerContext): Interactor {
   switch (device.platform) {
     case 'android':
       return {
@@ -62,16 +63,101 @@ export function getInteractor(device: DeviceInfo): Interactor {
         open: (app) => openIosApp(device, app),
         openDevice: () => openIosDevice(device),
         close: (app) => closeIosApp(device, app),
-        tap: (x, y) => pressIos(device, x, y),
-        longPress: (x, y, durationMs) => longPressIos(device, x, y, durationMs),
-        focus: (x, y) => focusIos(device, x, y),
-        type: (text) => typeIos(device, text),
-        fill: (x, y, text) => fillIos(device, x, y, text),
-        scroll: (direction, amount) => scrollIos(device, direction, amount),
-        scrollIntoView: (text) => scrollIntoViewIos(text),
         screenshot: (outPath) => screenshotIos(device, outPath),
+        ...iosRunnerOverrides(device, runnerContext),
       };
     default:
       throw new AppError('UNSUPPORTED_PLATFORM', `Unsupported platform: ${device.platform}`);
   }
 }
+
+type IoRunnerOverrides = Pick<Interactor, 'tap' | 'longPress' | 'focus' | 'type' | 'fill' | 'scroll' | 'scrollIntoView'>;
+
+function iosRunnerOverrides(device: DeviceInfo, ctx: RunnerContext): IoRunnerOverrides {
+  const runnerOpts = { verbose: ctx.verbose, logPath: ctx.logPath, traceLogPath: ctx.traceLogPath };
+
+  return {
+    tap: async (x, y) => {
+      await runIosRunnerCommand(
+        device,
+        { command: 'tap', x, y, appBundleId: ctx.appBundleId },
+        runnerOpts,
+      );
+    },
+    longPress: async (x, y, durationMs) => {
+      await runIosRunnerCommand(
+        device,
+        { command: 'longPress', x, y, durationMs, appBundleId: ctx.appBundleId },
+        runnerOpts,
+      );
+    },
+    focus: async (x, y) => {
+      await runIosRunnerCommand(
+        device,
+        { command: 'tap', x, y, appBundleId: ctx.appBundleId },
+        runnerOpts,
+      );
+    },
+    type: async (text) => {
+      await runIosRunnerCommand(
+        device,
+        { command: 'type', text, appBundleId: ctx.appBundleId },
+        runnerOpts,
+      );
+    },
+    fill: async (x, y, text) => {
+      await runIosRunnerCommand(
+        device,
+        { command: 'tap', x, y, appBundleId: ctx.appBundleId },
+        runnerOpts,
+      );
+      await runIosRunnerCommand(
+        device,
+        { command: 'type', text, clearFirst: true, appBundleId: ctx.appBundleId },
+        runnerOpts,
+      );
+    },
+    scroll: async (direction, _amount) => {
+      if (!['up', 'down', 'left', 'right'].includes(direction)) {
+        throw new AppError('INVALID_ARGS', `Unknown direction: ${direction}`);
+      }
+      const inverted = invertScrollDirection(direction as 'up' | 'down' | 'left' | 'right');
+      await runIosRunnerCommand(
+        device,
+        { command: 'swipe', direction: inverted, appBundleId: ctx.appBundleId },
+        runnerOpts,
+      );
+    },
+    scrollIntoView: async (text) => {
+      const maxAttempts = 8;
+      for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+        const found = (await runIosRunnerCommand(
+          device,
+          { command: 'findText', text, appBundleId: ctx.appBundleId },
+          runnerOpts,
+        )) as { found?: boolean };
+        if (found?.found) return { attempts: attempt + 1 };
+        await runIosRunnerCommand(
+          device,
+          { command: 'swipe', direction: 'up', appBundleId: ctx.appBundleId },
+          runnerOpts,
+        );
+        await new Promise((resolve) => setTimeout(resolve, 300));
+      }
+      throw new AppError('COMMAND_FAILED', `scrollintoview could not find text: ${text}`);
+    },
+  };
+}
+
+function invertScrollDirection(direction: 'up' | 'down' | 'left' | 'right'): 'up' | 'down' | 'left' | 'right' {
+  switch (direction) {
+    case 'up':
+      return 'down';
+    case 'down':
+      return 'up';
+    case 'left':
+      return 'right';
+    case 'right':
+      return 'left';
+  }
+}

package/src/utils/snapshot.ts

@@ -21,6 +21,7 @@ export type RawSnapshotNode = {
   identifier?: string;
   rect?: Rect;
   enabled?: boolean;
+  selected?: boolean;
   hittable?: boolean;
   depth?: number;
   parentIndex?: number;