agent-device 0.10.0 → 0.10.2

package/dist/src/platforms/android/index.d.ts

@@ -1,6 +1,6 @@
 export { ensureAdb } from './adb.ts';
 export { resolveAndroidApp, listAndroidApps, inferAndroidAppName, getAndroidAppState, openAndroidApp, isAmStartError, parseAndroidLaunchComponent, openAndroidDevice, closeAndroidApp, installAndroidInstallablePath, installAndroidInstallablePathAndResolvePackageName, installAndroidApp, reinstallAndroidApp, } from './app-lifecycle.ts';
-export { pressAndroid, swipeAndroid, backAndroid, homeAndroid, appSwitcherAndroid, longPressAndroid, typeAndroid, focusAndroid, fillAndroid, scrollAndroid, scrollIntoViewAndroid, } from './input-actions.ts';
+export { pressAndroid, swipeAndroid, backAndroid, homeAndroid, appSwitcherAndroid, longPressAndroid, typeAndroid, focusAndroid, fillAndroid, readAndroidTextAtPoint, scrollAndroid, scrollIntoViewAndroid, getAndroidScreenSize, } from './input-actions.ts';
 export { type AndroidKeyboardState, getAndroidKeyboardState, dismissAndroidKeyboard, readAndroidClipboardText, writeAndroidClipboardText, } from './device-input-state.ts';
 export { setAndroidSetting } from './settings.ts';
 export { pushAndroidNotification } from './notifications.ts';

package/dist/src/platforms/android/input-actions.d.ts

@@ -10,3 +10,8 @@ export declare function focusAndroid(device: DeviceInfo, x: number, y: number):
 export declare function fillAndroid(device: DeviceInfo, x: number, y: number, text: string): Promise<void>;
 export declare function scrollAndroid(device: DeviceInfo, direction: string, amount?: number): Promise<void>;
 export declare function scrollIntoViewAndroid(device: DeviceInfo, text: string): Promise<void>;
+export declare function getAndroidScreenSize(device: DeviceInfo): Promise<{
+    width: number;
+    height: number;
+}>;
+export declare function readAndroidTextAtPoint(device: DeviceInfo, x: number, y: number): Promise<string | null>;

package/dist/src/platforms/android/settings.d.ts

@@ -1,3 +1,3 @@
 import type { DeviceInfo } from '../../utils/device.ts';
 import { type PermissionSettingOptions } from '../permission-utils.ts';
-export declare function setAndroidSetting(device: DeviceInfo, setting: string, state: string, appPackage?: string, options?: PermissionSettingOptions): Promise<void>;
+export declare function setAndroidSetting(device: DeviceInfo, setting: string, state: string, appPackage?: string, options?: PermissionSettingOptions): Promise<Record<string, unknown> | void>;

package/dist/src/platforms/ios/apps.d.ts

@@ -29,6 +29,6 @@ export declare function installIosInstallablePath(device: DeviceInfo, installabl
 export declare function readIosClipboardText(device: DeviceInfo): Promise<string>;
 export declare function writeIosClipboardText(device: DeviceInfo, text: string): Promise<void>;
 export declare function pushIosNotification(device: DeviceInfo, bundleId: string, payload: Record<string, unknown>): Promise<void>;
-export declare function setIosSetting(device: DeviceInfo, setting: string, state: string, appBundleId?: string, options?: PermissionSettingOptions): Promise<void>;
+export declare function setIosSetting(device: DeviceInfo, setting: string, state: string, appBundleId?: string, options?: PermissionSettingOptions): Promise<Record<string, unknown> | void>;
 export declare function listIosApps(device: DeviceInfo, filter?: 'user-installed' | 'all'): Promise<IosAppInfo[]>;
 export declare function listSimulatorApps(device: DeviceInfo): Promise<IosAppInfo[]>;

package/dist/src/platforms/ios/macos-helper.d.ts

@@ -0,0 +1,69 @@
+import type { SessionSurface } from '../../core/session-surface.ts';
+export type MacOsPermissionTarget = 'accessibility' | 'screen-recording' | 'input-monitoring';
+export type MacOsSnapshotNode = {
+    index: number;
+    type?: string;
+    role?: string;
+    subrole?: string;
+    label?: string;
+    value?: string;
+    identifier?: string;
+    rect?: {
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+    };
+    enabled?: boolean;
+    selected?: boolean;
+    hittable?: boolean;
+    depth?: number;
+    parentIndex?: number;
+    pid?: number;
+    bundleId?: string;
+    appName?: string;
+    windowTitle?: string;
+    surface?: string;
+};
+export declare function resolveMacOsHelperPackageRootFrom(modulePath: string): string;
+export declare function resolveFrontmostMacOsApp(): Promise<{
+    bundleId?: string;
+    appName?: string;
+    pid?: number;
+}>;
+export declare function quitMacOsApp(bundleId: string): Promise<{
+    bundleId: string;
+    running: boolean;
+    terminated: boolean;
+    forceTerminated: boolean;
+}>;
+export declare function runMacOsPermissionAction(action: 'grant' | 'reset', target: MacOsPermissionTarget): Promise<{
+    target: MacOsPermissionTarget;
+    granted: boolean;
+    requested: boolean;
+    openedSettings: boolean;
+    action: 'grant' | 'reset';
+    message?: string;
+}>;
+export declare function runMacOsAlertAction(action: 'get' | 'accept' | 'dismiss', options?: {
+    bundleId?: string;
+    surface?: SessionSurface;
+}): Promise<{
+    title?: string;
+    role?: string;
+    buttons?: string[];
+    action?: string;
+    bundleId?: string;
+}>;
+export declare function runMacOsSnapshotAction(surface: Exclude<SessionSurface, 'app'>): Promise<{
+    surface: Exclude<SessionSurface, 'app'>;
+    nodes: MacOsSnapshotNode[];
+    truncated: boolean;
+    backend: 'macos-helper';
+}>;
+export declare function runMacOsReadTextAction(x: number, y: number, options?: {
+    bundleId?: string;
+    surface?: SessionSurface;
+}): Promise<{
+    text: string;
+}>;

package/dist/src/platforms/ios/runner-client.d.ts

@@ -1,7 +1,7 @@
 import type { DeviceInfo } from '../../utils/device.ts';
 import type { ClickButton } from '../../core/click-button.ts';
 export type RunnerCommand = {
-    command: 'tap' | 'mouseClick' | 'tapSeries' | 'longPress' | 'drag' | 'dragSeries' | 'type' | 'swipe' | 'findText' | 'snapshot' | 'screenshot' | 'back' | 'home' | 'appSwitcher' | 'alert' | 'pinch' | 'recordStart' | 'recordStop' | 'shutdown';
+    command: 'tap' | 'mouseClick' | 'tapSeries' | 'longPress' | 'drag' | 'dragSeries' | 'type' | 'swipe' | 'findText' | 'readText' | 'snapshot' | 'screenshot' | 'back' | 'home' | 'appSwitcher' | 'alert' | 'pinch' | 'recordStart' | 'recordStop' | 'uptime' | 'shutdown';
     appBundleId?: string;
     text?: string;
     action?: 'get' | 'accept' | 'dismiss';
@@ -35,4 +35,4 @@ export declare function runIosRunnerCommand(device: DeviceInfo, command: RunnerC
 }): Promise<Record<string, unknown>>;
 export { isRetryableRunnerError, shouldRetryRunnerConnectError, resolveRunnerEarlyExitHint, } from './runner-errors.ts';
 export { resolveRunnerDestination, resolveRunnerBuildDestination, resolveRunnerMaxConcurrentDestinationsFlag, resolveRunnerSigningBuildSettings, resolveRunnerBundleBuildSettings, assertSafeDerivedCleanup, IOS_RUNNER_CONTAINER_BUNDLE_IDS, } from './runner-xctestrun.ts';
-export { stopIosRunnerSession, abortAllIosRunnerSessions, stopAllIosRunnerSessions, } from './runner-session.ts';
+export { getRunnerSessionSnapshot, stopIosRunnerSession, abortAllIosRunnerSessions, stopAllIosRunnerSessions, } from './runner-session.ts';

package/dist/src/platforms/ios/runner-session.d.ts

@@ -2,6 +2,7 @@ import { type ExecResult, type ExecBackgroundResult } from '../../utils/exec.ts'
 import type { DeviceInfo } from '../../utils/device.ts';
 import type { RunnerCommand } from './runner-client.ts';
 export type RunnerSession = {
+    sessionId: string;
     device: DeviceInfo;
     deviceId: string;
     port: number;
@@ -16,6 +17,10 @@ export declare function ensureRunnerSession(device: DeviceInfo, options: {
     logPath?: string;
     traceLogPath?: string;
 }): Promise<RunnerSession>;
+export declare function getRunnerSessionSnapshot(deviceId: string): {
+    sessionId: string;
+    alive: boolean;
+} | null;
 export declare function stopRunnerSession(session: RunnerSession): Promise<void>;
 export declare function stopIosRunnerSession(deviceId: string): Promise<void>;
 export declare function abortAllIosRunnerSessions(): Promise<void>;

package/dist/src/platforms/ios/runner-xctestrun.d.ts

@@ -4,7 +4,7 @@ export declare const runnerPrepProcesses: Set<import("child_process").ChildProce
 export declare const IOS_RUNNER_CONTAINER_BUNDLE_IDS: string[];
 type EnsureXctestrunDeps = {
     findProjectRoot: () => string;
-    findXctestrun: (root: string) => string | null;
+    findXctestrun: (root: string, device?: DeviceInfo) => string | null;
     xctestrunReferencesProjectRoot: (xctestrunPath: string, projectRoot: string) => boolean;
     resolveExistingXctestrunProductPaths: (xctestrunPath: string) => string[] | null;
     repairRunnerProductsIfNeeded: (device: DeviceInfo, productPaths: string[], xctestrunPath: string) => Promise<void>;
@@ -22,6 +22,8 @@ export declare function ensureXctestrun(device: DeviceInfo, options: {
     traceLogPath?: string;
 }, deps?: EnsureXctestrunDeps): Promise<string>;
 export declare function shouldDeleteRunnerDerivedRootEntry(entryName: string): boolean;
+export declare function findXctestrun(root: string, device?: DeviceInfo): string | null;
+export declare function scoreXctestrunCandidate(candidatePath: string, device: DeviceInfo): number;
 export declare function xctestrunReferencesProjectRoot(xctestrunPath: string, projectRoot: string): boolean;
 export declare function prepareXctestrunWithEnv(xctestrunPath: string, envVars: Record<string, string>, suffix: string): Promise<{
     xctestrunPath: string;

package/dist/src/recording/overlay.d.ts

@@ -0,0 +1,10 @@
+export declare function getRecordingOverlaySupportWarning(hostPlatform?: NodeJS.Platform): string | undefined;
+export declare function trimRecordingStart(params: {
+    videoPath: string;
+    trimStartMs: number;
+}): Promise<void>;
+export declare function overlayRecordingTouches(params: {
+    videoPath: string;
+    telemetryPath: string;
+    targetLabel?: string;
+}): Promise<void>;

package/dist/src/utils/command-schema.d.ts

@@ -54,6 +54,7 @@ export type CliFlags = {
     appsFilter?: 'user-installed' | 'all';
     count?: number;
     fps?: number;
+    hideTouches?: boolean;
     intervalMs?: number;
     holdMs?: number;
     jitterPx?: number;
@@ -66,6 +67,7 @@ export type CliFlags = {
     saveScript?: boolean | string;
     shutdown?: boolean;
     relaunch?: boolean;
+    surface?: 'app' | 'frontmost-app' | 'desktop' | 'menubar';
     headless?: boolean;
     restart?: boolean;
     noRecord?: boolean;

package/dist/src/utils/interactors.d.ts

@@ -15,14 +15,14 @@ type Interactor = {
     }): Promise<void>;
     openDevice(): Promise<void>;
     close(app: string): Promise<void>;
-    tap(x: number, y: number): Promise<void>;
-    doubleTap(x: number, y: number): Promise<void>;
-    swipe(x1: number, y1: number, x2: number, y2: number, durationMs?: number): Promise<void>;
-    longPress(x: number, y: number, durationMs?: number): Promise<void>;
-    focus(x: number, y: number): Promise<void>;
+    tap(x: number, y: number): Promise<Record<string, unknown> | void>;
+    doubleTap(x: number, y: number): Promise<Record<string, unknown> | void>;
+    swipe(x1: number, y1: number, x2: number, y2: number, durationMs?: number): Promise<Record<string, unknown> | void>;
+    longPress(x: number, y: number, durationMs?: number): Promise<Record<string, unknown> | void>;
+    focus(x: number, y: number): Promise<Record<string, unknown> | void>;
     type(text: string): Promise<void>;
-    fill(x: number, y: number, text: string): Promise<void>;
-    scroll(direction: string, amount?: number): Promise<void>;
+    fill(x: number, y: number, text: string): Promise<Record<string, unknown> | void>;
+    scroll(direction: string, amount?: number): Promise<Record<string, unknown> | void>;
     scrollIntoView(text: string): Promise<{
         attempts?: number;
     } | void>;
@@ -32,7 +32,7 @@ type Interactor = {
     appSwitcher(): Promise<void>;
     readClipboard(): Promise<string>;
     writeClipboard(text: string): Promise<void>;
-    setSetting(setting: string, state: string, appId?: string, options?: PermissionSettingOptions): Promise<void>;
+    setSetting(setting: string, state: string, appId?: string, options?: PermissionSettingOptions): Promise<Record<string, unknown> | void>;
 };
 export declare function getInteractor(device: DeviceInfo, runnerContext: RunnerContext): Interactor;
 export {};

package/dist/src/utils/snapshot-lines.d.ts

@@ -5,8 +5,11 @@ type SnapshotDisplayLine = {
     type: string;
     text: string;
 };
-export declare function buildSnapshotDisplayLines(nodes: SnapshotNode[]): SnapshotDisplayLine[];
-export declare function formatSnapshotLine(node: SnapshotNode, depth: number, hiddenGroup: boolean, normalizedType?: string): string;
+type SnapshotLineFormatOptions = {
+    summarizeTextSurfaces?: boolean;
+};
+export declare function buildSnapshotDisplayLines(nodes: SnapshotNode[], options?: SnapshotLineFormatOptions): SnapshotDisplayLine[];
+export declare function formatSnapshotLine(node: SnapshotNode, depth: number, hiddenGroup: boolean, normalizedType?: string, options?: SnapshotLineFormatOptions): string;
 export declare function displayLabel(node: SnapshotNode, type: string): string;
 export declare function formatRole(type: string): string;
 export {};

package/dist/src/utils/snapshot.d.ts

@@ -14,6 +14,8 @@ export type SnapshotOptions = {
 export type RawSnapshotNode = {
     index: number;
     type?: string;
+    role?: string;
+    subrole?: string;
     label?: string;
     value?: string;
     identifier?: string;
@@ -23,6 +25,11 @@ export type RawSnapshotNode = {
     hittable?: boolean;
     depth?: number;
     parentIndex?: number;
+    pid?: number;
+    bundleId?: string;
+    appName?: string;
+    windowTitle?: string;
+    surface?: string;
 };
 export type SnapshotNode = RawSnapshotNode & {
     ref: string;
@@ -31,7 +38,7 @@ export type SnapshotState = {
     nodes: SnapshotNode[];
     createdAt: number;
     truncated?: boolean;
-    backend?: 'xctest' | 'android';
+    backend?: 'xctest' | 'android' | 'macos-helper';
 };
 export declare function attachRefs(nodes: RawSnapshotNode[]): SnapshotNode[];
 export declare function normalizeRef(input: string): string | null;

package/dist/src/utils/text-surface.d.ts

@@ -0,0 +1,19 @@
+type TextSurfaceNode = {
+    type?: string;
+    label?: string;
+    value?: string;
+    identifier?: string;
+    role?: string;
+    subrole?: string;
+};
+export declare function extractReadableText(node: TextSurfaceNode): string;
+export declare function isLargeTextSurface(node: TextSurfaceNode, displayType?: string): boolean;
+export declare function buildTextPreview(text: string): string;
+export declare function describeTextSurface(node: TextSurfaceNode, displayType?: string): {
+    text: string;
+    isLargeSurface: boolean;
+    shouldSummarize: boolean;
+};
+export declare function shouldSummarizeTextSurface(text: string): boolean;
+export declare function trimText(value: unknown): string;
+export {};

package/dist/src/utils/video.d.ts

@@ -0,0 +1,9 @@
+export declare function waitForStableFile(filePath: string, options?: {
+    pollMs?: number;
+    attempts?: number;
+}): Promise<void>;
+export declare function isPlayableVideo(filePath: string): Promise<boolean>;
+export declare function waitForPlayableVideo(filePath: string, options?: {
+    pollMs?: number;
+    attempts?: number;
+}): Promise<void>;

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -3,6 +3,16 @@ import XCTest
 extension RunnerTests {
   // MARK: - Main Thread Dispatch
 
+  private func currentUptimeMs() -> Double {
+    ProcessInfo.processInfo.systemUptime * 1000
+  }
+
+  private func measureGesture(_ action: () -> Void) -> (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double) {
+    let gestureStartUptimeMs = currentUptimeMs()
+    action()
+    return (gestureStartUptimeMs, currentUptimeMs())
+  }
+
   func execute(command: Command) throws -> Response {
     if Thread.isMainThread {
       return try executeOnMainSafely(command: command)
@@ -175,7 +185,7 @@ extension RunnerTests {
       }
       do {
         let resolvedOutPath = resolveRecordingOutPath(requestedOutPath)
-        let fpsLabel = command.fps.map(String.init) ?? "max"
+        let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps)
         NSLog(
           "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@",
           requestedOutPath,
@@ -204,26 +214,80 @@ extension RunnerTests {
         activeRecording = nil
         return Response(ok: false, error: ErrorPayload(message: "failed to stop recording: \(error.localizedDescription)"))
       }
+    case .uptime:
+      return Response(
+        ok: true,
+        data: DataPayload(currentUptimeMs: currentUptimeMs())
+      )
     case .tap:
       if let text = command.text {
         if let element = findElement(app: activeApp, text: text) {
-          element.tap()
-          return Response(ok: true, data: DataPayload(message: "tapped"))
+          let timing = measureGesture {
+            withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+              element.tap()
+            }
+          }
+          return Response(
+            ok: true,
+            data: DataPayload(
+              message: "tapped",
+              gestureStartUptimeMs: timing.gestureStartUptimeMs,
+              gestureEndUptimeMs: timing.gestureEndUptimeMs
+            )
+          )
         }
         return Response(ok: false, error: ErrorPayload(message: "element not found"))
       }
       if let x = command.x, let y = command.y {
-        tapAt(app: activeApp, x: x, y: y)
-        return Response(ok: true, data: DataPayload(message: "tapped"))
+        let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
+        let timing = measureGesture {
+          withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+            tapAt(app: activeApp, x: x, y: y)
+          }
+        }
+        return Response(
+          ok: true,
+          data: DataPayload(
+            message: "tapped",
+            gestureStartUptimeMs: timing.gestureStartUptimeMs,
+            gestureEndUptimeMs: timing.gestureEndUptimeMs,
+            x: touchFrame.x,
+            y: touchFrame.y,
+            referenceWidth: touchFrame.referenceWidth,
+            referenceHeight: touchFrame.referenceHeight
+          )
+        )
       }
       return Response(ok: false, error: ErrorPayload(message: "tap requires text or x/y"))
     case .mouseClick:
       guard let x = command.x, let y = command.y else {
         return Response(ok: false, error: ErrorPayload(message: "mouseClick requires x and y"))
       }
+      let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
       do {
-        try mouseClickAt(app: activeApp, x: x, y: y, button: command.button ?? "primary")
-        return Response(ok: true, data: DataPayload(message: "clicked"))
+        var clickError: Error?
+        let timing = measureGesture {
+          do {
+            try mouseClickAt(app: activeApp, x: x, y: y, button: command.button ?? "primary")
+          } catch {
+            clickError = error
+          }
+        }
+        if let clickError {
+          throw clickError
+        }
+        return Response(
+          ok: true,
+          data: DataPayload(
+            message: "clicked",
+            gestureStartUptimeMs: timing.gestureStartUptimeMs,
+            gestureEndUptimeMs: timing.gestureEndUptimeMs,
+            x: touchFrame.x,
+            y: touchFrame.y,
+            referenceWidth: touchFrame.referenceWidth,
+            referenceHeight: touchFrame.referenceHeight
+          )
+        )
       } catch {
         return Response(ok: false, error: ErrorPayload(message: error.localizedDescription))
       }
@@ -234,32 +298,95 @@ extension RunnerTests {
       let count = max(Int(command.count ?? 1), 1)
       let intervalMs = max(command.intervalMs ?? 0, 0)
       let doubleTap = command.doubleTap ?? false
+      let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
       if doubleTap {
-        runSeries(count: count, pauseMs: intervalMs) { _ in
-          doubleTapAt(app: activeApp, x: x, y: y)
+        let timing = measureGesture {
+          withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+            runSeries(count: count, pauseMs: intervalMs) { _ in
+              doubleTapAt(app: activeApp, x: x, y: y)
+            }
+          }
         }
-        return Response(ok: true, data: DataPayload(message: "tap series"))
+        return Response(
+          ok: true,
+          data: DataPayload(
+            message: "tap series",
+            gestureStartUptimeMs: timing.gestureStartUptimeMs,
+            gestureEndUptimeMs: timing.gestureEndUptimeMs,
+            x: touchFrame.x,
+            y: touchFrame.y,
+            referenceWidth: touchFrame.referenceWidth,
+            referenceHeight: touchFrame.referenceHeight
+          )
+        )
       }
-      runSeries(count: count, pauseMs: intervalMs) { _ in
-        tapAt(app: activeApp, x: x, y: y)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          runSeries(count: count, pauseMs: intervalMs) { _ in
+            tapAt(app: activeApp, x: x, y: y)
+          }
+        }
       }
-      return Response(ok: true, data: DataPayload(message: "tap series"))
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "tap series",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs,
+          x: touchFrame.x,
+          y: touchFrame.y,
+          referenceWidth: touchFrame.referenceWidth,
+          referenceHeight: touchFrame.referenceHeight
+        )
+      )
     case .longPress:
       guard let x = command.x, let y = command.y else {
         return Response(ok: false, error: ErrorPayload(message: "longPress requires x and y"))
       }
       let duration = (command.durationMs ?? 800) / 1000.0
-      longPressAt(app: activeApp, x: x, y: y, duration: duration)
-      return Response(ok: true, data: DataPayload(message: "long pressed"))
+      let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          longPressAt(app: activeApp, x: x, y: y, duration: duration)
+        }
+      }
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "long pressed",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs,
+          x: touchFrame.x,
+          y: touchFrame.y,
+          referenceWidth: touchFrame.referenceWidth,
+          referenceHeight: touchFrame.referenceHeight
+        )
+      )
     case .drag:
       guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
         return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2"))
       }
       let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
-      withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
-        dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+      let dragFrame = resolvedDragVisualizationFrame(app: activeApp, x: x, y: y, x2: x2, y2: y2)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+        }
       }
-      return Response(ok: true, data: DataPayload(message: "dragged"))
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "dragged",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs,
+          x: dragFrame.x,
+          y: dragFrame.y,
+          x2: dragFrame.x2,
+          y2: dragFrame.y2,
+          referenceWidth: dragFrame.referenceWidth,
+          referenceHeight: dragFrame.referenceHeight
+        )
+      )
     case .dragSeries:
       guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
         return Response(ok: false, error: ErrorPayload(message: "dragSeries requires x, y, x2, and y2"))
@@ -271,17 +398,26 @@ extension RunnerTests {
         return Response(ok: false, error: ErrorPayload(message: "dragSeries pattern must be one-way or ping-pong"))
       }
       let holdDuration = min(max((command.durationMs ?? 60) / 1000.0, 0.016), 10.0)
-      withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
-        runSeries(count: count, pauseMs: pauseMs) { idx in
-          let reverse = pattern == "ping-pong" && (idx % 2 == 1)
-          if reverse {
-            dragAt(app: activeApp, x: x2, y: y2, x2: x, y2: y, holdDuration: holdDuration)
-          } else {
-            dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          runSeries(count: count, pauseMs: pauseMs) { idx in
+            let reverse = pattern == "ping-pong" && (idx % 2 == 1)
+            if reverse {
+              dragAt(app: activeApp, x: x2, y: y2, x2: x, y2: y, holdDuration: holdDuration)
+            } else {
+              dragAt(app: activeApp, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+            }
           }
         }
       }
-      return Response(ok: true, data: DataPayload(message: "drag series"))
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "drag series",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs
+        )
+      )
     case .type:
       guard let text = command.text else {
         return Response(ok: false, error: ErrorPayload(message: "type requires text"))
@@ -304,16 +440,36 @@ extension RunnerTests {
       guard let direction = command.direction else {
         return Response(ok: false, error: ErrorPayload(message: "swipe requires direction"))
       }
-      withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
-        swipe(app: activeApp, direction: direction)
+      let referenceFrame = resolvedGestureReferenceFrame(app: activeApp)
+      let timing = measureGesture {
+        withTemporaryScrollIdleTimeoutIfSupported(activeApp) {
+          swipe(app: activeApp, direction: direction)
+        }
       }
-      return Response(ok: true, data: DataPayload(message: "swiped"))
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "swiped",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs,
+          referenceWidth: referenceFrame.referenceWidth,
+          referenceHeight: referenceFrame.referenceHeight
+        )
+      )
     case .findText:
       guard let text = command.text else {
         return Response(ok: false, error: ErrorPayload(message: "findText requires text"))
       }
       let found = findElement(app: activeApp, text: text) != nil
       return Response(ok: true, data: DataPayload(found: found))
+    case .readText:
+      guard let x = command.x, let y = command.y else {
+        return Response(ok: false, error: ErrorPayload(message: "readText requires x and y"))
+      }
+      guard let text = readTextAt(app: activeApp, x: x, y: y) else {
+        return Response(ok: false, error: ErrorPayload(message: "readText did not resolve text"))
+      }
+      return Response(ok: true, data: DataPayload(text: text))
     case .snapshot:
       let options = SnapshotOptions(
         interactiveOnly: command.interactiveOnly ?? false,
@@ -358,30 +514,15 @@ extension RunnerTests {
       if tapNavigationBack(app: activeApp) {
         return Response(ok: true, data: DataPayload(message: "back"))
       }
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "back button is not available on macOS"))
-#else
       performBackGesture(app: activeApp)
       return Response(ok: true, data: DataPayload(message: "back"))
-#endif
     case .home:
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "home is not supported on macOS"))
-#else
       pressHomeButton()
       return Response(ok: true, data: DataPayload(message: "home"))
-#endif
     case .appSwitcher:
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "appSwitcher is not supported on macOS"))
-#else
       performAppSwitcherGesture(app: activeApp)
       return Response(ok: true, data: DataPayload(message: "appSwitcher"))
-#endif
     case .alert:
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "alert is not supported on macOS"))
-#else
       let action = (command.action ?? "get").lowercased()
       let alert = activeApp.alerts.firstMatch
       if !alert.exists {
@@ -399,17 +540,21 @@ extension RunnerTests {
       }
       let buttonLabels = alert.buttons.allElementsBoundByIndex.map { $0.label }
       return Response(ok: true, data: DataPayload(message: alert.label, items: buttonLabels))
-#endif
     case .pinch:
-#if os(macOS)
-      return Response(ok: false, error: ErrorPayload(message: "pinch is not supported on macOS"))
-#else
       guard let scale = command.scale, scale > 0 else {
         return Response(ok: false, error: ErrorPayload(message: "pinch requires scale > 0"))
       }
-      pinch(app: activeApp, scale: scale, x: command.x, y: command.y)
-      return Response(ok: true, data: DataPayload(message: "pinched"))
-#endif
+      let timing = measureGesture {
+        pinch(app: activeApp, scale: scale, x: command.x, y: command.y)
+      }
+      return Response(
+        ok: true,
+        data: DataPayload(
+          message: "pinched",
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs
+        )
+      )
     }
   }
 }