agent-device 0.16.6 → 0.16.8

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -34,6 +34,74 @@ enum CommandType: String, Codable {
   case shutdown
 }
 
+/// Runner command traits — see CONTEXT.md ("Runner command traits").
+///
+/// Single source of truth for how the runner classifies a command across three
+/// independent axes, replacing the three hand-maintained switches that used to live
+/// in RunnerTests+Lifecycle.swift (isInteractionCommand / isReadOnlyCommand /
+/// isRunnerLifecycleCommand). The classification is load-bearing for ADR-0002 session
+/// invalidation: `readOnly` gates the retry that nulls currentApp/currentBundleId.
+struct CommandTraits {
+  /// Whether the command needs the foreground-guard + stabilization preflight before running.
+  let isInteraction: Bool
+  /// Whether the command is eligible for the session-invalidating retry.
+  /// `.conditional` is resolved against the request (alert is read-only only for its `get` action).
+  let readOnly: ReadOnly
+  /// Whether the command skips the app-activation preflight entirely.
+  let isLifecycle: Bool
+
+  enum ReadOnly {
+    case always
+    case never
+    /// Alert-only today. Resolved in `isReadOnlyCommand` with alert's rule (read-only for the
+    /// `get` action, mutating otherwise). A new `.conditional` command would inherit that rule
+    /// until the resolver is generalized — give it explicit handling there if its semantics differ.
+    case conditional
+  }
+}
+
+extension CommandType {
+  /// The classification for this command. Exhaustive by construction: a new CommandType
+  /// cannot compile without being classified here, so commands can no longer silently drift
+  /// out of classification the way the parallel switches allowed.
+  var traits: CommandTraits {
+    switch self {
+    // Interaction commands: require the foreground-guard + stabilization preflight.
+    // tapSeries/dragSeries are the series forms of tap/drag; keyboardReturn is the sibling
+    // of keyboardDismiss — all three were missing from the historical switch (drift the
+    // table now prevents) and are classified as interactions here.
+    case .tap, .tapSeries, .longPress, .drag, .dragSeries, .remotePress, .type, .swipe,
+         .back, .backInApp, .backSystem, .rotate, .appSwitcher,
+         .keyboardDismiss, .keyboardReturn, .pinch, .rotateGesture, .transformGesture:
+      return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false)
+
+    // Read-only reads: eligible for the session-invalidating retry.
+    case .interactionFrame, .findText, .readText, .snapshot:
+      return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: false)
+
+    // Screenshot is both a read and a runner-lifecycle command (skips app-activation preflight).
+    case .screenshot:
+      return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: true)
+
+    // Alert is read-only only for its `get` action (resolved by isReadOnlyCommand).
+    case .alert:
+      return CommandTraits(isInteraction: false, readOnly: .conditional, isLifecycle: false)
+
+    // Runner-lifecycle commands: skip the app-activation preflight.
+    case .recordStop, .uptime, .shutdown:
+      return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: true)
+
+    // Normal preflight, not retried.
+    // NOTE: mouseClick stays non-interaction for now — it is macOS-only and the foreground
+    // guard interacts with bespoke macOS activation, so classifying it needs a macOS smoke
+    // check first (tracked as a follow-up). Also preserved: querySelector is NOT read-only;
+    // recordStart is NOT a lifecycle command; home/alert remain non-interaction by design.
+    case .mouseClick, .querySelector, .home, .recordStart:
+      return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: false)
+    }
+  }
+}
+
 struct Command: Codable {
   let command: CommandType
   let appBundleId: String?
@@ -154,10 +222,12 @@ struct DataPayload: Codable {
 struct ErrorPayload: Codable {
   let code: String?
   let message: String
+  let hint: String?
 
-  init(code: String? = nil, message: String) {
+  init(code: String? = nil, message: String, hint: String? = nil) {
     self.code = code
     self.message = message
+    self.hint = hint
   }
 }
 

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -1,6 +1,9 @@
 import XCTest
 
 extension RunnerTests {
+  private static let axSnapshotErrorCode = "IOS_AX_SNAPSHOT_FAILED"
+  private static let axSnapshotHint =
+    "XCTest could not serialize this iOS accessibility tree. Try a smaller read such as snapshot -s <visible label or id> -d 8, use direct selector commands such as find id <value> click, or use screenshot/logs/appstate in the same session. If you own the app and need full-tree inspection, consider flagging this screen for accessibility-tree simplification: reduce unnecessary accessible wrapper nesting and expose stable ids on actionable controls."
   private static let collapsedTabCandidateTypes: Set<XCUIElement.ElementType> = [
     .button,
     .link,
@@ -33,6 +36,12 @@ extension RunnerTests {
     let visible: Bool
   }
 
+  struct SnapshotCaptureFailure: Error {
+    let code: String
+    let message: String
+    let hint: String
+  }
+
   // MARK: - Snapshot Entry
 
   func elementTypeName(_ type: XCUIElement.ElementType) -> String {
@@ -75,12 +84,12 @@ extension RunnerTests {
     }
   }
 
-  func snapshotFast(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
+  func snapshotFast(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload {
     if let blocking = blockingSystemAlertSnapshot() {
       return blocking
     }
 
-    guard let context = makeSnapshotTraversalContext(app: app, options: options) else {
+    guard let context = try makeSnapshotTraversalContext(app: app, options: options) else {
       return DataPayload(nodes: [], truncated: false)
     }
 
@@ -186,12 +195,12 @@ extension RunnerTests {
     return DataPayload(nodes: nodes, truncated: truncated)
   }
 
-  func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
+  func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload {
     if let blocking = blockingSystemAlertSnapshot() {
       return blocking
     }
 
-    guard let context = makeSnapshotTraversalContext(app: app, options: options) else {
+    guard let context = try makeSnapshotTraversalContext(app: app, options: options) else {
       return DataPayload(nodes: [], truncated: false)
     }
 
@@ -304,14 +313,11 @@ extension RunnerTests {
   private func makeSnapshotTraversalContext(
     app: XCUIApplication,
     options: SnapshotOptions
-  ) -> SnapshotTraversalContext? {
-    let viewport = snapshotViewport(app: app)
+  ) throws -> SnapshotTraversalContext? {
+    let viewport = safeSnapshotViewport(app: app)
     let queryRoot = options.scope.flatMap { findScopeElement(app: app, scope: $0) } ?? app
 
-    let rootSnapshot: XCUIElementSnapshot
-    do {
-      rootSnapshot = try queryRoot.snapshot()
-    } catch {
+    guard let rootSnapshot = try captureSnapshotRoot(queryRoot) else {
       return nil
     }
 
@@ -326,6 +332,70 @@ extension RunnerTests {
     )
   }
 
+  private func captureSnapshotRoot(_ element: XCUIElement) throws -> XCUIElementSnapshot? {
+    var rootSnapshot: XCUIElementSnapshot?
+    var swiftErrorMessage: String?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      do {
+        rootSnapshot = try element.snapshot()
+      } catch {
+        swiftErrorMessage = describeSnapshotError(error)
+      }
+    })
+
+    if let rootSnapshot {
+      return rootSnapshot
+    }
+    let message = exceptionMessage ?? swiftErrorMessage ?? "snapshot returned no root"
+    if Self.isAxIllegalArgument(message) {
+      throw axSnapshotFailure(message)
+    }
+    return nil
+  }
+
+  private func safeSnapshotViewport(app: XCUIApplication) -> CGRect {
+    var viewport = CGRect.infinite
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      viewport = snapshotViewport(app: app)
+    })
+    if let exceptionMessage {
+      NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_VIEWPORT_IGNORED_EXCEPTION=%@", exceptionMessage)
+    }
+    return viewport
+  }
+
+  private func describeSnapshotError(_ error: Error) -> String {
+    let localized = error.localizedDescription
+    let debug = String(describing: error)
+    if localized.isEmpty { return debug }
+    if debug == localized { return localized }
+    return "\(localized) (\(debug))"
+  }
+
+  private func axSnapshotFailure(_ message: String) -> SnapshotCaptureFailure {
+    let failureMessage: String
+    if Self.hasAxIllegalArgumentCode(message) {
+      failureMessage = "iOS XCTest snapshot failed with kAXErrorIllegalArgument. \(message)"
+    } else {
+      failureMessage = "iOS XCTest snapshot failed while serializing the accessibility tree. \(message)"
+    }
+    return SnapshotCaptureFailure(
+      code: Self.axSnapshotErrorCode,
+      message: failureMessage,
+      hint: Self.axSnapshotHint
+    )
+  }
+
+  private static func isAxIllegalArgument(_ message: String) -> Bool {
+    let normalized = message.lowercased()
+    return hasAxIllegalArgumentCode(normalized)
+      || (normalized.contains("illegal argument") && normalized.contains("snapshot"))
+  }
+
+  private static func hasAxIllegalArgumentCode(_ message: String) -> Bool {
+    return message.lowercased().contains("kaxerrorillegalargument")
+  }
+
   private func evaluateSnapshot(
     _ snapshot: XCUIElementSnapshot,
     in context: SnapshotTraversalContext

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift

@@ -2,7 +2,9 @@ import XCTest
 
 enum RunnerInteractionOutcome {
   case performed
-  case unsupported(String)
+  /// A capability/state gap, surfaced to the caller as an UNSUPPORTED_OPERATION error.
+  /// `hint` is an optional actionable next step (mapped to ErrorPayload.hint).
+  case unsupported(message: String, hint: String?)
 }
 
 enum TvRemoteButton {
@@ -85,7 +87,10 @@ extension RunnerTests {
   func selectFocusedTvElement(app: XCUIApplication, point: CGPoint, action: String) -> RunnerInteractionOutcome? {
 #if os(tvOS)
     guard let focused = focusedTvElement(app: app), !focused.frame.isEmpty, focused.frame.contains(point) else {
-      return .unsupported("\(action) is supported on tvOS only when the requested point is inside the focused element")
+      return .unsupported(
+        message: "\(action) is supported on tvOS only when the requested point is inside the focused element",
+        hint: "Move focus with swipe or scroll until the target is focused, then retry."
+      )
     }
     _ = pressTvRemote(.select)
     return .performed
@@ -97,7 +102,10 @@ extension RunnerTests {
   func longSelectFocusedTvElement(app: XCUIApplication, point: CGPoint, duration: TimeInterval) -> RunnerInteractionOutcome? {
 #if os(tvOS)
     guard let focused = focusedTvElement(app: app), !focused.frame.isEmpty, focused.frame.contains(point) else {
-      return .unsupported("long press is supported on tvOS only when the requested point is inside the focused element")
+      return .unsupported(
+        message: "long press is supported on tvOS only when the requested point is inside the focused element",
+        hint: "Move focus with swipe or scroll until the target is focused, then retry."
+      )
     }
     _ = pressTvRemote(.select, duration: duration)
     return .performed
@@ -108,17 +116,37 @@ extension RunnerTests {
 
   private func performElementTap(_ element: XCUIElement) -> RunnerInteractionOutcome {
 #if os(tvOS)
-    return .unsupported("element tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
+    return .unsupported(
+      message: "element tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",
+      hint: "Use swipe/scroll to move focus to the target, then select it; tvOS has no coordinate tap."
+    )
 #else
-    element.tap()
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      element.tap()
+    })
+    if let exceptionMessage {
+      NSLog("AGENT_DEVICE_RUNNER_ELEMENT_TAP_IGNORED_EXCEPTION=%@", exceptionMessage)
+      if isPostTapElementDisappearance(exceptionMessage) {
+        return .performed
+      }
+      return .unsupported(message: "element tap failed: \(exceptionMessage)", hint: nil)
+    }
     return .performed
 #endif
   }
 
+  private func isPostTapElementDisappearance(_ message: String) -> Bool {
+    message.contains("No matches found")
+      || message.contains("Failed to get matching snapshot")
+  }
+
   private func selectFocusedTvElement(app: XCUIApplication, element: XCUIElement, action: String) -> RunnerInteractionOutcome? {
 #if os(tvOS)
     guard tvFocusedElementMatches(app: app, target: element) else {
-      return .unsupported("\(action) is supported on tvOS only when the requested element is focused")
+      return .unsupported(
+        message: "\(action) is supported on tvOS only when the requested element is focused",
+        hint: "Move focus to the target element first, then retry."
+      )
     }
     _ = pressTvRemote(.select)
     return .performed

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-device",
-  "version": "0.16.6",
+  "version": "0.16.8",
   "description": "Agent-native CLI for AI mobile testing and app automation across iOS, Android, tvOS, Android TV, macOS, and Linux.",
   "mcpName": "io.github.callstackincubator/agent-device",
   "license": "MIT",
@@ -98,6 +98,9 @@
     "ad": "node bin/agent-device.mjs",
     "size": "node scripts/size-report.mjs",
     "size:markdown": "node scripts/size-report.mjs --json .tmp/size-report.json --markdown .tmp/size-report.md",
+    "perf": "node --experimental-strip-types scripts/perf/run.ts",
+    "perf:ios": "node --experimental-strip-types scripts/perf/run.ts --platform ios",
+    "perf:android": "node --experimental-strip-types scripts/perf/run.ts --platform android",
     "lint": "oxlint . --deny-warnings",
     "format": "oxfmt --write src test skills package.json tsconfig.json tsconfig.lib.json rslib.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
     "fallow": "fallow --summary",
@@ -160,11 +163,6 @@
     "README.md",
     "LICENSE"
   ],
-  "pnpm": {
-    "overrides": {
-      "lodash-es": "4.18.1"
-    }
-  },
   "keywords": [
     "agent",
     "device",

package/server.json

@@ -7,12 +7,12 @@
     "url": "https://github.com/callstackincubator/agent-device",
     "source": "github"
   },
-  "version": "0.16.6",
+  "version": "0.16.8",
   "packages": [
     {
       "registryType": "npm",
       "identifier": "agent-device",
-      "version": "0.16.6",
+      "version": "0.16.8",
       "transport": {
         "type": "stdio"
       }

package/android-multitouch-helper/dist/agent-device-android-multitouch-helper-0.16.6.apk.sha256

@@ -1 +0,0 @@
-fc975087b6d0ce63e25cb5d9475f896e828222ec4d7ed12d73e4045cb7cc6c5a  agent-device-android-multitouch-helper-0.16.6.apk

package/android-snapshot-helper/dist/agent-device-android-snapshot-helper-0.16.6.apk.sha256

@@ -1 +0,0 @@
-a6a8555dda194d1e882a930bc6ccf555ff48f1645faf92968a0832f864c91661  agent-device-android-snapshot-helper-0.16.6.apk

package/dist/src/5186.js

@@ -1 +0,0 @@
-let e={devices:"List available devices.",boot:"Boot or prepare a selected device without using CLI positional arguments.",apps:"List installed apps.",session:"List active sessions.",open:"Open an app, deep link, URL, or platform surface.",close:"Close an app or end the active session.",install:"Install an app binary.",reinstall:"Reinstall an app binary.","install-from-source":"Install an app from a structured source.",push:"Deliver a push payload.","trigger-app-event":"Trigger an app-defined event.",snapshot:"Capture an accessibility snapshot.",screenshot:"Capture a screenshot.",diff:"Diff accessibility snapshots.",wait:"Wait for duration, text, ref, or selector.",alert:"Inspect or handle platform alerts.",appstate:"Show foreground app or activity.",back:"Navigate back.",home:"Go to the home screen.",rotate:"Rotate device orientation.","app-switcher":"Open the app switcher.",keyboard:"Inspect or dismiss the keyboard.",clipboard:"Read or write clipboard text.","react-native":"Run supported React Native app automation helpers.",replay:"Replay a recorded session.",test:"Run one or more replay scripts.",perf:"Show session performance metrics.",logs:"Manage session app logs.",network:"Show recent HTTP traffic.",record:"Start or stop screen recording.",trace:"Start or stop trace capture.",settings:"Change OS settings and app permissions.",metro:"Prepare Metro runtime or reload React Native apps.",click:"Click or tap a semantic UI target by ref, selector, or point.",press:"Press a semantic UI target by ref, selector, or point.",fill:"Fill text into a semantic UI target by ref, selector, or point.",longpress:"Long press by ref, selector, or point.",swipe:"Swipe between two points.",focus:"Focus input at coordinates.",type:"Type text in the focused field.",scroll:"Scroll in a direction or to an edge.",get:"Get element text or attributes.",is:"Assert UI state.",find:"Find an element and optionally act on it.",gesture:"Run a structured gesture.",batch:"Run multiple structured command steps in one daemon request."};function t(t){let r=e[t];if(!r)throw Error(`Missing command description for ${t}`);return r}function r(){return Object.entries(e).map(([e,t])=>({name:e,description:t}))}export{r as listCommandDescriptionMetadata,t as requireCommandDescription};