agent-device 0.20.10 → 0.21.1

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -1,4 +1,6 @@
-// MARK: - Wire Models
+import AgentDeviceSnapshotPresentation
+
+
 
 enum CommandType: String, Codable {
   case tap
@@ -37,72 +39,72 @@ enum CommandType: String, Codable {
   case shutdown
 }
 
-/// Runner command traits — see CONTEXT.md ("Runner command traits").
-///
-/// Single source of truth for how the runner classifies a command across three
-/// independent axes, replacing the three hand-maintained switches that used to live
-/// in RunnerTests+Lifecycle.swift (isInteractionCommand / isReadOnlyCommand /
-/// isRunnerLifecycleCommand). The classification is load-bearing for ADR-0002 session
-/// invalidation: `readOnly` gates the retry that nulls currentApp/currentBundleId.
+
+
+
+
+
+
+
 struct CommandTraits {
-  /// Whether the command needs the foreground-guard + stabilization preflight before running.
+
   let isInteraction: Bool
-  /// Whether the command is eligible for the session-invalidating retry.
-  /// `.conditional` is resolved against the request (alert is read-only only for its `get` action).
+
+
   let readOnly: ReadOnly
-  /// Whether the command skips the app-activation preflight entirely.
+
   let isLifecycle: Bool
 
   enum ReadOnly {
     case always
     case never
-    /// Alert-only today. Resolved in `isReadOnlyCommand` with alert's rule (read-only for the
-    /// `get` action, mutating otherwise). A new `.conditional` command would inherit that rule
-    /// until the resolver is generalized — give it explicit handling there if its semantics differ.
+
+
+
     case conditional
   }
 }
 
 extension CommandType {
-  /// The classification for this command. Exhaustive by construction: a new CommandType
-  /// cannot compile without being classified here, so commands can no longer silently drift
-  /// out of classification the way the parallel switches allowed.
+
+
+
   var traits: CommandTraits {
     switch self {
-    // Interaction commands: require the foreground-guard + stabilization preflight.
-    // keyboardReturn is the sibling of keyboardDismiss (missing from the historical switch —
-    // drift the table now prevents). .scroll is the fused frame-resolve + drag scroll; same
-    // classification as .drag. .desktopScroll is the macOS frame-resolve + wheel event sibling.
-    // .sequence is the fused multi-step gesture batch.
+
+
+
+
+
     case .tap, .longPress, .drag, .remotePress, .type, .swipe, .scroll, .desktopScroll,
          .back, .backInApp, .backSystem, .rotate, .appSwitcher,
          .keyboardDismiss, .keyboardReturn, .sequence, .gesture:
       return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false)
 
-    // Read-only reads: eligible for the session-invalidating retry.
+
     case .findText, .readText, .snapshot, .gestureViewport:
       return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: false)
 
-    // Screenshot is both a read and a runner-lifecycle command (skips app-activation preflight).
+
     case .screenshot:
       return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: true)
 
-    // Alert is read-only only for its `get` action (resolved by isReadOnlyCommand).
+
     case .alert:
       return CommandTraits(isInteraction: false, readOnly: .conditional, isLifecycle: false)
 
-    // Runner-lifecycle commands: skip the app-activation preflight.
+
     case .recordStop, .uptime, .terminate, .targetReset, .shutdown:
       return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: true)
 
     case .status:
       return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: true)
 
-    // Normal preflight, not retried.
-    // NOTE: mouseClick stays non-interaction for now — it is macOS-only and the foreground
-    // guard interacts with bespoke macOS activation, so classifying it needs a macOS smoke
-    // check first (tracked as a follow-up). Also preserved: querySelector is NOT read-only;
-    // recordStart is NOT a lifecycle command; home/alert remain non-interaction by design.
+
+
+
+
+
     case .mouseClick, .querySelector, .home, .recordStart, .activate:
       return CommandTraits(isInteraction: false, readOnly: .never, isLifecycle: false)
     }
@@ -132,6 +134,7 @@ struct Command: Codable {
   let direction: String?
   let amount: Double?
   let pixels: Double?
+  let scrollReleaseBehavior: ScrollReleaseBehavior?
   let orientation: String?
   let gesturePlan: RunnerGesturePlan?
   let outPath: String?
@@ -148,7 +151,12 @@ struct Command: Codable {
   let steps: [SequenceStep]?
 }
 
-/// Canonical one- or two-pointer plan produced by the portable TypeScript planner.
+enum ScrollReleaseBehavior: String, Codable {
+  case controlled
+  case inertial
+}
+
+
 struct RunnerGesturePlan: Codable {
   let topology: String
   let intent: String
@@ -180,22 +188,22 @@ struct RunnerGesturePoint: Codable {
   let y: Double
 }
 
-/// One allowlisted coordinate gesture step inside a fused `sequence` command.
-/// `kind` is decoded as a raw String (not an enum) so the runner can return a clear
-/// INVALID_ARGS for an unknown kind instead of a generic decode failure.
+
+
+
 struct SequenceStep: Codable {
   let kind: String
   let x: Double?
   let y: Double?
   let durationMs: Double?
   let pauseMs: Double?
-  /// For `tap` steps on iOS non-tv: use the synthesized HID fast path instead of the
-  /// drag-based XCUICoordinate path, matching the individual command behavior.
+
+
   let synthesized: Bool?
 }
 
-/// Per-step result for a `sequence` response. `ok:false` carries the failing step's
-/// errorCode/errorMessage; execution stops at the first failed step.
+
+
 struct SequenceStepResult: Codable {
   let ok: Bool
   let kind: String
@@ -207,20 +215,14 @@ struct SequenceStepResult: Codable {
 
 struct Response: Codable {
   let ok: Bool
-  let data: DataPayload?
-  let error: ErrorPayload?
-
-  init(ok: Bool, data: DataPayload? = nil, error: ErrorPayload? = nil) {
-    self.ok = ok
-    self.data = data
-    self.error = error
-  }
+  var data: DataPayload?
+  var error: ErrorPayload?
 }
 
 extension Response {
-  // The daemon pairs this gesture-clock anchor with its own receipt time to map
-  // gesture uptimes onto wall-clock for the recording touch overlay. Error responses
-  // carry no anchor so the daemon falls back instead of pairing a stale value.
+
+
+
   func stampingCurrentUptimeMs(_ value: Double) -> Response {
     guard ok else { return self }
     var payload = data ?? DataPayload()
@@ -230,161 +232,84 @@ extension Response {
 }
 
 struct DataPayload: Codable {
-  let message: String?
-  let imageBase64: String?
-  let text: String?
-  let found: Bool?
-  let items: [String]?
-  let nodes: [PresentedNode]?
-  let truncated: Bool?
-  let snapshotQuality: SnapshotQuality?
-  let gestureStartUptimeMs: Double?
-  let gestureEndUptimeMs: Double?
-  let x: Double?
-  let y: Double?
-  let x2: Double?
-  let y2: Double?
-  let referenceWidth: Double?
-  let referenceHeight: Double?
+  var message: String?
+  var imageBase64: String?
+  var text: String?
+  var found: Bool?
+  var items: [String]?
+  var nodes: [PresentedNode]?
+  var truncated: Bool?
+  var qualityPayload: SnapshotQualityPayload? = nil
+  var snapshotQuality: SnapshotQuality?
+
+  var systemSurface: SystemSurfaceProvenancePayload?
+  var gestureStartUptimeMs: Double?
+  var gestureEndUptimeMs: Double?
+  var x: Double?
+  var y: Double?
+  var x2: Double?
+  var y2: Double?
+  var referenceWidth: Double?
+  var referenceHeight: Double?
   var currentUptimeMs: Double?
-  let commandId: String?
-  let lifecycleState: String?
-  let lifecycleCommand: String?
-  let lifecycleResponseOk: Bool?
-  let lifecycleResponseJson: String?
-  let lifecycleErrorCode: String?
-  let lifecycleErrorMessage: String?
-  let lifecycleErrorHint: String?
-  let visible: Bool?
-  let wasVisible: Bool?
-  let dismissed: Bool?
-  let keyboardDismissMechanism: String?
-  let orientation: String?
-  let gestureFallback: String?
-  let gestureFallbackMessage: String?
-  let gestureFallbackHint: String?
-  let maestroNonHittableCoordinateFallbackUsed: Bool?
-  let textEntryRoute: String?
-  let runnerFatal: Bool?
-  let runnerFatalReason: String?
-  let completedSteps: Int?
-  let failedStepIndex: Int?
-  let sequenceResults: [SequenceStepResult]?
-
-  init(
-    message: String? = nil,
-    imageBase64: String? = nil,
-    text: String? = nil,
-    found: Bool? = nil,
-    items: [String]? = nil,
-    nodes: [PresentedNode]? = nil,
-    truncated: Bool? = nil,
-    snapshotQuality: SnapshotQuality? = nil,
-    gestureStartUptimeMs: Double? = nil,
-    gestureEndUptimeMs: Double? = nil,
-    x: Double? = nil,
-    y: Double? = nil,
-    x2: Double? = nil,
-    y2: Double? = nil,
-    referenceWidth: Double? = nil,
-    referenceHeight: Double? = nil,
-    currentUptimeMs: Double? = nil,
-    commandId: String? = nil,
-    lifecycleState: String? = nil,
-    lifecycleCommand: String? = nil,
-    lifecycleResponseOk: Bool? = nil,
-    lifecycleResponseJson: String? = nil,
-    lifecycleErrorCode: String? = nil,
-    lifecycleErrorMessage: String? = nil,
-    lifecycleErrorHint: String? = nil,
-    visible: Bool? = nil,
-    wasVisible: Bool? = nil,
-    dismissed: Bool? = nil,
-    keyboardDismissMechanism: String? = nil,
-    orientation: String? = nil,
-    gestureFallback: String? = nil,
-    gestureFallbackMessage: String? = nil,
-    gestureFallbackHint: String? = nil,
-    maestroNonHittableCoordinateFallbackUsed: Bool? = nil,
-    textEntryRoute: String? = nil,
-    runnerFatal: Bool? = nil,
-    runnerFatalReason: String? = nil,
-    completedSteps: Int? = nil,
-    failedStepIndex: Int? = nil,
-    sequenceResults: [SequenceStepResult]? = nil
-  ) {
-    self.message = message
-    self.imageBase64 = imageBase64
-    self.text = text
-    self.found = found
-    self.items = items
+  var commandId: String?
+  var lifecycleState: String?
+  var lifecycleCommand: String?
+  var lifecycleResponseOk: Bool?
+  var lifecycleResponseJson: String?
+  var lifecycleErrorCode: String?
+  var lifecycleErrorMessage: String?
+  var lifecycleErrorHint: String?
+  var visible: Bool?
+  var wasVisible: Bool?
+  var dismissed: Bool?
+  var keyboardDismissMechanism: String?
+  var orientation: String?
+  var gestureFallback: String?
+  var gestureFallbackMessage: String?
+  var gestureFallbackHint: String?
+  var maestroNonHittableCoordinateFallbackUsed: Bool?
+  var textEntryRoute: String?
+  var runnerFatal: Bool?
+  var runnerFatalReason: String?
+  var completedSteps: Int?
+  var failedStepIndex: Int?
+  var sequenceResults: [SequenceStepResult]?
+}
+
+
+struct SystemSurfaceProvenancePayload: Codable {
+  let bundleId: String
+  let kind: String
+}
+
+struct SnapshotQualityPayload: Codable {
+  let nodes: [PresentedNode]
+  let truncated: Bool
+  let scope: String?
+
+  init(nodes: [PresentedNode], truncated: Bool) {
     self.nodes = nodes
     self.truncated = truncated
-    self.snapshotQuality = snapshotQuality
-    self.gestureStartUptimeMs = gestureStartUptimeMs
-    self.gestureEndUptimeMs = gestureEndUptimeMs
-    self.x = x
-    self.y = y
-    self.x2 = x2
-    self.y2 = y2
-    self.referenceWidth = referenceWidth
-    self.referenceHeight = referenceHeight
-    self.currentUptimeMs = currentUptimeMs
-    self.commandId = commandId
-    self.lifecycleState = lifecycleState
-    self.lifecycleCommand = lifecycleCommand
-    self.lifecycleResponseOk = lifecycleResponseOk
-    self.lifecycleResponseJson = lifecycleResponseJson
-    self.lifecycleErrorCode = lifecycleErrorCode
-    self.lifecycleErrorMessage = lifecycleErrorMessage
-    self.lifecycleErrorHint = lifecycleErrorHint
-    self.visible = visible
-    self.wasVisible = wasVisible
-    self.dismissed = dismissed
-    self.keyboardDismissMechanism = keyboardDismissMechanism
-    self.orientation = orientation
-    self.gestureFallback = gestureFallback
-    self.gestureFallbackMessage = gestureFallbackMessage
-    self.gestureFallbackHint = gestureFallbackHint
-    self.maestroNonHittableCoordinateFallbackUsed = maestroNonHittableCoordinateFallbackUsed
-    self.textEntryRoute = textEntryRoute
-    self.runnerFatal = runnerFatal
-    self.runnerFatalReason = runnerFatalReason
-    self.completedSteps = completedSteps
-    self.failedStepIndex = failedStepIndex
-    self.sequenceResults = sequenceResults
+    self.scope = nil
   }
-}
-
-struct ErrorPayload: Codable {
-  let code: String?
-  let message: String
-  let hint: String?
 
-  init(code: String? = nil, message: String, hint: String? = nil) {
-    self.code = code
-    self.message = message
-    self.hint = hint
+  private enum CodingKeys: String, CodingKey {
+    case nodes
+    case truncated
+    case scope
   }
-}
 
-struct SnapshotRect: Codable {
-  let x: Double
-  let y: Double
-  let width: Double
-  let height: Double
+  func encode(to encoder: Encoder) throws {
+    var container = encoder.container(keyedBy: CodingKeys.self)
+    try container.encode(nodes, forKey: .nodes)
+    try container.encode(truncated, forKey: .truncated)
+    try container.encodeNil(forKey: .scope)
+  }
 }
 
-struct PresentationOptions {
-  let interactiveOnly: Bool
-  let depth: Int?
-  let scope: String?
-  let raw: Bool
-  /// Internal daemon ask: capture with this backend first regardless of channel
-  /// health ("private-ax"). Same-backend evidence probes (tap-outcome
-  /// corroboration) must be captured the way their baseline was.
-  var preferredBackend: String? = nil
-  /// Read UIAccessibilityCustomActions for merged leaves. Opt-in: each element
-  /// costs its own AX round trip (see RunnerAXSnapshotBridge).
-  var customActions: Bool = false
+struct ErrorPayload: Codable {
+  var code: String?
+  let message: String
+  var hint: String?
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -81,7 +81,7 @@ extension RunnerTests {
     return navigationBackKeywords.firstIndex { text.contains($0) }
   }
 
-  // isFinite/>0 alone don't reject CGRect.infinite — its origin (~-9e307) is finite.
+
   static func isUsableNavigationFrame(_ frame: CGRect) -> Bool {
     guard frame.width.isFinite, frame.height.isFinite, frame.width > 0, frame.height > 0 else {
       return false
@@ -93,7 +93,7 @@ extension RunnerTests {
     guard isUsableNavigationFrame(candidate), isUsableNavigationFrame(window) else {
       return false
     }
-    // Accept the compact navigation/search header band without matching deep content controls.
+
     let maxY = window.minY + min(max(window.height * 0.22, 96), 180)
     return candidate.midY >= window.minY && candidate.midY <= maxY
   }
@@ -102,9 +102,9 @@ extension RunnerTests {
     guard isUsableNavigationFrame(frame) else {
       return nil
     }
-    // Aim at the standard leading navigation slot, bounded for compact and tablet widths.
+
     let xOffset = min(max(frame.width * 0.08, 28), 44)
-    // Sit below the status/dynamic-island region and inside common custom RN search headers.
+
     let yOffset = min(max(frame.height * 0.155, 56), 132)
     return CGPoint(x: frame.minX + xOffset, y: frame.minY + yOffset)
   }
@@ -169,4 +169,71 @@ extension RunnerTests {
     return element.exists ? element : nil
   }
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXElementTypes.swift

@@ -0,0 +1,12 @@
+import XCTest
+
+extension RunnerTests {
+  func privateAXElementType(rawElementType: Int) -> XCUIElement.ElementType? {
+    guard let raw = UInt(exactly: rawElementType),
+      let type = XCUIElement.ElementType(rawValue: raw)
+    else {
+      return nil
+    }
+    return type
+  }
+}