agent-device 0.21.6 → 0.21.8

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -12,10 +12,24 @@ extension RunnerTests {
   private static let rawSnapshotMaxNodes = 5_000
   private static let rawSnapshotTooLargeHint =
     "Raw iOS snapshot exceeded the runner payload guard. Use regular snapshot for visible UI, or scope/depth-limit raw snapshot when inspecting a large accessibility tree."
+
+
+  private static let regularSnapshotMaxNodes = 50_000
+  private static let regularSnapshotTooLargeCode = "IOS_SNAPSHOT_TOO_LARGE"
+  private static let regularSnapshotTooLargeHint =
+    "iOS snapshot walked an unexpectedly large accessibility tree. Scope the snapshot to a subtree or use screenshot."
   struct SnapshotTraversalContext {
     let queryRoot: XCUIElement
     let rootSnapshot: XCUIElementSnapshot
     let viewport: CGRect
+
+    let interfaceOrientation: Int
+
+
+
+
+
+    let keyboardBand: RunnerKeyboardBandFact?
   }
 
   private struct SnapshotEvaluation {
@@ -30,8 +44,16 @@ extension RunnerTests {
     let snapshot: XCUIElementSnapshot
     let depth: Int
     let parentIndex: Int?
-    let parentPresentedDepth: Int
-    let parentTraversal: SnapshotVisibilityFold.TraversalState
+  }
+
+
+
+
+
+
+  static func canDescendAtRawDepth(_ depth: Int, hint: CaptureHint) -> Bool {
+    guard let rawLimit = hint.rawTraversalDepth else { return true }
+    return depth < rawLimit
   }
 
   struct SnapshotCaptureFailure: Error {
@@ -116,6 +138,13 @@ extension RunnerTests {
 
 
 
+  static let keyboardBandProbeBudget: TimeInterval = 0.3
+
+
+
+
+
+
   func snapshotFast(app: XCUIApplication, options: PresentationOptions) throws -> DataPayload {
     let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget)
     if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) {
@@ -133,7 +162,7 @@ extension RunnerTests {
   func recursiveTreeSnapshotAcquisition(
     context: SnapshotTraversalContext,
     hint: CaptureHint
-  ) -> SnapshotAcquisition {
+  ) throws -> SnapshotAcquisition {
     var cachedDescendantElements: [XCUIElement]?
     func collapsedTabDescendants() -> [XCUIElement] {
       if let cachedDescendantElements {
@@ -149,54 +178,37 @@ extension RunnerTests {
 
 
 
-
-
     var nodes: [RawAXNode] = []
-    let rootEvaluation = evaluateSnapshot(context.rootSnapshot)
     nodes.append(
       makeSnapshotNode(
         snapshot: context.rootSnapshot,
-        evaluation: rootEvaluation,
+        evaluation: evaluateSnapshot(context.rootSnapshot),
         depth: 0,
         index: 0,
-        parentIndex: nil,
-        viewport: context.viewport
+        parentIndex: nil
       )
     )
-    let shouldVisitRootChildren = SnapshotPresentation.shouldAcquireChildren(
-      for: hint,
-      rawDepth: 0,
-      regularPresentedDepth: 0
-    )
-    if shouldVisitRootChildren {
+    if Self.canDescendAtRawDepth(0, hint: hint) {
       appendCollapsedTabFallbackNodes(
         to: &nodes,
         containerSnapshot: context.rootSnapshot,
         resolveElements: collapsedTabDescendants,
         depth: 1,
-        parentIndex: 0,
-        viewport: context.viewport
+        parentIndex: 0
       )
     }
 
     var seen = Set<String>()
     var stack: [SnapshotTraversalEntry] = []
-    if shouldVisitRootChildren {
+    if Self.canDescendAtRawDepth(0, hint: hint) {
       stack = context.rootSnapshot.children.map {
-        SnapshotTraversalEntry(
-          snapshot: $0,
-          depth: 1,
-          parentIndex: 0,
-          parentPresentedDepth: 0,
-          parentTraversal: .root
-        )
+        SnapshotTraversalEntry(snapshot: $0, depth: 1, parentIndex: 0)
       }
     }
 
     while let entry = stack.popLast() {
       let snapshot = entry.snapshot
       let depth = entry.depth
-      let parentIndex = entry.parentIndex
       if let limit = hint.rawTraversalDepth, depth > limit { continue }
 
       let evaluation = evaluateSnapshot(snapshot)
@@ -205,8 +217,7 @@ extension RunnerTests {
         evaluation: evaluation,
         depth: depth,
         index: nodes.count,
-        parentIndex: parentIndex,
-        viewport: context.viewport
+        parentIndex: entry.parentIndex
       )
       let key = Self.snapshotTraversalIdentity(
         elementType: snapshot.elementType,
@@ -219,43 +230,28 @@ extension RunnerTests {
         seen.insert(key)
       }
 
-      let currentIndex = !isDuplicate ? nodes.count : parentIndex
-      let transition = SnapshotPresentation.regularTraversalTransition(
-        for: node,
-        parentPresentedDepth: entry.parentPresentedDepth,
-        parentTraversal: entry.parentTraversal,
-        hint: hint,
-        rawDepth: depth,
-        viewport: context.viewport,
-        hasChildren: !snapshot.children.isEmpty,
-        isDuplicate: isDuplicate,
-        policy: .platformDefault
-      )
-      if transition.shouldVisitChildren {
-        for child in snapshot.children.reversed() {
-          stack.append(
-            SnapshotTraversalEntry(
-              snapshot: child,
-              depth: depth + 1,
-              parentIndex: currentIndex,
-              parentPresentedDepth: transition.presentedDepth,
-              parentTraversal: transition.traversal
-            )
-          )
-        }
+
+
+      let currentIndex = isDuplicate ? entry.parentIndex : nodes.count
+      for child in snapshot.children.reversed() {
+        stack.append(
+          SnapshotTraversalEntry(snapshot: child, depth: depth + 1, parentIndex: currentIndex)
+        )
       }
 
       if isDuplicate { continue }
 
       nodes.append(node)
-      if transition.shouldVisitChildren {
+      if nodes.count > Self.regularSnapshotMaxNodes {
+        throw regularSnapshotTooLargeFailure(nodeCount: nodes.count)
+      }
+      if Self.canDescendAtRawDepth(depth, hint: hint) {
         appendCollapsedTabFallbackNodes(
           to: &nodes,
           containerSnapshot: snapshot,
           resolveElements: collapsedTabDescendants,
           depth: depth + 1,
-          parentIndex: node.index,
-          viewport: context.viewport
+          parentIndex: node.index
         )
       }
     }
@@ -265,7 +261,8 @@ extension RunnerTests {
       nodes: nodes,
       truncated: false,
       effectiveDepth: nil,
-      viewport: context.viewport
+      viewport: context.viewport,
+      interfaceOrientation: context.interfaceOrientation
     )
   }
 
@@ -366,7 +363,11 @@ extension RunnerTests {
   ) throws -> SnapshotAcquisition {
     var nodes: [RawAXNode] = []
 
-    func walk(_ snapshot: XCUIElementSnapshot, depth: Int, parentIndex: Int?) throws {
+    func walk(
+      _ snapshot: XCUIElementSnapshot,
+      depth: Int,
+      parentIndex: Int?
+    ) throws {
       if let limit = hint.rawTraversalDepth, depth > limit { return }
 
       let evaluation = evaluateSnapshot(snapshot)
@@ -380,24 +381,32 @@ extension RunnerTests {
           evaluation: evaluation,
           depth: depth,
           index: currentIndex,
-          parentIndex: parentIndex,
-          viewport: context.viewport
+          parentIndex: parentIndex
         )
       )
 
       let children = snapshot.children
       for child in children {
-        try walk(child, depth: depth + 1, parentIndex: currentIndex)
+        try walk(
+          child,
+          depth: depth + 1,
+          parentIndex: currentIndex
+        )
       }
     }
 
-    try walk(context.rootSnapshot, depth: 0, parentIndex: nil)
+    try walk(
+      context.rootSnapshot,
+      depth: 0,
+      parentIndex: nil
+    )
     return SnapshotAcquisition(
       hint: hint,
       nodes: nodes,
       truncated: false,
       effectiveDepth: nil,
-      viewport: context.viewport
+      viewport: context.viewport,
+      interfaceOrientation: context.interfaceOrientation
     )
   }
 
@@ -415,7 +424,8 @@ extension RunnerTests {
         nodes: nodes,
         truncated: false,
         effectiveDepth: nil,
-        viewport: .infinite
+        viewport: .infinite,
+        interfaceOrientation: RunnerInterfaceOrientation.unknown
       )
     }
 
@@ -436,12 +446,7 @@ extension RunnerTests {
         truncated = true
         break
       }
-      guard let node = flatSnapshotNode(
-        element: element,
-        index: 0,
-        parentIndex: 0,
-        viewport: viewport
-      ) else {
+      guard let node = flatSnapshotNode(element: element, index: 0, parentIndex: 0) else {
         continue
       }
       let key = "\(node.type)-\(node.label ?? "")-\(node.identifier ?? "")-\(node.value ?? "")-\(node.rect.x)-\(node.rect.y)-\(node.rect.width)-\(node.rect.height)"
@@ -491,7 +496,8 @@ extension RunnerTests {
       nodes: nodes,
       truncated: truncated,
       effectiveDepth: nil,
-      viewport: viewport
+      viewport: viewport,
+      interfaceOrientation: RunnerInterfaceOrientation.unknown
     )
   }
 
@@ -529,6 +535,14 @@ extension RunnerTests {
     )
   }
 
+  private func regularSnapshotTooLargeFailure(nodeCount: Int) -> SnapshotCaptureFailure {
+    SnapshotCaptureFailure(
+      code: Self.regularSnapshotTooLargeCode,
+      message: "iOS snapshot exceeded \(Self.regularSnapshotMaxNodes) nodes while walking node \(nodeCount).",
+      hint: Self.regularSnapshotTooLargeHint
+    )
+  }
+
   func sparseTruncatedSnapshotPayload(
     message: String? = nil,
     snapshotQuality: SnapshotQuality? = nil,
@@ -793,7 +807,7 @@ extension RunnerTests {
       label: nil,
       identifier: nil,
       value: nil,
-      rect: snapshotRect(from: rect),
+      rect: SnapshotRect(rect),
       enabled: true,
       focused: nil,
       selected: nil,
@@ -814,15 +828,6 @@ extension RunnerTests {
     return CGRect(x: 0, y: 0, width: max(1, maxX), height: max(1, maxY))
   }
 
-  func snapshotRect(from frame: CGRect) -> SnapshotRect {
-    return SnapshotRect(
-      x: Double(frame.origin.x),
-      y: Double(frame.origin.y),
-      width: Double(frame.size.width),
-      height: Double(frame.size.height)
-    )
-  }
-
 
 
   func makeSnapshotTraversalContext(
@@ -831,24 +836,40 @@ extension RunnerTests {
     captureDeadline: Date = .distantFuture,
     treeCaptureSliceBudgetOverride: TimeInterval? = nil
   ) throws -> SnapshotTraversalContext? {
-    let viewport = try runMainThreadWork(
+
+
+
+    let geometry = try runMainThreadWork(
       "snapshot_viewport",
       timeout: min(1.0, max(0.1, captureDeadline.timeIntervalSinceNow)),
       timeoutError: snapshotMainThreadTimeoutError("preparing tree snapshot")
     ) {
-      self.safeSnapshotViewport(app: app)
+      (
+        viewport: self.safeSnapshotViewport(app: app),
+        interfaceOrientation: self.capturedInterfaceOrientation(app: app)
+      )
     }
-
+    let viewport = geometry.viewport
+    let interfaceOrientation = geometry.interfaceOrientation
     let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget
     let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow))
     guard let rootSnapshot = try captureSnapshotRootBounded(app, sliceSeconds: slice) else {
       return nil
     }
 
+
+
+
+
+
+    let keyboardBand = captureKeyboardBandFact(app: app, deadline: captureDeadline)
+
     return SnapshotTraversalContext(
       queryRoot: app,
       rootSnapshot: rootSnapshot,
-      viewport: viewport
+      viewport: viewport,
+      interfaceOrientation: interfaceOrientation,
+      keyboardBand: keyboardBand
     )
   }
 
@@ -972,24 +993,21 @@ extension RunnerTests {
     evaluation: SnapshotEvaluation,
     depth: Int,
     index: Int,
-    parentIndex: Int?,
-    viewport: CGRect
+    parentIndex: Int?
   ) -> RawAXNode {
+
+
     return RawAXNode(
       index: index,
       type: elementTypeName(snapshot.elementType),
       label: evaluation.label.isEmpty ? nil : evaluation.label,
       identifier: evaluation.identifier.isEmpty ? nil : evaluation.identifier,
       value: evaluation.valueText,
-      rect: snapshotRect(from: snapshot.frame),
+      rect: SnapshotRect(snapshot.frame),
       enabled: snapshot.isEnabled,
       focused: evaluation.focused ? true : nil,
       selected: evaluation.selected ? true : nil,
-      hittable: parentIndex != nil && SnapshotGeometry.isGeometricallyActionable(
-        enabled: snapshot.isEnabled,
-        frame: snapshot.frame,
-        viewport: viewport
-      ),
+      hittable: false,
       depth: depth,
       parentIndex: parentIndex,
       hiddenContentAbove: nil,
@@ -997,6 +1015,14 @@ extension RunnerTests {
     )
   }
 
+
+
+  func capturedInterfaceOrientation(app: XCUIApplication) -> Int {
+    safely("SNAPSHOT_INTERFACE_ORIENTATION", RunnerInterfaceOrientation.unknown) {
+      Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
+    }
+  }
+
   private func snapshotValueText(_ snapshot: XCUIElementSnapshot) -> String? {
     guard let value = snapshot.value else { return nil }
     let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
@@ -1042,16 +1068,14 @@ extension RunnerTests {
     containerSnapshot: XCUIElementSnapshot,
     resolveElements: () -> [XCUIElement],
     depth: Int,
-    parentIndex: Int,
-    viewport: CGRect
+    parentIndex: Int
   ) {
     let fallbackNodes = collapsedTabFallbackNodes(
       for: containerSnapshot,
       resolveElements: resolveElements,
       startingIndex: nodes.count,
       depth: depth,
-      parentIndex: parentIndex,
-      viewport: viewport
+      parentIndex: parentIndex
     )
     nodes.append(contentsOf: fallbackNodes)
   }
@@ -1061,10 +1085,13 @@ extension RunnerTests {
     resolveElements: () -> [XCUIElement],
     startingIndex: Int,
     depth: Int,
-    parentIndex: Int,
-    viewport: CGRect
+    parentIndex: Int
   ) -> [RawAXNode] {
     if !containerSnapshot.children.isEmpty { return [] }
+
+
+
+
     guard shouldExpandCollapsedTabContainer(containerSnapshot) else { return [] }
     let containerFrame = containerSnapshot.frame
     if containerFrame.isNull || containerFrame.isEmpty { return [] }
@@ -1076,8 +1103,7 @@ extension RunnerTests {
       collapsedTabCandidateNode(
         element: element,
         containerSnapshot: containerSnapshot,
-        containerFrame: containerFrame,
-        viewport: viewport
+        containerFrame: containerFrame
       )
     }
     .sorted { left, right in
@@ -1125,8 +1151,7 @@ extension RunnerTests {
   private func collapsedTabCandidateNode(
     element: XCUIElement,
     containerSnapshot: XCUIElementSnapshot,
-    containerFrame: CGRect,
-    viewport: CGRect
+    containerFrame: CGRect
   ) -> RawAXNode? {
     var node: RawAXNode?
     let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
@@ -1156,21 +1181,19 @@ extension RunnerTests {
         return
       }
 
+
+
       node = RawAXNode(
         index: 0,
         type: elementTypeName(elementType),
         label: label.isEmpty ? nil : label,
         identifier: identifier.isEmpty ? nil : identifier,
         value: valueText,
-        rect: snapshotRect(from: frame),
+        rect: SnapshotRect(frame),
         enabled: element.isEnabled,
         focused: elementHasFocus(element) ? true : nil,
         selected: element.isSelected ? true : nil,
-        hittable: SnapshotGeometry.isGeometricallyActionable(
-          enabled: element.isEnabled,
-          frame: frame,
-          viewport: viewport
-        ),
+        hittable: false,
         depth: 0,
         parentIndex: nil,
         hiddenContentAbove: nil,
@@ -1295,8 +1318,7 @@ extension RunnerTests {
   private func flatSnapshotNode(
     element: XCUIElement,
     index: Int,
-    parentIndex: Int?,
-    viewport: CGRect
+    parentIndex: Int?
   ) -> RawAXNode? {
     var node: RawAXNode?
     let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
@@ -1310,11 +1332,6 @@ extension RunnerTests {
       let valueText = snapshotValueText(element)
       let elementType = element.elementType
       let enabled = element.isEnabled
-      let hittable = SnapshotGeometry.isGeometricallyActionable(
-        enabled: enabled,
-        frame: frame,
-        viewport: viewport
-      )
 
       node = RawAXNode(
         index: index,
@@ -1322,11 +1339,11 @@ extension RunnerTests {
         label: label.isEmpty ? nil : label,
         identifier: identifier.isEmpty ? nil : identifier,
         value: valueText,
-        rect: snapshotRect(from: frame),
+        rect: SnapshotRect(frame),
         enabled: enabled,
         focused: elementHasFocus(element) ? true : nil,
         selected: element.isSelected ? true : nil,
-        hittable: hittable,
+        hittable: false,
         depth: 1,
         parentIndex: parentIndex,
         hiddenContentAbove: nil,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift

@@ -11,8 +11,6 @@ enum SnapshotBackendEnvironment {
 
 enum SnapshotRegularDepthCapability: String {
 
-  case presentedFrontier = "presented-frontier"
-
 
   case flat
 
@@ -63,7 +61,7 @@ enum SnapshotBackendKind: String, CaseIterable {
   var regularDepthCapability: SnapshotRegularDepthCapability {
     switch self {
     case .recursiveTree:
-      return .presentedFrontier
+      return .presentationCut
     case .querySweep:
       return .flat
     case .privateAX:

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift

@@ -62,6 +62,10 @@ struct SnapshotBackendCapture {
   var customActions: SnapshotCustomActionCoverage? = nil
   var qualityPayload: DataPayload? = nil
   var timing: SnapshotCaptureTiming? = nil
+
+
+
+  var keyboardBand: KeyboardBandFactPayload? = nil
 }
 
 extension RunnerTests {
@@ -416,6 +420,9 @@ extension RunnerTests {
     let hint = SnapshotPresentation.captureHint(for: options)
     var timer = SnapshotPhaseTimer()
     let acquisition: SnapshotAcquisition?
+
+
+    var keyboardBand: RunnerKeyboardBandFact?
     do {
       acquisition = try timer.measure(.acquisition) {
         switch kind {
@@ -430,6 +437,7 @@ extension RunnerTests {
           else {
             return nil
           }
+          keyboardBand = context.keyboardBand
           return try self.runMainThreadWork(
             "tree_processing",
             timeout: min(self.treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)),
@@ -437,7 +445,7 @@ extension RunnerTests {
           ) {
             hint.isRaw
               ? try self.rawTreeSnapshotAcquisition(context: context, hint: hint)
-              : self.recursiveTreeSnapshotAcquisition(context: context, hint: hint)
+              : try self.recursiveTreeSnapshotAcquisition(context: context, hint: hint)
           }
         case .querySweep:
           return try self.runMainThreadWork(
@@ -472,19 +480,30 @@ extension RunnerTests {
       )
     }
 
+
+
+
+    let normalizedAcquisition = acquisition.replacingNodes(
+      SnapshotGeometrySpace.normalized(
+        nodes: acquisition.nodes,
+        viewport: acquisition.viewport,
+        interfaceOrientation: acquisition.interfaceOrientation
+      )
+    )
+
     let presented: SnapshotBackendCapture
     do {
       presented = try timer.measure(.presentation) {
-        guard let result = try SnapshotPresentation.present(acquisition, options: options) else {
+        guard let result = try SnapshotPresentation.present(normalizedAcquisition, options: options) else {
           NSLog(
             "AGENT_DEVICE_RUNNER_SNAPSHOT_PROJECTION_MISMATCH requested=%@ acquired=%@",
             hint.projection.rawValue,
-            acquisition.hint.projection.rawValue
+            normalizedAcquisition.hint.projection.rawValue
           )
           throw Self.snapshotProjectionMismatchFailure(
             kind,
             requested: hint.projection,
-            acquired: acquisition.hint.projection
+            acquired: normalizedAcquisition.hint.projection
           )
         }
         return Self.makeSnapshotBackendCapture(from: result)
@@ -503,6 +522,7 @@ extension RunnerTests {
 
     var capture = presented
     capture.timing = timer.timing
+    capture.keyboardBand = keyboardBand?.payload
     return SnapshotBackendAttempt(
       outcome: .captured(capture),
       timing: timer.timing
@@ -641,6 +661,7 @@ extension RunnerTests {
         return SnapshotQualityPayload(nodes: nodes, truncated: quality.truncated == true)
       },
       snapshotQuality: quality,
+      keyboard: capture.keyboardBand,
       runnerFatal: payload.runnerFatal,
       runnerFatalReason: payload.runnerFatalReason
     )

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift

@@ -232,7 +232,7 @@ extension RunnerTests {
         label: label.isEmpty ? nil : label,
         identifier: identifier.isEmpty ? nil : identifier,
         value: nil,
-        rect: snapshotRect(from: element.frame),
+        rect: SnapshotRect(element.frame),
         enabled: element.isEnabled,
         focused: nil,
         selected: nil,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -57,6 +57,9 @@ final class RunnerTests: XCTestCase {
   var currentAppProcessIdentifier: Int?
 
 
+  var pendingTargetActivation: TargetActivationFactPayload?
+
+
 
 
   var textEntryTapWitness: TextEntryTapWitness?