agent-device 0.20.8 → 0.20.10

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -30,6 +30,15 @@ extension RunnerTests {
     let visible: Bool
   }
 
+  private struct SnapshotTraversalEntry {
+    let snapshot: XCUIElementSnapshot
+    let depth: Int
+    let visibleDepth: Int
+    let parentIndex: Int?
+    let nearestScrollAnchor: (index: Int, rect: CGRect)?
+    let projectionCursor: FlatSnapshotProjectionCursor
+  }
+
   struct SnapshotCaptureFailure: Error {
     let code: String
     let message: String
@@ -108,7 +117,7 @@ extension RunnerTests {
   // `boundedBlockingSystemAlertSnapshot`'s probe closure (see `systemModalProbeOverrideForTesting`
   // in RunnerTests.swift), so reverting this entry point to bypass the bounded probe fails the
   // regression test.
-  func snapshotFast(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload {
+  func snapshotFast(app: XCUIApplication, options: PresentationOptions) throws -> DataPayload {
     let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget)
     if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) {
       return blocking
@@ -122,10 +131,10 @@ extension RunnerTests {
     )
   }
 
-  func recursiveTreeSnapshotPayload(
+  func recursiveTreeSnapshotAcquisition(
     context: SnapshotTraversalContext,
-    options: SnapshotOptions
-  ) -> DataPayload {
+    options: PresentationOptions
+  ) -> SnapshotAcquisition {
     var cachedDescendantElements: [XCUIElement]?
     func collapsedTabDescendants() -> [XCUIElement] {
       if let cachedDescendantElements {
@@ -138,7 +147,7 @@ extension RunnerTests {
       return result.elements
     }
 
-    var nodes: [SnapshotNode] = []
+    var nodes: [RawAXNode] = []
     var hiddenContentHintsByNodeIndex: [Int: (above: Bool, below: Bool)] = [:]
     let rootEvaluation = evaluateSnapshot(context.rootSnapshot, in: context)
     nodes.append(
@@ -166,35 +175,79 @@ extension RunnerTests {
       visible: rootEvaluation.visible,
       nodeIndex: 0
     )
-    var stack: [(XCUIElementSnapshot, Int, Int, Int?, (index: Int, rect: CGRect)?)] =
-      context.rootSnapshot.children.map {
-        ($0, 1, 1, 0, rootScrollAnchor)
-      }
+    #if os(iOS)
+      let rootProjection = flatSnapshotProjectionTransition(
+        frame: context.rootSnapshot.frame,
+        intersectsViewportAndScrollClip: rootEvaluation.visible,
+        elementType: context.rootSnapshot.elementType,
+        hasChildren: !context.rootSnapshot.children.isEmpty,
+        cursor: .root
+      )
+      let rootDescendantProjectionCursor = rootProjection.decision.descendants
+    #else
+      let rootDescendantProjectionCursor = FlatSnapshotProjectionCursor.root
+    #endif
+    var stack: [SnapshotTraversalEntry] = context.rootSnapshot.children.map {
+      SnapshotTraversalEntry(
+        snapshot: $0,
+        depth: 1,
+        visibleDepth: 1,
+        parentIndex: 0,
+        nearestScrollAnchor: rootScrollAnchor,
+        projectionCursor: rootDescendantProjectionCursor
+      )
+    }
 
-    while let (snapshot, depth, visibleDepth, parentIndex, nearestScrollAnchor) = stack.popLast() {
+    while let entry = stack.popLast() {
+      let snapshot = entry.snapshot
+      let depth = entry.depth
+      let visibleDepth = entry.visibleDepth
+      let parentIndex = entry.parentIndex
+      let nearestScrollAnchor = entry.nearestScrollAnchor
+      let projectionCursor = entry.projectionCursor
       if let limit = options.depth, depth > limit { continue }
 
       let evaluation = evaluateSnapshot(snapshot, in: context)
-      let regularVisible = isVisibleInRegularSnapshot(
+      let intersectsViewportAndScrollClip = isVisibleInRegularSnapshot(
         snapshot.frame,
         viewport: context.viewport,
         scrollContainerAnchor: nearestScrollAnchor
       )
-      if !regularVisible, let nearestScrollAnchor {
+      #if os(iOS)
+        let projectionTransition = flatSnapshotProjectionTransition(
+          frame: snapshot.frame,
+          intersectsViewportAndScrollClip: intersectsViewportAndScrollClip,
+          elementType: snapshot.elementType,
+          hasChildren: !snapshot.children.isEmpty,
+          cursor: projectionCursor
+        )
+        let projection = projectionTransition.decision
+      #else
+        let projectionTransition = FlatSnapshotProjectionTransition(
+          decision: FlatSnapshotProjectionDecision(
+            presentationVisible: intersectsViewportAndScrollClip,
+            descendants: .root
+          ),
+          hiddenContentFrame: !intersectsViewportAndScrollClip ? snapshot.frame : nil
+        )
+        let projection = projectionTransition.decision
+      #endif
+      if let hiddenFrame = projectionTransition.hiddenContentFrame, let nearestScrollAnchor
+      {
         rememberHiddenContentHint(
-          for: snapshot.frame,
+          for: hiddenFrame,
           relativeTo: nearestScrollAnchor,
           hints: &hiddenContentHintsByNodeIndex
         )
       }
-      let include = shouldInclude(
+      let include = shouldAcquireSnapshotNode(
         snapshot: snapshot,
         label: evaluation.label,
         identifier: evaluation.identifier,
         valueText: evaluation.valueText,
         options: options,
         hittable: evaluation.hittable,
-        visible: regularVisible,
+        visible: projection.presentationVisible,
         regularSnapshot: true
       )
 
@@ -217,7 +270,7 @@ extension RunnerTests {
           nextScrollContainerAnchor =
             scrollContainerAnchor(
               for: snapshot,
-              visible: regularVisible,
+              visible: intersectsViewportAndScrollClip,
               nodeIndex: currentIndex
             )
             ?? nearestScrollAnchor
@@ -225,7 +278,16 @@ extension RunnerTests {
           nextScrollContainerAnchor = nearestScrollAnchor
         }
         for child in snapshot.children.reversed() {
-          stack.append((child, depth + 1, nextVisibleDepth, currentIndex, nextScrollContainerAnchor))
+          stack.append(
+            SnapshotTraversalEntry(
+              snapshot: child,
+              depth: depth + 1,
+              visibleDepth: nextVisibleDepth,
+              parentIndex: currentIndex,
+              nearestScrollAnchor: nextScrollContainerAnchor,
+              projectionCursor: projection.descendants
+            )
+          )
         }
       }
 
@@ -253,14 +315,15 @@ extension RunnerTests {
 
     }
 
-    return DataPayload(
+    return SnapshotAcquisition(
       nodes: applyHiddenContentHints(hiddenContentHintsByNodeIndex, to: nodes),
-      truncated: false
+      truncated: false,
+      effectiveDepth: nil
     )
   }
 
   // See `snapshotFast` above: the single production entry point, no unit-test overload.
-  func snapshotRaw(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload {
+  func snapshotRaw(app: XCUIApplication, options: PresentationOptions) throws -> DataPayload {
     let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget)
     if let blocking = boundedBlockingSystemAlertSnapshot(deadline: deadline) {
       return blocking
@@ -351,17 +414,17 @@ extension RunnerTests {
     return min(budget, deadlineRemaining)
   }
 
-  func rawTreeSnapshotPayload(
+  func rawTreeSnapshotAcquisition(
     context: SnapshotTraversalContext,
-    options: SnapshotOptions
-  ) throws -> DataPayload {
-    var nodes: [SnapshotNode] = []
+    options: PresentationOptions
+  ) throws -> SnapshotAcquisition {
+    var nodes: [RawAXNode] = []
 
     func walk(_ snapshot: XCUIElementSnapshot, depth: Int, parentIndex: Int?) throws {
       if let limit = options.depth, depth > limit { return }
 
       let evaluation = evaluateSnapshot(snapshot, in: context)
-      let include = shouldInclude(
+      let include = shouldAcquireSnapshotNode(
         snapshot: snapshot,
         label: evaluation.label,
         identifier: evaluation.identifier,
@@ -393,19 +456,19 @@ extension RunnerTests {
     }
 
     try walk(context.rootSnapshot, depth: 0, parentIndex: nil)
-    return DataPayload(nodes: nodes, truncated: false)
+    return SnapshotAcquisition(nodes: nodes, truncated: false, effectiveDepth: nil)
   }
 
-  func snapshotFlatInteractive(
+  func querySweepSnapshotAcquisition(
     app: XCUIApplication,
-    options: SnapshotOptions,
+    options: PresentationOptions,
     planDeadline: Date = .distantFuture
-  ) -> DataPayload {
-    var nodes: [SnapshotNode] = [
+  ) -> SnapshotAcquisition {
+    var nodes: [RawAXNode] = [
       interactiveRootNode(rect: .zero)
     ]
     if options.depth == 0 {
-      return DataPayload(nodes: nodes, truncated: false)
+      return SnapshotAcquisition(nodes: nodes, truncated: false, effectiveDepth: nil)
     }
 
     // Bounded by both its own sweep budget and the umbrella capture-plan deadline, so a
@@ -416,7 +479,7 @@ extension RunnerTests {
     let deadline = min(sweepDeadline, planDeadline)
     let viewport = safeSnapshotViewport(app: app)
     var seen = Set<String>()
-    var candidates: [SnapshotNode] = []
+    var candidates: [RawAXNode] = []
     let flatElements = flatInteractiveElements(app: app, deadline: deadline)
     var truncated = flatElements.truncated
     for element in flatElements.elements {
@@ -458,7 +521,7 @@ extension RunnerTests {
     nodes[0] = interactiveRootNode(rect: rootRect)
     for candidate in candidates {
       nodes.append(
-        SnapshotNode(
+        RawAXNode(
           index: nodes.count,
           type: candidate.type,
           label: candidate.label,
@@ -476,7 +539,7 @@ extension RunnerTests {
         )
       )
     }
-    return DataPayload(nodes: nodes, truncated: truncated)
+    return SnapshotAcquisition(nodes: nodes, truncated: truncated, effectiveDepth: nil)
   }
 
   func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload {
@@ -521,7 +584,7 @@ extension RunnerTests {
   ) -> DataPayload {
     return DataPayload(
       message: message,
-      nodes: [interactiveRootNode(rect: .zero)],
+      nodes: [SnapshotPresentation.singleElementRead(interactiveRootNode(rect: .zero))],
       truncated: true,
       snapshotQuality: snapshotQuality,
       runnerFatal: runnerFatal,
@@ -530,8 +593,8 @@ extension RunnerTests {
   }
 
 
-  private func interactiveRootNode(rect: CGRect) -> SnapshotNode {
-    SnapshotNode(
+  private func interactiveRootNode(rect: CGRect) -> RawAXNode {
+    RawAXNode(
       index: 0,
       type: "Application",
       label: nil,
@@ -549,7 +612,7 @@ extension RunnerTests {
     )
   }
 
-  private func interactiveRootFrame(for candidates: [SnapshotNode]) -> CGRect {
+  private func interactiveRootFrame(for candidates: [RawAXNode]) -> CGRect {
     guard !candidates.isEmpty else {
       return .zero
     }
@@ -569,34 +632,41 @@ extension RunnerTests {
 
   // MARK: - Snapshot Filtering
 
-  private func shouldInclude(
+  /// Conservative acquisition gate. Regular capture keeps every node its visibility policy can
+  /// supply; `SnapshotPresentation` alone decides eligibility after the backend returns.
+  private func shouldAcquireSnapshotNode(
     snapshot: XCUIElementSnapshot,
     label: String,
     identifier: String,
     valueText: String?,
-    options: SnapshotOptions,
+    options: PresentationOptions,
     hittable: Bool,
     visible: Bool,
     regularSnapshot: Bool = false
   ) -> Bool {
     let type = snapshot.elementType
     let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
-    if options.interactiveOnly {
-      if isScrollableContainer(snapshot, visible: visible) { return true }
+    #if os(iOS)
+      if regularSnapshot && !visible && type != .application && type != .window {
+        return false
+      }
+    #endif
+    if regularSnapshot {
       #if os(macOS)
-        if !visible && type != .application {
+        if options.interactiveOnly && !visible && type != .application {
           return false
         }
       #endif
+      if type == .application || type == .window { return true }
+      return visible
+    }
+    if options.interactiveOnly {
+      if isScrollableContainer(snapshot, visible: visible) { return true }
       if interactiveTypes.contains(type) { return true }
       if hittable && type != .other { return true }
       if hasContent { return true }
       return false
     }
-    if regularSnapshot {
-      if type == .application || type == .window { return true }
-      return visible
-    }
     return true
   }
 
@@ -621,7 +691,7 @@ extension RunnerTests {
 
   func makeSnapshotTraversalContext(
     app: XCUIApplication,
-    options: SnapshotOptions,
+    options: PresentationOptions,
     captureDeadline: Date = .distantFuture,
     treeCaptureSliceBudgetOverride: TimeInterval? = nil
   ) throws -> SnapshotTraversalContext? {
@@ -815,8 +885,8 @@ extension RunnerTests {
     depth: Int,
     index: Int,
     parentIndex: Int?
-  ) -> SnapshotNode {
-    return SnapshotNode(
+  ) -> RawAXNode {
+    return RawAXNode(
       index: index,
       type: elementTypeName(snapshot.elementType),
       label: evaluation.label.isEmpty ? nil : evaluation.label,
@@ -925,7 +995,7 @@ extension RunnerTests {
     return rect.intersects(viewport)
   }
 
-  private func isVisibleInRegularSnapshot(
+  func isVisibleInRegularSnapshot(
     _ rect: CGRect,
     viewport: CGRect,
     scrollContainerAnchor: (index: Int, rect: CGRect)?
@@ -936,7 +1006,7 @@ extension RunnerTests {
   }
 
   private func appendCollapsedTabFallbackNodes(
-    to nodes: inout [SnapshotNode],
+    to nodes: inout [RawAXNode],
     containerSnapshot: XCUIElementSnapshot,
     resolveElements: () -> [XCUIElement],
     depth: Int,
@@ -956,13 +1026,33 @@ extension RunnerTests {
     for snapshot: XCUIElementSnapshot,
     visible: Bool,
     nodeIndex: Int?
+  ) -> (index: Int, rect: CGRect)? {
+    return scrollContainerAnchor(
+      for: snapshot.elementType,
+      hasChildren: !snapshot.children.isEmpty,
+      visible: visible,
+      frame: snapshot.frame,
+      nodeIndex: nodeIndex
+    )
+  }
+
+  func scrollContainerAnchor(
+    for elementType: XCUIElement.ElementType,
+    hasChildren: Bool,
+    visible: Bool,
+    frame: CGRect,
+    nodeIndex: Int?
   ) -> (index: Int, rect: CGRect)? {
     guard let nodeIndex else { return nil }
-    if !isScrollableContainer(snapshot, visible: visible) { return nil }
-    return (nodeIndex, snapshot.frame)
+    if !isScrollableContainer(
+      elementType: elementType,
+      hasChildren: hasChildren,
+      visible: visible
+    ) { return nil }
+    return (nodeIndex, frame)
   }
 
-  private func rememberHiddenContentHint(
+  func rememberHiddenContentHint(
     for frame: CGRect,
     relativeTo scrollContainerAnchor: (index: Int, rect: CGRect),
     hints: inout [Int: (above: Bool, below: Bool)]
@@ -979,16 +1069,16 @@ extension RunnerTests {
     hints[scrollContainerAnchor.index] = hint
   }
 
-  private func applyHiddenContentHints(
+  func applyHiddenContentHints(
     _ hints: [Int: (above: Bool, below: Bool)],
-    to nodes: [SnapshotNode]
-  ) -> [SnapshotNode] {
+    to nodes: [RawAXNode]
+  ) -> [RawAXNode] {
     if hints.isEmpty { return nodes }
     return nodes.map { node in
       guard let hint = hints[node.index] else { return node }
       let hiddenContentAbove: Bool? = (node.hiddenContentAbove == true || hint.above) ? true : nil
       let hiddenContentBelow: Bool? = (node.hiddenContentBelow == true || hint.below) ? true : nil
-      return SnapshotNode(
+      return RawAXNode(
         index: node.index,
         type: node.type,
         label: node.label,
@@ -1002,7 +1092,8 @@ extension RunnerTests {
         depth: node.depth,
         parentIndex: node.parentIndex,
         hiddenContentAbove: hiddenContentAbove,
-        hiddenContentBelow: hiddenContentBelow
+        hiddenContentBelow: hiddenContentBelow,
+        actions: node.actions
       )
     }
   }
@@ -1013,7 +1104,7 @@ extension RunnerTests {
     startingIndex: Int,
     depth: Int,
     parentIndex: Int
-  ) -> [SnapshotNode] {
+  ) -> [RawAXNode] {
     if !containerSnapshot.children.isEmpty { return [] }
     guard shouldExpandCollapsedTabContainer(containerSnapshot) else { return [] }
     let containerFrame = containerSnapshot.frame
@@ -1052,7 +1143,7 @@ extension RunnerTests {
     if uniqueCandidates.count < 2 { return [] }
 
     return uniqueCandidates.enumerated().map { offset, node in
-      SnapshotNode(
+      RawAXNode(
         index: startingIndex + offset,
         type: node.type,
         label: node.label,
@@ -1075,8 +1166,8 @@ extension RunnerTests {
     element: XCUIElement,
     containerSnapshot: XCUIElementSnapshot,
     containerFrame: CGRect
-  ) -> SnapshotNode? {
-    var node: SnapshotNode?
+  ) -> RawAXNode? {
+    var node: RawAXNode?
     let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
       if !element.exists { return }
       let elementType = element.elementType
@@ -1104,7 +1195,7 @@ extension RunnerTests {
         return
       }
 
-      node = SnapshotNode(
+      node = RawAXNode(
         index: 0,
         type: elementTypeName(elementType),
         label: label.isEmpty ? nil : label,
@@ -1241,9 +1332,9 @@ extension RunnerTests {
     index: Int,
     parentIndex: Int?,
     viewport: CGRect,
-    options: SnapshotOptions
-  ) -> SnapshotNode? {
-    var node: SnapshotNode?
+    options: PresentationOptions
+  ) -> RawAXNode? {
+    var node: RawAXNode?
     let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
       if !element.exists { return }
       let frame = element.frame
@@ -1266,11 +1357,16 @@ extension RunnerTests {
         valueText: valueText,
         visible: visible
       )
-      if !flatSnapshotFilterDecision(filterNode, options: options, insideMatchedScope: false).include {
+      if !flatSnapshotFilterDecision(
+        filterNode,
+        options: options,
+        visibilityPolicy: .interactiveOnly,
+        insideMatchedScope: false
+      ).include {
         return
       }
 
-      node = SnapshotNode(
+      node = RawAXNode(
         index: index,
         type: elementTypeName(elementType),
         label: label.isEmpty ? nil : label,
@@ -1295,8 +1391,18 @@ extension RunnerTests {
   }
 
   private func isScrollableContainer(_ snapshot: XCUIElementSnapshot, visible: Bool) -> Bool {
-    if !visible { return false }
-    if !Self.scrollContainerTypes.contains(snapshot.elementType) { return false }
-    return !snapshot.children.isEmpty
+    return isScrollableContainer(
+      elementType: snapshot.elementType,
+      hasChildren: !snapshot.children.isEmpty,
+      visible: visible
+    )
+  }
+
+  private func isScrollableContainer(
+    elementType: XCUIElement.ElementType,
+    hasChildren: Bool,
+    visible: Bool
+  ) -> Bool {
+    return visible && hasChildren && Self.scrollContainerTypes.contains(elementType)
   }
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift

@@ -250,7 +250,7 @@ extension RunnerTests {
   func runSnapshotCapturePlan(
     _ plan: [SnapshotBackendKind],
     app: XCUIApplication,
-    options: SnapshotOptions,
+    options: PresentationOptions,
     terminal: SnapshotCaptureTerminalPolicy,
     deadline: Date? = nil
   ) throws -> DataPayload {
@@ -452,10 +452,11 @@ extension RunnerTests {
   private func captureWithBackend(
     _ kind: SnapshotBackendKind,
     app: XCUIApplication,
-    options: SnapshotOptions,
+    options: PresentationOptions,
     deadline: Date,
     treeCaptureSliceBudgetOverride: TimeInterval?
   ) throws -> SnapshotBackendCapture? {
+    let acquisition: SnapshotAcquisition?
     switch kind {
     case .recursiveTree:
       guard
@@ -468,28 +469,28 @@ extension RunnerTests {
       else {
         return nil
       }
-      let payload = try runMainThreadWork(
+      acquisition = try runMainThreadWork(
         command: nil,
         timeout: min(treeCaptureSliceBudget, max(0.5, deadline.timeIntervalSinceNow)),
         timeoutError: snapshotMainThreadTimeoutError("processing tree snapshot")
       ) {
         options.raw
-          ? try self.rawTreeSnapshotPayload(context: context, options: options)
-          : self.recursiveTreeSnapshotPayload(context: context, options: options)
+          ? try self.rawTreeSnapshotAcquisition(context: context, options: options)
+          : self.recursiveTreeSnapshotAcquisition(context: context, options: options)
       }
-      return SnapshotBackendCapture(payload: payload, effectiveDepth: nil)
     case .querySweep:
-      let payload = try runMainThreadWork(
+      acquisition = try runMainThreadWork(
         command: nil,
         timeout: min(Self.flatInteractiveFallbackBudget, max(0.1, deadline.timeIntervalSinceNow)),
         timeoutError: snapshotMainThreadTimeoutError("running query-sweep snapshot")
       ) {
-        self.snapshotFlatInteractive(app: app, options: options, planDeadline: deadline)
+        self.querySweepSnapshotAcquisition(app: app, options: options, planDeadline: deadline)
       }
-      return SnapshotBackendCapture(payload: payload, effectiveDepth: nil)
     case .privateAX:
-      return privateAXSnapshotCapture(app: app, options: options, deadline: deadline)
+      acquisition = privateAXSnapshotAcquisition(app: app, options: options, deadline: deadline)
     }
+    guard let acquisition else { return nil }
+    return SnapshotPresentation.present(acquisition, options: options)
   }
 
   // MARK: Quality classifier (the single source of "is this snapshot degraded")
@@ -529,7 +530,7 @@ extension RunnerTests {
     }
   }
 
-  static func isSparseApplicationWindowTree(_ nodes: [SnapshotNode]) -> Bool {
+  static func isSparseApplicationWindowTree(_ nodes: [PresentedNode]) -> Bool {
     guard !nodes.isEmpty else { return false }
     let rootRects = nodes.compactMap { node in
       node.type == "Application" || node.type == "Window" ? node.rect : nil
@@ -564,7 +565,7 @@ extension RunnerTests {
   /// A leaf whose label joins many short segments is a container marked as an accessibility
   /// element: the platform folds every descendant into one merged node. Nothing below it can
   /// be addressed — by automation or by assistive tech. This is app-side; no backend recovers it.
-  static func collapsedLeafIndexes(_ nodes: [SnapshotNode]) -> [Int]? {
+  static func collapsedLeafIndexes(_ nodes: [PresentedNode]) -> [Int]? {
     let parents = Set(nodes.compactMap { $0.parentIndex })
     let collapsed = nodes.filter { node in
       guard !parents.contains(node.index) else { return false }