agent-device 0.20.8 → 0.20.10

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -229,21 +229,23 @@ extension RunnerTests {
     let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
     let valueText = String(describing: element.value ?? "")
       .trimmingCharacters(in: .whitespacesAndNewlines)
-    let node = SnapshotNode(
-      index: 0,
-      type: elementTypeName(element.elementType),
-      label: label.isEmpty ? nil : label,
-      identifier: identifier.isEmpty ? nil : identifier,
-      value: valueText.isEmpty ? nil : valueText,
-      rect: snapshotRect(from: element.frame),
-      enabled: element.isEnabled,
-      focused: nil,
-      selected: element.isSelected ? true : nil,
-      hittable: element.isHittable,
-      depth: 0,
-      parentIndex: nil,
-      hiddenContentAbove: nil,
-      hiddenContentBelow: nil
+    let node = SnapshotPresentation.singleElementRead(
+      RawAXNode(
+        index: 0,
+        type: elementTypeName(element.elementType),
+        label: label.isEmpty ? nil : label,
+        identifier: identifier.isEmpty ? nil : identifier,
+        value: valueText.isEmpty ? nil : valueText,
+        rect: snapshotRect(from: element.frame),
+        enabled: element.isEnabled,
+        focused: nil,
+        selected: element.isSelected ? true : nil,
+        hittable: element.isHittable,
+        depth: 0,
+        parentIndex: nil,
+        hiddenContentAbove: nil,
+        hiddenContentBelow: nil
+      )
     )
     return Response(
       ok: true,
@@ -317,7 +319,11 @@ extension RunnerTests {
         .filter { element in
           guard element.exists else { return false }
           let frame = element.frame
-          return !frame.isEmpty && frameContainsPoint(frame, point, tolerance: 2)
+          return isCoordinateTextInputCandidate(
+            enabled: element.isEnabled,
+            frame: frame,
+            point: point
+          )
         }
         .sorted { left, right in
           let leftArea = max(1, left.frame.width * left.frame.height)
@@ -336,13 +342,6 @@ extension RunnerTests {
     }
   }
 
-  private func frameContainsPoint(_ frame: CGRect, _ point: CGPoint, tolerance: CGFloat) -> Bool {
-    point.x >= frame.minX - tolerance
-      && point.x <= frame.maxX + tolerance
-      && point.y >= frame.minY - tolerance
-      && point.y <= frame.maxY + tolerance
-  }
-
   private func readableText(for element: XCUIElement) -> String? {
     let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
     let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -235,7 +235,7 @@ struct DataPayload: Codable {
   let text: String?
   let found: Bool?
   let items: [String]?
-  let nodes: [SnapshotNode]?
+  let nodes: [PresentedNode]?
   let truncated: Bool?
   let snapshotQuality: SnapshotQuality?
   let gestureStartUptimeMs: Double?
@@ -277,7 +277,7 @@ struct DataPayload: Codable {
     text: String? = nil,
     found: Bool? = nil,
     items: [String]? = nil,
-    nodes: [SnapshotNode]? = nil,
+    nodes: [PresentedNode]? = nil,
     truncated: Bool? = nil,
     snapshotQuality: SnapshotQuality? = nil,
     gestureStartUptimeMs: Double? = nil,
@@ -375,27 +375,7 @@ struct SnapshotRect: Codable {
   let height: Double
 }
 
-struct SnapshotNode: Codable {
-  let index: Int
-  let type: String
-  let label: String?
-  let identifier: String?
-  let value: String?
-  let rect: SnapshotRect
-  let enabled: Bool
-  let focused: Bool?
-  let selected: Bool?
-  let hittable: Bool
-  let depth: Int
-  let parentIndex: Int?
-  let hiddenContentAbove: Bool?
-  let hiddenContentBelow: Bool?
-  /// Names of the element's UIAccessibilityCustomActions, present only when the
-  /// capture asked for them and the element carries some.
-  var actions: [String]? = nil
-}
-
-struct SnapshotOptions {
+struct PresentationOptions {
   let interactiveOnly: Bool
   let depth: Int?
   let scope: String?

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -81,17 +81,16 @@ extension RunnerTests {
     return navigationBackKeywords.firstIndex { text.contains($0) }
   }
 
+  // isFinite/>0 alone don't reject CGRect.infinite — its origin (~-9e307) is finite.
+  static func isUsableNavigationFrame(_ frame: CGRect) -> Bool {
+    guard frame.width.isFinite, frame.height.isFinite, frame.width > 0, frame.height > 0 else {
+      return false
+    }
+    return !frame.isInfinite
+  }
+
   static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool {
-    guard
-      candidate.width.isFinite,
-      candidate.height.isFinite,
-      window.width.isFinite,
-      window.height.isFinite,
-      candidate.width > 0,
-      candidate.height > 0,
-      window.width > 0,
-      window.height > 0
-    else {
+    guard isUsableNavigationFrame(candidate), isUsableNavigationFrame(window) else {
       return false
     }
     // Accept the compact navigation/search header band without matching deep content controls.
@@ -100,7 +99,7 @@ extension RunnerTests {
   }
 
   static func topLeadingNavigationFallbackPoint(in frame: CGRect) -> CGPoint? {
-    guard frame.width.isFinite, frame.height.isFinite, frame.width > 0, frame.height > 0 else {
+    guard isUsableNavigationFrame(frame) else {
       return nil
     }
     // Aim at the standard leading navigation slot, bounded for compact and tablet widths.

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift

@@ -0,0 +1,108 @@
+import XCTest
+
+extension RunnerTests {
+  private static let privateAXProjectionTolerance: CGFloat = 1
+
+  func privateAXPresentation(rawRoot: [String: Any], options: PresentationOptions, viewport: CGRect)
+    -> [RawAXNode]
+  {
+    var nodes: [RawAXNode] = []
+    var hints: [Int: (above: Bool, below: Bool)] = [:]
+    appendPrivateAXNode(rawRoot, to: &nodes, hints: &hints, options: options, viewport: viewport,
+      depth: 0, parentIndex: nil, insideMatchedScope: false, scrollContext: nil,
+      projectionCursor: .root)
+    return applyHiddenContentHints(hints, to: nodes)
+  }
+
+  private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [RawAXNode],
+    hints: inout [Int: (above: Bool, below: Bool)], options: PresentationOptions, viewport: CGRect,
+    depth: Int, parentIndex: Int?, insideMatchedScope: Bool,
+    scrollContext: (index: Int, rect: CGRect)?, projectionCursor: FlatSnapshotProjectionCursor)
+  {
+    if let limit = options.depth, depth > limit { return }
+    let rect = privateAXRect(raw["frame"])
+    let label = privateAXPresentationString(raw["label"])
+    let identifier = privateAXPresentationString(raw["identifier"])
+    let value = privateAXPresentationString(raw["value"])
+    let rawType = privateAXPresentationInt(raw["type"]) ?? 0
+    let enabled = privateAXPresentationBool(raw["enabled"]) ?? true
+    let children = raw["children"] as? [[String: Any]] ?? []
+    let hasSemanticContent = [label, identifier, value].contains(where: { !$0.isEmpty })
+    let elementType = flatSnapshotElementType(rawElementType: rawType)
+    let hasFrame = !rect.isNull && !rect.isEmpty
+    let negligibleDecoration = parentIndex != nil
+      && !hasSemanticContent
+      && (!hasFrame
+        || rect.width <= Self.privateAXProjectionTolerance
+        || rect.height <= Self.privateAXProjectionTolerance)
+    let onScreen = hasFrame
+      && isVisibleInRegularSnapshot(
+        rect,
+        viewport: viewport,
+        scrollContainerAnchor: scrollContext
+      )
+    let projectionTransition = flatSnapshotProjectionTransition(
+      frame: rect,
+      intersectsViewportAndScrollClip: onScreen,
+      elementType: elementType,
+      hasChildren: !children.isEmpty,
+      cursor: projectionCursor
+    )
+    let projection = projectionTransition.decision
+    let presentationVisible = projection.presentationVisible && !negligibleDecoration
+    let decision = flatSnapshotFilterDecision(
+      FlatSnapshotFilterNode(isRoot: parentIndex == nil, label: label, identifier: identifier,
+        valueText: value.isEmpty ? nil : value, visible: presentationVisible),
+      options: options, visibilityPolicy: .viewportProjected,
+      insideMatchedScope: insideMatchedScope)
+    let include = decision.include
+
+    if let hiddenFrame = projectionTransition.hiddenContentFrame, let scrollContext {
+      rememberHiddenContentHint(for: hiddenFrame, relativeTo: scrollContext, hints: &hints)
+    }
+
+    let currentIndex: Int?
+    if include {
+      currentIndex = nodes.count
+      let typeName = elementType.map(elementTypeName) ?? "Element(\(rawType))"
+      nodes.append(RawAXNode(index: nodes.count, type: typeName,
+        label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier,
+        value: value.isEmpty ? nil : value, rect: snapshotRect(from: rect), enabled: enabled,
+        focused: privateAXPresentationBool(raw["focused"]) == true ? true : nil,
+        selected: privateAXPresentationBool(raw["selected"]) == true ? true : nil,
+        hittable: onScreen && enabled && privateAXInteractiveCandidate(rawElementType: rawType),
+        depth: depth, parentIndex: parentIndex, hiddenContentAbove: nil, hiddenContentBelow: nil,
+        actions: raw["actions"] as? [String]))
+    } else { currentIndex = parentIndex }
+
+    let nextScrollContext: (index: Int, rect: CGRect)?
+    if include, let elementType, let currentIndex {
+      nextScrollContext = scrollContainerAnchor(
+        for: elementType,
+        hasChildren: !children.isEmpty,
+        visible: onScreen,
+        frame: rect,
+        nodeIndex: currentIndex
+      ) ?? scrollContext
+    } else { nextScrollContext = scrollContext }
+    for child in children {
+      appendPrivateAXNode(child, to: &nodes, hints: &hints, options: options, viewport: viewport,
+        depth: depth + 1, parentIndex: currentIndex,
+        insideMatchedScope: decision.insideMatchedScope, scrollContext: nextScrollContext,
+        projectionCursor: projection.descendants)
+    }
+  }
+
+  private func privateAXPresentationString(_ value: Any?) -> String {
+    guard let value else { return "" }
+    return (value as? String ?? String(describing: value))
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+  }
+  private func privateAXPresentationInt(_ value: Any?) -> Int? {
+    (value as? Int) ?? (value as? NSNumber)?.intValue
+  }
+  private func privateAXPresentationBool(_ value: Any?) -> Bool? {
+    (value as? Bool) ?? (value as? NSNumber)?.boolValue
+  }
+}
+

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift

@@ -3,10 +3,10 @@ import XCTest
 // Swift port of buildScrollGesturePlan from packages/contracts/src/scroll-gesture.ts.
 //
 // This is a deliberate two-place invariant: the daemon keeps the TS implementation (for Android,
-// recording, and reported-pixels), and the runner places the gesture with this Swift copy. The
-// parity test vectors at the bottom of this file mirror
-// packages/contracts/src/scroll-gesture.test.ts —
-// if you change the math in either language, update the other and both vector sets.
+// recording, and reported-pixels), and the runner places the gesture with this Swift copy. Both
+// ports are asserted against the same table, contracts/fixtures/scroll-gesture.json (gated
+// XCTest at the bottom of this file, vitest twin packages/contracts/src/scroll-gesture.test.ts) —
+// if you change the math in either language, update the other and the table.
 //
 // All inputs here are positive (reference dims, travel, center), so Swift's `.rounded()`
 // (half away from zero) matches JS `Math.round` (half up) on every value computed below.
@@ -20,7 +20,10 @@ struct RunnerScrollGesturePlan {
 }
 
 private let runnerDefaultScrollAmount = 0.6
-private let runnerDefaultEdgePaddingFraction = 0.05
+// Both constants are pinned by contracts/fixtures/scroll-gesture.json (`constants`). Scroll gestures
+// stay out of the outer 10% of each axis so a saturated scroll never touches down inside the status
+// bar / Dynamic Island band (#1781 A1).
+private let runnerDefaultEdgePaddingFraction = 0.1
 
 func runnerScrollGesturePlan(
   direction: String,