agent-device 0.20.10 → 0.21.0

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift

@@ -1,96 +1,92 @@
 import XCTest
+import AgentDeviceSnapshotPresentation
 
-extension RunnerTests {
-  private static let privateAXProjectionTolerance: CGFloat = 1
+/// Reported facts for one private-AX element, read once so every consumer describes a node the
+/// same way.
+struct PrivateAXFields {
+  let rect: CGRect
+  let label: String
+  let identifier: String
+  let value: String
+  let rawType: Int
+  let elementType: XCUIElement.ElementType?
+  let enabled: Bool
+  let focused: Bool?
+  let selected: Bool?
+  let actions: [String]?
+  let children: [[String: Any]]
+}
 
-  func privateAXPresentation(rawRoot: [String: Any], options: PresentationOptions, viewport: CGRect)
+extension RunnerTests {
+  /// Private-AX acquisition: ONE serializer for both projections. Reported facts at traversal
+  /// depth -- membership, the clip fold, scroll hints, and collapsed depth are
+  /// `SnapshotPresentation`'s alone (#1797). The traversal-depth cut is the backend's one
+  /// narrowing, complete for raw (raw depth *is* traversal depth) and a declared residue for
+  /// regular; `hittable` is the shared geometric fact the fold recomputes for effective geometry.
+  func privateAXAcquisition(rawRoot: [String: Any], hint: CaptureHint, viewport: CGRect)
     -> [RawAXNode]
   {
     var nodes: [RawAXNode] = []
-    var hints: [Int: (above: Bool, below: Bool)] = [:]
-    appendPrivateAXNode(rawRoot, to: &nodes, hints: &hints, options: options, viewport: viewport,
-      depth: 0, parentIndex: nil, insideMatchedScope: false, scrollContext: nil,
-      projectionCursor: .root)
-    return applyHiddenContentHints(hints, to: nodes)
+    appendPrivateAXNode(rawRoot, to: &nodes, hint: hint, viewport: viewport,
+      depth: 0, parentIndex: nil)
+    return nodes
   }
 
   private func appendPrivateAXNode(_ raw: [String: Any], to nodes: inout [RawAXNode],
-    hints: inout [Int: (above: Bool, below: Bool)], options: PresentationOptions, viewport: CGRect,
-    depth: Int, parentIndex: Int?, insideMatchedScope: Bool,
-    scrollContext: (index: Int, rect: CGRect)?, projectionCursor: FlatSnapshotProjectionCursor)
+    hint: CaptureHint, viewport: CGRect, depth: Int, parentIndex: Int?)
   {
-    if let limit = options.depth, depth > limit { return }
-    let rect = privateAXRect(raw["frame"])
-    let label = privateAXPresentationString(raw["label"])
-    let identifier = privateAXPresentationString(raw["identifier"])
-    let value = privateAXPresentationString(raw["value"])
-    let rawType = privateAXPresentationInt(raw["type"]) ?? 0
-    let enabled = privateAXPresentationBool(raw["enabled"]) ?? true
-    let children = raw["children"] as? [[String: Any]] ?? []
-    let hasSemanticContent = [label, identifier, value].contains(where: { !$0.isEmpty })
-    let elementType = flatSnapshotElementType(rawElementType: rawType)
-    let hasFrame = !rect.isNull && !rect.isEmpty
-    let negligibleDecoration = parentIndex != nil
-      && !hasSemanticContent
-      && (!hasFrame
-        || rect.width <= Self.privateAXProjectionTolerance
-        || rect.height <= Self.privateAXProjectionTolerance)
-    let onScreen = hasFrame
-      && isVisibleInRegularSnapshot(
-        rect,
-        viewport: viewport,
-        scrollContainerAnchor: scrollContext
+    if let limit = hint.rawTraversalDepth, depth > limit { return }
+    let fields = privateAXFields(raw)
+    let index = nodes.count
+    nodes.append(
+      privateAXNode(
+        fields,
+        index: index,
+        depth: depth,
+        parentIndex: parentIndex,
+        viewport: viewport
       )
-    let projectionTransition = flatSnapshotProjectionTransition(
-      frame: rect,
-      intersectsViewportAndScrollClip: onScreen,
-      elementType: elementType,
-      hasChildren: !children.isEmpty,
-      cursor: projectionCursor
     )
-    let projection = projectionTransition.decision
-    let presentationVisible = projection.presentationVisible && !negligibleDecoration
-    let decision = flatSnapshotFilterDecision(
-      FlatSnapshotFilterNode(isRoot: parentIndex == nil, label: label, identifier: identifier,
-        valueText: value.isEmpty ? nil : value, visible: presentationVisible),
-      options: options, visibilityPolicy: .viewportProjected,
-      insideMatchedScope: insideMatchedScope)
-    let include = decision.include
-
-    if let hiddenFrame = projectionTransition.hiddenContentFrame, let scrollContext {
-      rememberHiddenContentHint(for: hiddenFrame, relativeTo: scrollContext, hints: &hints)
+    for child in fields.children {
+      appendPrivateAXNode(child, to: &nodes, hint: hint, viewport: viewport,
+        depth: depth + 1, parentIndex: index)
     }
+  }
 
-    let currentIndex: Int?
-    if include {
-      currentIndex = nodes.count
-      let typeName = elementType.map(elementTypeName) ?? "Element(\(rawType))"
-      nodes.append(RawAXNode(index: nodes.count, type: typeName,
-        label: label.isEmpty ? nil : label, identifier: identifier.isEmpty ? nil : identifier,
-        value: value.isEmpty ? nil : value, rect: snapshotRect(from: rect), enabled: enabled,
-        focused: privateAXPresentationBool(raw["focused"]) == true ? true : nil,
-        selected: privateAXPresentationBool(raw["selected"]) == true ? true : nil,
-        hittable: onScreen && enabled && privateAXInteractiveCandidate(rawElementType: rawType),
-        depth: depth, parentIndex: parentIndex, hiddenContentAbove: nil, hiddenContentBelow: nil,
-        actions: raw["actions"] as? [String]))
-    } else { currentIndex = parentIndex }
+  private func privateAXFields(_ raw: [String: Any]) -> PrivateAXFields {
+    let rawType = privateAXPresentationInt(raw["type"]) ?? 0
+    return PrivateAXFields(
+      rect: privateAXRect(raw["frame"]),
+      label: privateAXPresentationString(raw["label"]),
+      identifier: privateAXPresentationString(raw["identifier"]),
+      value: privateAXPresentationString(raw["value"]),
+      rawType: rawType,
+      elementType: privateAXElementType(rawElementType: rawType),
+      enabled: privateAXPresentationBool(raw["enabled"]) ?? true,
+      focused: privateAXPresentationBool(raw["focused"]) == true ? true : nil,
+      selected: privateAXPresentationBool(raw["selected"]) == true ? true : nil,
+      actions: raw["actions"] as? [String],
+      children: raw["children"] as? [[String: Any]] ?? []
+    )
+  }
 
-    let nextScrollContext: (index: Int, rect: CGRect)?
-    if include, let elementType, let currentIndex {
-      nextScrollContext = scrollContainerAnchor(
-        for: elementType,
-        hasChildren: !children.isEmpty,
-        visible: onScreen,
-        frame: rect,
-        nodeIndex: currentIndex
-      ) ?? scrollContext
-    } else { nextScrollContext = scrollContext }
-    for child in children {
-      appendPrivateAXNode(child, to: &nodes, hints: &hints, options: options, viewport: viewport,
-        depth: depth + 1, parentIndex: currentIndex,
-        insideMatchedScope: decision.insideMatchedScope, scrollContext: nextScrollContext,
-        projectionCursor: projection.descendants)
-    }
+  private func privateAXNode(_ fields: PrivateAXFields, index: Int, depth: Int, parentIndex: Int?,
+    viewport: CGRect) -> RawAXNode
+  {
+    RawAXNode(index: index,
+      type: fields.elementType.map(elementTypeName) ?? "Element(\(fields.rawType))",
+      label: fields.label.isEmpty ? nil : fields.label,
+      identifier: fields.identifier.isEmpty ? nil : fields.identifier,
+      value: fields.value.isEmpty ? nil : fields.value,
+      rect: snapshotRect(from: fields.rect), enabled: fields.enabled,
+      focused: fields.focused, selected: fields.selected,
+      hittable: parentIndex != nil && SnapshotGeometry.isGeometricallyActionable(
+        enabled: fields.enabled,
+        frame: fields.rect,
+        viewport: viewport
+      ),
+      depth: depth, parentIndex: parentIndex, hiddenContentAbove: nil, hiddenContentBelow: nil,
+      actions: fields.actions)
   }
 
   private func privateAXPresentationString(_ value: Any?) -> String {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift

@@ -199,16 +199,9 @@ extension RunnerTests {
       }
       guard input.isReadyForMoreMediaData else { return }
       guard let pixelBuffer = makePixelBuffer(from: cgImage) else { return }
-      let nowUptime = ProcessInfo.processInfo.systemUptime
-      if recordingStartUptime == nil {
-        recordingStartUptime = nowUptime
-      }
-      let elapsed = max(0, nowUptime - (recordingStartUptime ?? nowUptime))
+      let candidateTimestampValue = timestampCandidateValue(for: ProcessInfo.processInfo.systemUptime)
+      let timestampValue = monotonicTimestampValue(for: candidateTimestampValue)
       let timescale = effectiveFps
-      var timestampValue = Int64((elapsed * Double(timescale)).rounded(.down))
-      if timestampValue <= lastTimestampValue {
-        timestampValue = lastTimestampValue + 1
-      }
       let timestamp = CMTime(value: timestampValue, timescale: timescale)
       if !adaptor.append(pixelBuffer, withPresentationTime: timestamp) {
         startError = writer.error ?? NSError(
@@ -221,6 +214,20 @@ extension RunnerTests {
       lastTimestampValue = timestampValue
     }
 
+    private func timestampCandidateValue(for nowUptime: TimeInterval) -> Int64 {
+      let startUptime = recordingStartUptime ?? nowUptime
+      recordingStartUptime = startUptime
+      let elapsed = max(0, nowUptime - startUptime)
+      return Int64((elapsed * Double(effectiveFps)).rounded(.down))
+    }
+
+    private func monotonicTimestampValue(for candidateTimestampValue: Int64) -> Int64 {
+      if candidateTimestampValue <= lastTimestampValue {
+        return lastTimestampValue + 1
+      }
+      return candidateTimestampValue
+    }
+
     private func shouldStop() -> Bool {
       lock.lock()
       defer { lock.unlock() }
@@ -261,3 +268,4 @@ extension RunnerTests {
 
   }
 }
+

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift

@@ -19,14 +19,29 @@ struct RunnerScrollGesturePlan {
   let travelPixels: Double
 }
 
+enum RunnerScrollDirection: String {
+  case up
+  case down
+  case left
+  case right
+
+  var isVertical: Bool {
+    self == .up || self == .down
+  }
+}
+
 private let runnerDefaultScrollAmount = 0.6
-// Both constants are pinned by contracts/fixtures/scroll-gesture.json (`constants`). Scroll gestures
-// stay out of the outer 10% of each axis so a saturated scroll never touches down inside the status
-// bar / Dynamic Island band (#1781 A1).
+private let runnerDefaultIosScrollAmount = 0.65
+private let runnerDefaultIosScrollDurationMs = 400.0
+let runnerDefaultDragDurationMs = 250.0
+// The platform scroll defaults and planner constants are pinned by
+// contracts/fixtures/scroll-gesture.json (`constants`). Scroll gestures stay out of the outer 10%
+// of each axis so a saturated scroll never touches down inside the status bar / Dynamic Island band
+// (#1781 A1).
 private let runnerDefaultEdgePaddingFraction = 0.1
 
 func runnerScrollGesturePlan(
-  direction: String,
+  direction: RunnerScrollDirection,
   amount: Double?,
   pixels: Double?,
   referenceWidth: Double,
@@ -37,7 +52,7 @@ func runnerScrollGesturePlan(
   // this only triggers for non-daemon wire clients.
   if let amount, !(amount.isFinite && amount > 0) { return nil }
   if let pixels, !(pixels.isFinite && pixels > 0) { return nil }
-  let axisLength = (direction == "up" || direction == "down") ? referenceHeight : referenceWidth
+  let axisLength = direction.isVertical ? referenceHeight : referenceWidth
   let requestedAmount = amount ?? runnerDefaultScrollAmount
   let requestedPixels: Double =
     pixels.map { max(1, $0.rounded()) } ?? (axisLength * requestedAmount).rounded()
@@ -53,16 +68,32 @@ func runnerScrollGesturePlan(
   }
 
   switch direction {
-  case "up":
+  case .up:
     return plan(centerX, centerY - halfTravel, centerX, centerY + halfTravel)
-  case "down":
+  case .down:
     return plan(centerX, centerY + halfTravel, centerX, centerY - halfTravel)
-  case "left":
+  case .left:
     return plan(centerX - halfTravel, centerY, centerX + halfTravel, centerY)
-  case "right":
+  case .right:
     return plan(centerX + halfTravel, centerY, centerX - halfTravel, centerY)
-  default:
-    return nil
   }
 }
 
+struct RunnerDragCommandDefaults {
+  let scrollAmount: Double?
+  let durationMs: Double
+}
+
+func runnerDragCommandDefaults(_ command: Command) -> RunnerDragCommandDefaults {
+  if command.command == .scroll {
+    return RunnerDragCommandDefaults(
+      scrollAmount: command.pixels == nil ? (command.amount ?? runnerDefaultIosScrollAmount) : command.amount,
+      durationMs: command.durationMs ?? runnerDefaultIosScrollDurationMs
+    )
+  }
+  return RunnerDragCommandDefaults(
+    scrollAmount: nil,
+    durationMs: command.durationMs ?? runnerDefaultDragDurationMs
+  )
+}
+