agent-device 0.20.9 → 0.21.0

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentation.swift

@@ -0,0 +1,153 @@
+import Foundation
+import CoreGraphics
+
+public enum SnapshotPresentation {
+  public static func present(
+    _ acquisition: SnapshotAcquisition,
+    options: PresentationOptions
+  ) throws -> SnapshotPresentationResult? {
+    let requested = captureHint(for: options)
+    guard acquisition.hint.projection == requested.projection else {
+      return nil
+    }
+    switch requested.projection {
+    case .regular:
+      return try presentRegular(acquisition, options: options)
+    case .raw:
+      return presentRaw(acquisition, options: options)
+    }
+  }
+
+  public static func presentRegular(
+    _ acquisition: SnapshotAcquisition,
+    options: PresentationOptions,
+    policy: SnapshotVisibilityFold.Policy = .platformDefault
+  ) throws -> SnapshotPresentationResult {
+    let folded = SnapshotVisibilityFold.fold(
+      acquisition.nodes,
+      viewport: acquisition.viewport,
+      interactiveOnly: options.interactiveOnly,
+      policy: policy
+    )
+    try SnapshotPresentationInvariant.validateRegular(
+      folded,
+      viewport: acquisition.viewport,
+      policy: policy
+    )
+    return project(
+      folded,
+      acquisition: acquisition,
+      options: options,
+      projection: .regular
+    )
+  }
+
+  public static func presentRaw(
+    _ acquisition: SnapshotAcquisition,
+    options: PresentationOptions
+  ) -> SnapshotPresentationResult {
+    project(
+      acquisition.nodes.map(SnapshotPresentationNode.reported),
+      acquisition: acquisition,
+      options: options,
+      projection: .raw
+    )
+  }
+
+  public static func captureHint(for options: PresentationOptions) -> CaptureHint {
+    let scoped = SnapshotScopePolicy.isActive(options.scope)
+    let projection: CaptureHint.Projection = options.raw ? .raw : .regular
+    return CaptureHint(
+      projection: projection,
+      depth: scoped || projection == .regular ? nil : options.depth,
+      regularPresentedDepth: scoped || projection == .raw ? nil : options.depth,
+      interactiveOnly: projection == .raw ? false : options.interactiveOnly,
+      customActions: options.customActions
+    )
+  }
+
+  public static func shouldAcquireChildren(
+    for hint: CaptureHint,
+    rawDepth: Int,
+    regularPresentedDepth: Int
+  ) -> Bool {
+    if let rawLimit = hint.rawTraversalDepth {
+      return rawDepth < rawLimit
+    }
+    if let presentedLimit = hint.regularPresentedDepth {
+      return regularPresentedDepth < presentedLimit
+    }
+    return true
+  }
+
+  public static func regularTraversalTransition(
+    for raw: RawAXNode,
+    parentPresentedDepth: Int,
+    parentTraversal: SnapshotVisibilityFold.TraversalState,
+    hint: CaptureHint,
+    rawDepth: Int,
+    viewport: CGRect,
+    hasChildren: Bool,
+    isDuplicate: Bool,
+    policy: SnapshotVisibilityFold.Policy = .platformDefault
+  ) -> (
+    presentedDepth: Int,
+    traversal: SnapshotVisibilityFold.TraversalState,
+    shouldVisitChildren: Bool
+  ) {
+    guard hint.regularPresentedDepth != nil else {
+      return (
+        parentPresentedDepth,
+        parentTraversal,
+        shouldAcquireChildren(
+          for: hint,
+          rawDepth: rawDepth,
+          regularPresentedDepth: parentPresentedDepth
+        )
+      )
+    }
+
+    let visibility = SnapshotVisibilityFold.traversalDecision(
+      for: raw,
+      parent: parentTraversal,
+      viewport: viewport,
+      interactiveOnly: hint.interactiveOnly,
+      hasChildren: hasChildren,
+      policy: policy
+    )
+    let presentedDepth = regularPresentedDepth(
+      for: raw,
+      parentPresentedDepth: parentPresentedDepth,
+      visibility: visibility
+    )
+    let nextPresentedDepth = isDuplicate ? parentPresentedDepth : presentedDepth
+    let nextTraversal = isDuplicate ? parentTraversal : visibility.descendants
+    let descendantsMayBeVisible = isDuplicate
+      ? parentTraversal.descendantsMayBeVisible
+      : visibility.descendantsMayBeVisible
+    return (
+      nextPresentedDepth,
+      nextTraversal,
+      shouldAcquireChildren(
+        for: hint,
+        rawDepth: rawDepth,
+        regularPresentedDepth: nextPresentedDepth
+      ) && descendantsMayBeVisible
+    )
+  }
+
+  public static func regularPresentedDepth(
+    for raw: RawAXNode,
+    parentPresentedDepth: Int,
+    visibility: SnapshotVisibilityFold.TraversalDecision
+  ) -> Int {
+    guard raw.parentIndex != nil else { return 0 }
+    return visibility.isIncluded && isEligibleForRegularPresentation(raw)
+      ? parentPresentedDepth + 1
+      : parentPresentedDepth
+  }
+
+  public static func singleElementRead(_ node: RawAXNode) -> PresentedNode {
+    PresentedNode(presenting: node)
+  }
+}

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationFailure.swift

@@ -0,0 +1,27 @@
+import Foundation
+
+public enum SnapshotPresentationFailure: Error {
+  case regularNodeOutsideCumulativeClip(index: Int, frame: SnapshotRect, clip: SnapshotRect)
+  case regularDegenerateNodeIsActionable(index: Int, frame: SnapshotRect)
+
+  public var code: String {
+    "IOS_SNAPSHOT_PRESENTATION_FAILED"
+  }
+
+  public var qualityReasonCode: String {
+    "presentation-failed"
+  }
+
+  public var message: String {
+    switch self {
+    case .regularNodeOutsideCumulativeClip(let index, _, _):
+      return "regular snapshot node \(index) escaped its cumulative clip"
+    case .regularDegenerateNodeIsActionable(let index, _):
+      return "regular snapshot node \(index) with a frameless or degenerate frame was marked hittable"
+    }
+  }
+
+  public var hint: String {
+    "This is a runner presentation bug: report it with the failing command and the app under test."
+  }
+}

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationInvariant.swift

@@ -0,0 +1,91 @@
+import Foundation
+import CoreGraphics
+
+public enum SnapshotPresentationInvariant {
+  struct ValidationStats: Equatable {
+    let parentClipLookups: Int
+
+    init(parentClipLookups: Int) {
+      self.parentClipLookups = parentClipLookups
+    }
+  }
+
+  public static func validateRegular(
+    _ nodes: [SnapshotPresentationNode],
+    viewport: CGRect,
+    policy: SnapshotVisibilityFold.Policy
+  ) throws {
+    _ = try validateRegularWithStats(nodes, viewport: viewport, policy: policy)
+  }
+
+  static func validateRegularWithStats(
+    _ nodes: [SnapshotPresentationNode],
+    viewport: CGRect,
+    policy: SnapshotVisibilityFold.Policy
+  ) throws -> ValidationStats {
+    var parentClipLookups = 0
+    var clipIncludingNodeByIndex: [Int: CGRect] = [:]
+    clipIncludingNodeByIndex.reserveCapacity(nodes.count)
+
+    for node in nodes {
+      let ancestorClip: CGRect
+      if let parentIndex = node.raw.parentIndex {
+        parentClipLookups += 1
+        ancestorClip = clipIncludingNodeByIndex[parentIndex] ?? viewport
+      } else {
+        ancestorClip = viewport
+      }
+
+      let frame = node.effectiveRect.cgRect
+      let clipIncludingNode: CGRect
+      if policy == .cursorProjected,
+        SnapshotVisibilityFold.scrollContainerTypeNames.contains(node.raw.type),
+        !frame.isNull,
+        !frame.isEmpty
+      {
+        clipIncludingNode = frame
+      } else {
+        clipIncludingNode = ancestorClip
+      }
+      clipIncludingNodeByIndex[node.raw.index] = clipIncludingNode
+
+      guard !frame.isNull, !frame.isEmpty else {
+        if node.raw.hittable {
+          throw SnapshotPresentationFailure.regularDegenerateNodeIsActionable(
+            index: node.raw.index,
+            frame: node.effectiveRect
+          )
+        }
+        continue
+      }
+
+      guard contains(frame, in: ancestorClip) else {
+        throw SnapshotPresentationFailure.regularNodeOutsideCumulativeClip(
+          index: node.raw.index,
+          frame: node.effectiveRect,
+          clip: snapshotRect(from: ancestorClip)
+        )
+      }
+    }
+
+    return ValidationStats(parentClipLookups: parentClipLookups)
+  }
+
+  private static func contains(_ frame: CGRect, in clip: CGRect) -> Bool {
+    guard !clip.isNull, !clip.isEmpty else { return false }
+    let tolerance = 0.0001
+    return frame.minX >= clip.minX - tolerance
+      && frame.minY >= clip.minY - tolerance
+      && frame.maxX <= clip.maxX + tolerance
+      && frame.maxY <= clip.maxY + tolerance
+  }
+
+  private static func snapshotRect(from rect: CGRect) -> SnapshotRect {
+    SnapshotRect(
+      x: Double(rect.minX),
+      y: Double(rect.minY),
+      width: Double(rect.width),
+      height: Double(rect.height)
+    )
+  }
+}

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift

@@ -0,0 +1,161 @@
+import Foundation
+
+extension SnapshotPresentation {
+  static let eligibleInteractiveTypes: Set<String> = [
+    "Button",
+    "Cell",
+    "CheckBox",
+    "CollectionView",
+    "Link",
+    "MenuItem",
+    "Picker",
+    "SearchField",
+    "SecureTextField",
+    "SegmentedControl",
+    "Slider",
+    "ScrollView",
+    "Stepper",
+    "Switch",
+    "TabBar",
+    "Table",
+    "TextField",
+    "TextView",
+    "WebView",
+  ]
+
+  static func project(
+    _ projectionNodes: [SnapshotPresentationNode],
+    acquisition: SnapshotAcquisition,
+    options: PresentationOptions,
+    projection: CaptureHint.Projection
+  ) -> SnapshotPresentationResult {
+    let scopedRawNodes = applyScope(to: projectionNodes, options: options, projection: projection)
+    let nodes = presentedNodes(
+      from: scopedRawNodes,
+      projection: projection,
+      maximumDepth: projection == .regular ? options.depth : nil
+    )
+    let qualityNodes: [PresentedNode]? = SnapshotScopePolicy.isActive(options.scope)
+      ? presentedNodes(from: projectionNodes, projection: projection, maximumDepth: nil)
+      : nil
+    return SnapshotPresentationResult(
+      nodes: nodes,
+      truncated: acquisition.truncated,
+      effectiveDepth: acquisition.effectiveDepth,
+      customActions: acquisition.customActions,
+      qualityNodes: qualityNodes
+    )
+  }
+
+  static func presentedNodes(
+    from rawNodes: [SnapshotPresentationNode],
+    projection: CaptureHint.Projection,
+    maximumDepth: Int? = nil
+  ) -> [PresentedNode] {
+    if projection == .raw {
+      return rawNodes.map { PresentedNode(presenting: $0) }
+    }
+
+    var nodes: [PresentedNode] = []
+    var nearestPresentedNodeByRawIndex: [Int: (index: Int, depth: Int)] = [:]
+    for node in rawNodes {
+      let raw = node.raw
+      let presentedParent = raw.parentIndex.flatMap { nearestPresentedNodeByRawIndex[$0] }
+      guard isEligibleForRegularPresentation(raw) else {
+        if let presentedParent {
+          nearestPresentedNodeByRawIndex[raw.index] = presentedParent
+        }
+        continue
+      }
+
+      let presentedIndex = nodes.count
+      let presentedDepth = presentedParent.map { $0.depth + 1 } ?? 0
+      if let maximumDepth, presentedDepth > maximumDepth {
+        if let presentedParent {
+          nearestPresentedNodeByRawIndex[raw.index] = presentedParent
+        }
+        continue
+      }
+      nearestPresentedNodeByRawIndex[raw.index] = (presentedIndex, presentedDepth)
+      nodes.append(
+        PresentedNode(
+          presenting: raw,
+          rect: node.effectiveRect,
+          index: presentedIndex,
+          depth: presentedDepth,
+          parentIndex: presentedParent?.index
+        )
+      )
+    }
+    return nodes
+  }
+
+  static func applyScope(
+    to rawNodes: [SnapshotPresentationNode],
+    options: PresentationOptions,
+    projection: CaptureHint.Projection
+  ) -> [SnapshotPresentationNode] {
+    switch SnapshotScopePolicy.select(
+      fromPreorder: rawNodes,
+      scope: options.scope,
+      depth: { $0.raw.depth },
+      semanticValues: { [$0.raw.label, $0.raw.identifier, $0.raw.value] },
+      subtreeContributes: { range in
+        projection == .raw || rawNodes[range].contains { isEligibleForRegularPresentation($0.raw) }
+      }
+    ) {
+    case .unscoped:
+      return rawNodes
+    case .missing:
+      return []
+    case .matched(let startIndex):
+      let startDepth = rawNodes[startIndex].raw.depth
+      let range = SnapshotScopePolicy.subtreeRange(
+        from: startIndex,
+        in: rawNodes,
+        depth: { $0.raw.depth }
+      )
+      let maxDepth = options.depth ?? Int.max
+      let scopedNodes = projection == .raw
+        ? Array(rawNodes[range]).filter { $0.raw.depth - startDepth <= maxDepth }
+        : Array(rawNodes[range])
+      return reindex(scopedNodes, depthOffset: startDepth)
+    }
+  }
+
+  static func reindex(
+    _ nodes: [SnapshotPresentationNode],
+    depthOffset: Int
+  ) -> [SnapshotPresentationNode] {
+    let indexMap = Dictionary(
+      uniqueKeysWithValues: nodes.enumerated().map { ($0.element.raw.index, $0.offset) })
+    return nodes.enumerated().map { offset, node in
+      let raw = node.raw
+      return SnapshotPresentationNode(
+        raw: RawAXNode(
+          index: offset,
+          type: raw.type,
+          label: raw.label,
+          identifier: raw.identifier,
+          value: raw.value,
+          rect: raw.rect,
+          enabled: raw.enabled,
+          focused: raw.focused,
+          selected: raw.selected,
+          hittable: raw.hittable,
+          depth: max(0, raw.depth - depthOffset),
+          parentIndex: raw.parentIndex.flatMap { indexMap[$0] },
+          hiddenContentAbove: raw.hiddenContentAbove,
+          hiddenContentBelow: raw.hiddenContentBelow,
+          actions: raw.actions
+        ),
+        effectiveRect: node.effectiveRect
+      )
+    }
+  }
+
+  static func isEligibleForRegularPresentation(_ node: RawAXNode) -> Bool {
+    if node.parentIndex == nil { return true }
+    return eligibleInteractiveTypes.contains(node.type) || node.hasSemanticContent
+  }
+}

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotScopePolicy.swift

@@ -0,0 +1,53 @@
+import Foundation
+
+public enum SnapshotScopeSelection: Equatable {
+  case unscoped
+  case matched(Int)
+  case missing
+}
+
+public enum SnapshotScopePolicy {
+  public static func select<Node>(
+    fromPreorder nodes: [Node],
+    scope: String?,
+    depth: (Node) -> Int,
+    semanticValues: (Node) -> [String?],
+    subtreeContributes: (Range<Int>) -> Bool
+  ) -> SnapshotScopeSelection {
+    guard let query = normalized(scope) else { return .unscoped }
+    for (index, node) in nodes.enumerated() {
+      guard semanticValues(node).contains(where: { value in
+        value?.lowercased().contains(query) == true
+      }) else { continue }
+      let range = subtreeRange(from: index, in: nodes, depth: depth)
+      if subtreeContributes(range) {
+        return .matched(index)
+      }
+    }
+    return .missing
+  }
+
+  public static func isActive(_ scope: String?) -> Bool {
+    normalized(scope) != nil
+  }
+
+  public static func subtreeRange<Node>(
+    from start: Int,
+    in nodes: [Node],
+    depth: (Node) -> Int
+  ) -> Range<Int> {
+    let rootDepth = depth(nodes[start])
+    var end = start + 1
+    while end < nodes.count, depth(nodes[end]) > rootDepth {
+      end += 1
+    }
+    return start..<end
+  }
+
+  private static func normalized(_ scope: String?) -> String? {
+    guard let query = scope?.trimmingCharacters(in: .whitespacesAndNewlines), !query.isEmpty else {
+      return nil
+    }
+    return query.lowercased()
+  }
+}