agent-device 0.20.10 → 0.21.1

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift

@@ -0,0 +1,259 @@
+import Foundation
+import CoreGraphics
+
+public struct SnapshotRect: Codable, Equatable {
+  public let x: Double
+  public let y: Double
+  public let width: Double
+  public let height: Double
+
+  public init(x: Double, y: Double, width: Double, height: Double) {
+    self.x = x
+    self.y = y
+    self.width = width
+    self.height = height
+  }
+
+  public var cgRect: CGRect {
+    CGRect(x: x, y: y, width: width, height: height)
+  }
+}
+
+public struct RawAXNode: Equatable {
+  public let index: Int
+  public let type: String
+  public let label: String?
+  public let identifier: String?
+  public let value: String?
+  public let rect: SnapshotRect
+  public let enabled: Bool
+  public let focused: Bool?
+  public let selected: Bool?
+  public let hittable: Bool
+  public let depth: Int
+  public let parentIndex: Int?
+  public let hiddenContentAbove: Bool?
+  public let hiddenContentBelow: Bool?
+  public var actions: [String]?
+
+  public init(
+    index: Int,
+    type: String,
+    label: String?,
+    identifier: String?,
+    value: String?,
+    rect: SnapshotRect,
+    enabled: Bool,
+    focused: Bool?,
+    selected: Bool?,
+    hittable: Bool,
+    depth: Int,
+    parentIndex: Int?,
+    hiddenContentAbove: Bool?,
+    hiddenContentBelow: Bool?,
+    actions: [String]? = nil
+  ) {
+    self.index = index
+    self.type = type
+    self.label = label
+    self.identifier = identifier
+    self.value = value
+    self.rect = rect
+    self.enabled = enabled
+    self.focused = focused
+    self.selected = selected
+    self.hittable = hittable
+    self.depth = depth
+    self.parentIndex = parentIndex
+    self.hiddenContentAbove = hiddenContentAbove
+    self.hiddenContentBelow = hiddenContentBelow
+    self.actions = actions
+  }
+
+  public var hasSemanticContent: Bool {
+    [label, identifier, value].contains {
+      !($0?.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty ?? true)
+    }
+  }
+}
+
+public struct CaptureHint: Equatable {
+  public enum Projection: String {
+    case regular
+    case raw
+  }
+
+  public let projection: Projection
+  public let depth: Int?
+  public let regularPresentedDepth: Int?
+  public let interactiveOnly: Bool
+  public let customActions: Bool
+
+  public init(
+    projection: Projection,
+    depth: Int?,
+    regularPresentedDepth: Int?,
+    interactiveOnly: Bool,
+    customActions: Bool
+  ) {
+    self.projection = projection
+    self.depth = depth
+    self.regularPresentedDepth = regularPresentedDepth
+    self.interactiveOnly = interactiveOnly
+    self.customActions = customActions
+  }
+
+  public var rawTraversalDepth: Int? {
+    projection == .raw ? depth : nil
+  }
+
+  public var isRaw: Bool {
+    projection == .raw
+  }
+}
+
+public struct PresentationOptions: Equatable {
+  public let interactiveOnly: Bool
+  public let depth: Int?
+  public let scope: String?
+  public let raw: Bool
+  public var preferredBackend: String?
+  public var customActions: Bool
+
+  public init(
+    interactiveOnly: Bool,
+    depth: Int?,
+    scope: String?,
+    raw: Bool,
+    preferredBackend: String? = nil,
+    customActions: Bool = false
+  ) {
+    self.interactiveOnly = interactiveOnly
+    self.depth = depth
+    self.scope = scope
+    self.raw = raw
+    self.preferredBackend = preferredBackend
+    self.customActions = customActions
+  }
+}
+
+public struct SnapshotAcquisition {
+  public let hint: CaptureHint
+  public let nodes: [RawAXNode]
+  public let truncated: Bool
+  public let effectiveDepth: Int?
+  public var customActions: SnapshotCustomActionCoverage?
+  public let viewport: CGRect
+
+  public init(
+    hint: CaptureHint,
+    nodes: [RawAXNode],
+    truncated: Bool,
+    effectiveDepth: Int?,
+    customActions: SnapshotCustomActionCoverage? = nil,
+    viewport: CGRect
+  ) {
+    self.hint = hint
+    self.nodes = nodes
+    self.truncated = truncated
+    self.effectiveDepth = effectiveDepth
+    self.customActions = customActions
+    self.viewport = viewport
+  }
+}
+
+public struct SnapshotCustomActionCoverage: Codable, Equatable {
+  public let read: Int
+  public let candidates: Int
+  public let truncated: Int
+  public let blocked: Bool
+
+  public init(read: Int, candidates: Int, truncated: Int, blocked: Bool) {
+    self.read = read
+    self.candidates = candidates
+    self.truncated = truncated
+    self.blocked = blocked
+  }
+}
+
+public struct SnapshotPresentationNode {
+  public let raw: RawAXNode
+  public let effectiveRect: SnapshotRect
+
+  public init(raw: RawAXNode, effectiveRect: SnapshotRect) {
+    self.raw = raw
+    self.effectiveRect = effectiveRect
+  }
+
+  public static func reported(_ raw: RawAXNode) -> Self {
+    Self(raw: raw, effectiveRect: raw.rect)
+  }
+}
+
+public struct PresentedNode: Codable, Equatable {
+  public let index: Int
+  public let type: String
+  public let label: String?
+  public let identifier: String?
+  public let value: String?
+  public let rect: SnapshotRect
+  public let enabled: Bool
+  public let focused: Bool?
+  public let selected: Bool?
+  public let hittable: Bool
+  public let depth: Int
+  public let parentIndex: Int?
+  public let hiddenContentAbove: Bool?
+  public let hiddenContentBelow: Bool?
+  public let actions: [String]?
+
+  internal init(
+    presenting raw: RawAXNode,
+    rect: SnapshotRect? = nil,
+    index: Int? = nil,
+    depth: Int? = nil,
+    parentIndex: Int?? = nil
+  ) {
+    self.index = index ?? raw.index
+    self.type = raw.type
+    self.label = raw.label
+    self.identifier = raw.identifier
+    self.value = raw.value
+    self.rect = rect ?? raw.rect
+    self.enabled = raw.enabled
+    self.focused = raw.focused
+    self.selected = raw.selected
+    self.hittable = raw.hittable
+    self.depth = depth ?? raw.depth
+    self.parentIndex = parentIndex ?? raw.parentIndex
+    self.hiddenContentAbove = raw.hiddenContentAbove
+    self.hiddenContentBelow = raw.hiddenContentBelow
+    self.actions = raw.actions
+  }
+
+  internal init(presenting node: SnapshotPresentationNode) {
+    self.init(presenting: node.raw, rect: node.effectiveRect)
+  }
+}
+
+public struct SnapshotPresentationResult {
+  public let nodes: [PresentedNode]
+  public let truncated: Bool
+  public let effectiveDepth: Int?
+  public let customActions: SnapshotCustomActionCoverage?
+  public let qualityNodes: [PresentedNode]?
+
+  public init(
+    nodes: [PresentedNode],
+    truncated: Bool,
+    effectiveDepth: Int?,
+    customActions: SnapshotCustomActionCoverage?,
+    qualityNodes: [PresentedNode]?
+  ) {
+    self.nodes = nodes
+    self.truncated = truncated
+    self.effectiveDepth = effectiveDepth
+    self.customActions = customActions
+    self.qualityNodes = qualityNodes
+  }
+}

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentation.swift

@@ -0,0 +1,153 @@
+import Foundation
+import CoreGraphics
+
+public enum SnapshotPresentation {
+  public static func present(
+    _ acquisition: SnapshotAcquisition,
+    options: PresentationOptions
+  ) throws -> SnapshotPresentationResult? {
+    let requested = captureHint(for: options)
+    guard acquisition.hint.projection == requested.projection else {
+      return nil
+    }
+    switch requested.projection {
+    case .regular:
+      return try presentRegular(acquisition, options: options)
+    case .raw:
+      return presentRaw(acquisition, options: options)
+    }
+  }
+
+  public static func presentRegular(
+    _ acquisition: SnapshotAcquisition,
+    options: PresentationOptions,
+    policy: SnapshotVisibilityFold.Policy = .platformDefault
+  ) throws -> SnapshotPresentationResult {
+    let folded = SnapshotVisibilityFold.fold(
+      acquisition.nodes,
+      viewport: acquisition.viewport,
+      interactiveOnly: options.interactiveOnly,
+      policy: policy
+    )
+    try SnapshotPresentationInvariant.validateRegular(
+      folded,
+      viewport: acquisition.viewport,
+      policy: policy
+    )
+    return project(
+      folded,
+      acquisition: acquisition,
+      options: options,
+      projection: .regular
+    )
+  }
+
+  public static func presentRaw(
+    _ acquisition: SnapshotAcquisition,
+    options: PresentationOptions
+  ) -> SnapshotPresentationResult {
+    project(
+      acquisition.nodes.map(SnapshotPresentationNode.reported),
+      acquisition: acquisition,
+      options: options,
+      projection: .raw
+    )
+  }
+
+  public static func captureHint(for options: PresentationOptions) -> CaptureHint {
+    let scoped = SnapshotScopePolicy.isActive(options.scope)
+    let projection: CaptureHint.Projection = options.raw ? .raw : .regular
+    return CaptureHint(
+      projection: projection,
+      depth: scoped || projection == .regular ? nil : options.depth,
+      regularPresentedDepth: scoped || projection == .raw ? nil : options.depth,
+      interactiveOnly: projection == .raw ? false : options.interactiveOnly,
+      customActions: options.customActions
+    )
+  }
+
+  public static func shouldAcquireChildren(
+    for hint: CaptureHint,
+    rawDepth: Int,
+    regularPresentedDepth: Int
+  ) -> Bool {
+    if let rawLimit = hint.rawTraversalDepth {
+      return rawDepth < rawLimit
+    }
+    if let presentedLimit = hint.regularPresentedDepth {
+      return regularPresentedDepth < presentedLimit
+    }
+    return true
+  }
+
+  public static func regularTraversalTransition(
+    for raw: RawAXNode,
+    parentPresentedDepth: Int,
+    parentTraversal: SnapshotVisibilityFold.TraversalState,
+    hint: CaptureHint,
+    rawDepth: Int,
+    viewport: CGRect,
+    hasChildren: Bool,
+    isDuplicate: Bool,
+    policy: SnapshotVisibilityFold.Policy = .platformDefault
+  ) -> (
+    presentedDepth: Int,
+    traversal: SnapshotVisibilityFold.TraversalState,
+    shouldVisitChildren: Bool
+  ) {
+    guard hint.regularPresentedDepth != nil else {
+      return (
+        parentPresentedDepth,
+        parentTraversal,
+        shouldAcquireChildren(
+          for: hint,
+          rawDepth: rawDepth,
+          regularPresentedDepth: parentPresentedDepth
+        )
+      )
+    }
+
+    let visibility = SnapshotVisibilityFold.traversalDecision(
+      for: raw,
+      parent: parentTraversal,
+      viewport: viewport,
+      interactiveOnly: hint.interactiveOnly,
+      hasChildren: hasChildren,
+      policy: policy
+    )
+    let presentedDepth = regularPresentedDepth(
+      for: raw,
+      parentPresentedDepth: parentPresentedDepth,
+      visibility: visibility
+    )
+    let nextPresentedDepth = isDuplicate ? parentPresentedDepth : presentedDepth
+    let nextTraversal = isDuplicate ? parentTraversal : visibility.descendants
+    let descendantsMayBeVisible = isDuplicate
+      ? parentTraversal.descendantsMayBeVisible
+      : visibility.descendantsMayBeVisible
+    return (
+      nextPresentedDepth,
+      nextTraversal,
+      shouldAcquireChildren(
+        for: hint,
+        rawDepth: rawDepth,
+        regularPresentedDepth: nextPresentedDepth
+      ) && descendantsMayBeVisible
+    )
+  }
+
+  public static func regularPresentedDepth(
+    for raw: RawAXNode,
+    parentPresentedDepth: Int,
+    visibility: SnapshotVisibilityFold.TraversalDecision
+  ) -> Int {
+    guard raw.parentIndex != nil else { return 0 }
+    return visibility.isIncluded && isEligibleForRegularPresentation(raw)
+      ? parentPresentedDepth + 1
+      : parentPresentedDepth
+  }
+
+  public static func singleElementRead(_ node: RawAXNode) -> PresentedNode {
+    PresentedNode(presenting: node)
+  }
+}

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationFailure.swift

@@ -0,0 +1,27 @@
+import Foundation
+
+public enum SnapshotPresentationFailure: Error {
+  case regularNodeOutsideCumulativeClip(index: Int, frame: SnapshotRect, clip: SnapshotRect)
+  case regularDegenerateNodeIsActionable(index: Int, frame: SnapshotRect)
+
+  public var code: String {
+    "IOS_SNAPSHOT_PRESENTATION_FAILED"
+  }
+
+  public var qualityReasonCode: String {
+    "presentation-failed"
+  }
+
+  public var message: String {
+    switch self {
+    case .regularNodeOutsideCumulativeClip(let index, _, _):
+      return "regular snapshot node \(index) escaped its cumulative clip"
+    case .regularDegenerateNodeIsActionable(let index, _):
+      return "regular snapshot node \(index) with a frameless or degenerate frame was marked hittable"
+    }
+  }
+
+  public var hint: String {
+    "This is a runner presentation bug: report it with the failing command and the app under test."
+  }
+}

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationInvariant.swift

@@ -0,0 +1,91 @@
+import Foundation
+import CoreGraphics
+
+public enum SnapshotPresentationInvariant {
+  struct ValidationStats: Equatable {
+    let parentClipLookups: Int
+
+    init(parentClipLookups: Int) {
+      self.parentClipLookups = parentClipLookups
+    }
+  }
+
+  public static func validateRegular(
+    _ nodes: [SnapshotPresentationNode],
+    viewport: CGRect,
+    policy: SnapshotVisibilityFold.Policy
+  ) throws {
+    _ = try validateRegularWithStats(nodes, viewport: viewport, policy: policy)
+  }
+
+  static func validateRegularWithStats(
+    _ nodes: [SnapshotPresentationNode],
+    viewport: CGRect,
+    policy: SnapshotVisibilityFold.Policy
+  ) throws -> ValidationStats {
+    var parentClipLookups = 0
+    var clipIncludingNodeByIndex: [Int: CGRect] = [:]
+    clipIncludingNodeByIndex.reserveCapacity(nodes.count)
+
+    for node in nodes {
+      let ancestorClip: CGRect
+      if let parentIndex = node.raw.parentIndex {
+        parentClipLookups += 1
+        ancestorClip = clipIncludingNodeByIndex[parentIndex] ?? viewport
+      } else {
+        ancestorClip = viewport
+      }
+
+      let frame = node.effectiveRect.cgRect
+      let clipIncludingNode: CGRect
+      if policy == .cursorProjected,
+        SnapshotVisibilityFold.scrollContainerTypeNames.contains(node.raw.type),
+        !frame.isNull,
+        !frame.isEmpty
+      {
+        clipIncludingNode = frame
+      } else {
+        clipIncludingNode = ancestorClip
+      }
+      clipIncludingNodeByIndex[node.raw.index] = clipIncludingNode
+
+      guard !frame.isNull, !frame.isEmpty else {
+        if node.raw.hittable {
+          throw SnapshotPresentationFailure.regularDegenerateNodeIsActionable(
+            index: node.raw.index,
+            frame: node.effectiveRect
+          )
+        }
+        continue
+      }
+
+      guard contains(frame, in: ancestorClip) else {
+        throw SnapshotPresentationFailure.regularNodeOutsideCumulativeClip(
+          index: node.raw.index,
+          frame: node.effectiveRect,
+          clip: snapshotRect(from: ancestorClip)
+        )
+      }
+    }
+
+    return ValidationStats(parentClipLookups: parentClipLookups)
+  }
+
+  private static func contains(_ frame: CGRect, in clip: CGRect) -> Bool {
+    guard !clip.isNull, !clip.isEmpty else { return false }
+    let tolerance = 0.0001
+    return frame.minX >= clip.minX - tolerance
+      && frame.minY >= clip.minY - tolerance
+      && frame.maxX <= clip.maxX + tolerance
+      && frame.maxY <= clip.maxY + tolerance
+  }
+
+  private static func snapshotRect(from rect: CGRect) -> SnapshotRect {
+    SnapshotRect(
+      x: Double(rect.minX),
+      y: Double(rect.minY),
+      width: Double(rect.width),
+      height: Double(rect.height)
+    )
+  }
+}

package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationProjection.swift

@@ -0,0 +1,161 @@
+import Foundation
+
+extension SnapshotPresentation {
+  static let eligibleInteractiveTypes: Set<String> = [
+    "Button",
+    "Cell",
+    "CheckBox",
+    "CollectionView",
+    "Link",
+    "MenuItem",
+    "Picker",
+    "SearchField",
+    "SecureTextField",
+    "SegmentedControl",
+    "Slider",
+    "ScrollView",
+    "Stepper",
+    "Switch",
+    "TabBar",
+    "Table",
+    "TextField",
+    "TextView",
+    "WebView",
+  ]
+
+  static func project(
+    _ projectionNodes: [SnapshotPresentationNode],
+    acquisition: SnapshotAcquisition,
+    options: PresentationOptions,
+    projection: CaptureHint.Projection
+  ) -> SnapshotPresentationResult {
+    let scopedRawNodes = applyScope(to: projectionNodes, options: options, projection: projection)
+    let nodes = presentedNodes(
+      from: scopedRawNodes,
+      projection: projection,
+      maximumDepth: projection == .regular ? options.depth : nil
+    )
+    let qualityNodes: [PresentedNode]? = SnapshotScopePolicy.isActive(options.scope)
+      ? presentedNodes(from: projectionNodes, projection: projection, maximumDepth: nil)
+      : nil
+    return SnapshotPresentationResult(
+      nodes: nodes,
+      truncated: acquisition.truncated,
+      effectiveDepth: acquisition.effectiveDepth,
+      customActions: acquisition.customActions,
+      qualityNodes: qualityNodes
+    )
+  }
+
+  static func presentedNodes(
+    from rawNodes: [SnapshotPresentationNode],
+    projection: CaptureHint.Projection,
+    maximumDepth: Int? = nil
+  ) -> [PresentedNode] {
+    if projection == .raw {
+      return rawNodes.map { PresentedNode(presenting: $0) }
+    }
+
+    var nodes: [PresentedNode] = []
+    var nearestPresentedNodeByRawIndex: [Int: (index: Int, depth: Int)] = [:]
+    for node in rawNodes {
+      let raw = node.raw
+      let presentedParent = raw.parentIndex.flatMap { nearestPresentedNodeByRawIndex[$0] }
+      guard isEligibleForRegularPresentation(raw) else {
+        if let presentedParent {
+          nearestPresentedNodeByRawIndex[raw.index] = presentedParent
+        }
+        continue
+      }
+
+      let presentedIndex = nodes.count
+      let presentedDepth = presentedParent.map { $0.depth + 1 } ?? 0
+      if let maximumDepth, presentedDepth > maximumDepth {
+        if let presentedParent {
+          nearestPresentedNodeByRawIndex[raw.index] = presentedParent
+        }
+        continue
+      }
+      nearestPresentedNodeByRawIndex[raw.index] = (presentedIndex, presentedDepth)
+      nodes.append(
+        PresentedNode(
+          presenting: raw,
+          rect: node.effectiveRect,
+          index: presentedIndex,
+          depth: presentedDepth,
+          parentIndex: presentedParent?.index
+        )
+      )
+    }
+    return nodes
+  }
+
+  static func applyScope(
+    to rawNodes: [SnapshotPresentationNode],
+    options: PresentationOptions,
+    projection: CaptureHint.Projection
+  ) -> [SnapshotPresentationNode] {
+    switch SnapshotScopePolicy.select(
+      fromPreorder: rawNodes,
+      scope: options.scope,
+      depth: { $0.raw.depth },
+      semanticValues: { [$0.raw.label, $0.raw.identifier, $0.raw.value] },
+      subtreeContributes: { range in
+        projection == .raw || rawNodes[range].contains { isEligibleForRegularPresentation($0.raw) }
+      }
+    ) {
+    case .unscoped:
+      return rawNodes
+    case .missing:
+      return []
+    case .matched(let startIndex):
+      let startDepth = rawNodes[startIndex].raw.depth
+      let range = SnapshotScopePolicy.subtreeRange(
+        from: startIndex,
+        in: rawNodes,
+        depth: { $0.raw.depth }
+      )
+      let maxDepth = options.depth ?? Int.max
+      let scopedNodes = projection == .raw
+        ? Array(rawNodes[range]).filter { $0.raw.depth - startDepth <= maxDepth }
+        : Array(rawNodes[range])
+      return reindex(scopedNodes, depthOffset: startDepth)
+    }
+  }
+
+  static func reindex(
+    _ nodes: [SnapshotPresentationNode],
+    depthOffset: Int
+  ) -> [SnapshotPresentationNode] {
+    let indexMap = Dictionary(
+      uniqueKeysWithValues: nodes.enumerated().map { ($0.element.raw.index, $0.offset) })
+    return nodes.enumerated().map { offset, node in
+      let raw = node.raw
+      return SnapshotPresentationNode(
+        raw: RawAXNode(
+          index: offset,
+          type: raw.type,
+          label: raw.label,
+          identifier: raw.identifier,
+          value: raw.value,
+          rect: raw.rect,
+          enabled: raw.enabled,
+          focused: raw.focused,
+          selected: raw.selected,
+          hittable: raw.hittable,
+          depth: max(0, raw.depth - depthOffset),
+          parentIndex: raw.parentIndex.flatMap { indexMap[$0] },
+          hiddenContentAbove: raw.hiddenContentAbove,
+          hiddenContentBelow: raw.hiddenContentBelow,
+          actions: raw.actions
+        ),
+        effectiveRect: node.effectiveRect
+      )
+    }
+  }
+
+  static func isEligibleForRegularPresentation(_ node: RawAXNode) -> Bool {
+    if node.parentIndex == nil { return true }
+    return eligibleInteractiveTypes.contains(node.type) || node.hasSemanticContent
+  }
+}