agent-device 0.21.7 → 0.21.9

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift

@@ -0,0 +1,566 @@
+import XCTest
+import AgentDeviceSnapshotPresentation
+
+extension RunnerTests {
+  func interactiveRootNode(rect: CGRect) -> RawAXNode {
+    RawAXNode(
+      index: 0,
+      type: "Application",
+      label: nil,
+      identifier: nil,
+      value: nil,
+      rect: SnapshotRect(rect),
+      enabled: true,
+      focused: nil,
+      selected: nil,
+      hittable: false,
+      depth: 0,
+      parentIndex: nil,
+      hiddenContentAbove: nil,
+      hiddenContentBelow: nil
+    )
+  }
+
+  func interactiveRootFrame(for candidates: [RawAXNode]) -> CGRect {
+    guard !candidates.isEmpty else {
+      return .zero
+    }
+    let maxX = candidates.map { CGFloat($0.rect.x + $0.rect.width) }.max() ?? 0
+    let maxY = candidates.map { CGFloat($0.rect.y + $0.rect.height) }.max() ?? 0
+    return CGRect(x: 0, y: 0, width: max(1, maxX), height: max(1, maxY))
+  }
+
+
+
+  func makeSnapshotTraversalContext(
+    app: XCUIApplication,
+    hint: CaptureHint,
+    captureDeadline: Date = .distantFuture,
+    treeCaptureSliceBudgetOverride: TimeInterval? = nil
+  ) throws -> SnapshotTraversalContext? {
+
+
+
+    let geometry = try runMainThreadWork(
+      "snapshot_viewport",
+      timeout: min(1.0, max(0.1, captureDeadline.timeIntervalSinceNow)),
+      timeoutError: snapshotMainThreadTimeoutError("preparing tree snapshot")
+    ) {
+      (
+        viewport: self.safeSnapshotViewport(app: app),
+        interfaceOrientation: self.capturedInterfaceOrientation(app: app)
+      )
+    }
+    let viewport = geometry.viewport
+    let interfaceOrientation = geometry.interfaceOrientation
+    let treeSliceBudget = treeCaptureSliceBudgetOverride ?? treeCaptureSliceBudget
+    let slice = min(treeSliceBudget, max(0.5, captureDeadline.timeIntervalSinceNow))
+    guard let rootSnapshot = try captureSnapshotRootBounded(app, sliceSeconds: slice) else {
+      return nil
+    }
+
+
+
+
+
+
+    let keyboardBand = captureKeyboardBandFact(app: app, deadline: captureDeadline)
+
+    return SnapshotTraversalContext(
+      queryRoot: app,
+      rootSnapshot: rootSnapshot,
+      viewport: viewport,
+      interfaceOrientation: interfaceOrientation,
+      keyboardBand: keyboardBand
+    )
+  }
+
+  static let xCTestSnapshotTimeoutCode = "IOS_TREE_CAPTURE_TIMEOUT"
+
+
+
+
+
+
+
+  private func captureSnapshotRootBounded(
+    _ element: XCUIElement,
+    sliceSeconds: TimeInterval
+  ) throws -> XCUIElementSnapshot? {
+    if Thread.isMainThread {
+      return try captureSnapshotRoot(element)
+    }
+    return try runMainThreadWork(
+      "tree_capture",
+      timeout: sliceSeconds,
+      timeoutError: treeCaptureTimeoutError(sliceSeconds: sliceSeconds)
+    ) {
+      try self.captureSnapshotRoot(element)
+    }
+  }
+
+  private func treeCaptureTimeoutError(sliceSeconds: TimeInterval) -> () -> Error {
+    {
+      SnapshotCaptureFailure(
+        code: Self.xCTestSnapshotTimeoutCode,
+        message: "the XCTest tree capture exceeded its \(Int(sliceSeconds))s time slice",
+        hint: "The capture plan will avoid or tightly bound XCTest-backed snapshot tiers on this screen."
+      )
+    }
+  }
+
+  func snapshotMainThreadTimeoutError(_ operation: String) -> () -> Error {
+    {
+      SnapshotCaptureFailure(
+        code: Self.xCTestSnapshotTimeoutCode,
+        message: "timed out while \(operation) on the XCTest main thread",
+        hint: "The capture plan will skip XCTest-backed snapshot tiers while the previous main-thread work drains."
+      )
+    }
+  }
+
+  private func captureSnapshotRoot(_ element: XCUIElement) throws -> XCUIElementSnapshot? {
+    var rootSnapshot: XCUIElementSnapshot?
+    var swiftErrorMessage: String?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      do {
+        rootSnapshot = try element.snapshot()
+      } catch {
+        swiftErrorMessage = describeSnapshotError(error)
+      }
+    })
+
+    if let rootSnapshot {
+      return rootSnapshot
+    }
+    let message = exceptionMessage ?? swiftErrorMessage ?? "snapshot returned no root"
+    if Self.isAxIllegalArgument(message) {
+      throw axSnapshotFailure(message)
+    }
+    return nil
+  }
+
+  func safeSnapshotViewport(app: XCUIApplication) -> CGRect {
+    safely("SNAPSHOT_VIEWPORT", CGRect.infinite) { snapshotViewport(app: app) }
+  }
+
+  private func describeSnapshotError(_ error: Error) -> String {
+    let localized = error.localizedDescription
+    let debug = String(describing: error)
+    if localized.isEmpty { return debug }
+    if debug == localized { return localized }
+    return "\(localized) (\(debug))"
+  }
+
+  private func axSnapshotFailure(_ message: String) -> SnapshotCaptureFailure {
+    let detail = message.trimmingCharacters(in: .whitespacesAndNewlines)
+    let failureMessage: String
+    if detail.isEmpty {
+      failureMessage = Self.axSnapshotFailureMessage
+    } else {
+      failureMessage = "\(Self.axSnapshotFailureMessage) \(detail)"
+    }
+    return SnapshotCaptureFailure(
+      code: Self.axSnapshotErrorCode,
+      message: failureMessage,
+      hint: Self.axSnapshotHint
+    )
+  }
+
+  private static func isAxIllegalArgument(_ message: String) -> Bool {
+    let normalized = message.lowercased()
+    return normalized.contains("kaxerrorillegalargument")
+      || (normalized.contains("illegal argument") && normalized.contains("snapshot"))
+  }
+
+  static func isAxSnapshotFailure(_ failure: SnapshotCaptureFailure) -> Bool {
+    failure.code == Self.axSnapshotErrorCode || isAxIllegalArgument(failure.message)
+  }
+
+  func evaluateSnapshot(_ snapshot: XCUIElementSnapshot) -> SnapshotEvaluation {
+    let label = aggregatedLabel(for: snapshot) ?? snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
+    let identifier = snapshot.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
+    let valueText = snapshotValueText(snapshot)
+    return SnapshotEvaluation(
+      label: label,
+      identifier: identifier,
+      valueText: valueText,
+      focused: snapshotHasFocus(snapshot),
+      selected: snapshotIsSelected(snapshot)
+    )
+  }
+
+  func makeSnapshotNode(
+    snapshot: XCUIElementSnapshot,
+    evaluation: SnapshotEvaluation,
+    depth: Int,
+    index: Int,
+    parentIndex: Int?
+  ) -> RawAXNode {
+
+
+    return RawAXNode(
+      index: index,
+      type: elementTypeName(snapshot.elementType),
+      label: evaluation.label.isEmpty ? nil : evaluation.label,
+      identifier: evaluation.identifier.isEmpty ? nil : evaluation.identifier,
+      value: evaluation.valueText,
+      rect: SnapshotRect(snapshot.frame),
+      enabled: snapshot.isEnabled,
+      focused: evaluation.focused ? true : nil,
+      selected: evaluation.selected ? true : nil,
+      hittable: false,
+      depth: depth,
+      parentIndex: parentIndex,
+      hiddenContentAbove: nil,
+      hiddenContentBelow: nil
+    )
+  }
+
+
+
+  func capturedInterfaceOrientation(app: XCUIApplication) -> Int {
+    safely("SNAPSHOT_INTERFACE_ORIENTATION", RunnerInterfaceOrientation.unknown) {
+      Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
+    }
+  }
+
+  private func snapshotValueText(_ snapshot: XCUIElementSnapshot) -> String? {
+    guard let value = snapshot.value else { return nil }
+    let text = String(describing: value).trimmingCharacters(in: .whitespacesAndNewlines)
+    return text.isEmpty ? nil : text
+  }
+
+  private func snapshotViewport(app: XCUIApplication) -> CGRect {
+#if os(iOS)
+    let appFrame = onScreenWindowFrame(app: app)
+#else
+    let appFrame = app.frame
+#endif
+    if !appFrame.isNull && !appFrame.isEmpty {
+      return appFrame
+    }
+    return .infinite
+  }
+
+  static func snapshotTraversalIdentity(
+    elementType: XCUIElement.ElementType,
+    label: String,
+    identifier: String,
+    frame: CGRect
+  ) -> String {
+    #if os(iOS)
+    "\(elementType)-\(label)-\(identifier)-\(frame.origin.x)-\(frame.origin.y)-\(frame.width)-\(frame.height)"
+    #else
+    return "\(elementType)-\(label)-\(identifier)-\(frame.origin.x)-\(frame.origin.y)"
+    #endif
+  }
+
+  private func aggregatedLabel(for snapshot: XCUIElementSnapshot, depth: Int = 0) -> String? {
+    if depth > 4 { return nil }
+    let text = snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
+    if !text.isEmpty { return text }
+    if let valueText = snapshotValueText(snapshot) { return valueText }
+    for child in snapshot.children {
+      if let childLabel = aggregatedLabel(for: child, depth: depth + 1) {
+        return childLabel
+      }
+    }
+    return nil
+  }
+
+  func appendCollapsedTabFallbackNodes(
+    to nodes: inout [RawAXNode],
+    containerSnapshot: XCUIElementSnapshot,
+    resolveElements: () -> [XCUIElement],
+    depth: Int,
+    parentIndex: Int
+  ) {
+    let fallbackNodes = collapsedTabFallbackNodes(
+      for: containerSnapshot,
+      resolveElements: resolveElements,
+      startingIndex: nodes.count,
+      depth: depth,
+      parentIndex: parentIndex
+    )
+    nodes.append(contentsOf: fallbackNodes)
+  }
+
+  private func collapsedTabFallbackNodes(
+    for containerSnapshot: XCUIElementSnapshot,
+    resolveElements: () -> [XCUIElement],
+    startingIndex: Int,
+    depth: Int,
+    parentIndex: Int
+  ) -> [RawAXNode] {
+    if !containerSnapshot.children.isEmpty { return [] }
+
+
+
+
+    guard shouldExpandCollapsedTabContainer(containerSnapshot) else { return [] }
+    let containerFrame = containerSnapshot.frame
+    if containerFrame.isNull || containerFrame.isEmpty { return [] }
+
+
+
+    let elements = resolveElements()
+    let candidates = elements.compactMap { element in
+      collapsedTabCandidateNode(
+        element: element,
+        containerSnapshot: containerSnapshot,
+        containerFrame: containerFrame
+      )
+    }
+    .sorted { left, right in
+      if left.rect.x != right.rect.x {
+        return left.rect.x < right.rect.x
+      }
+      return left.rect.y < right.rect.y
+    }
+
+    if candidates.count < 2 { return [] }
+    let rowMidpoints = candidates.map { $0.rect.y + ($0.rect.height / 2) }
+    let rowSpread = (rowMidpoints.max() ?? 0) - (rowMidpoints.min() ?? 0)
+
+    if rowSpread > max(24.0, Double(containerFrame.height) * 0.6) { return [] }
+
+    var seen = Set<String>()
+    let uniqueCandidates = candidates.filter { node in
+      let key = "\(node.type)-\(node.label ?? "")-\(node.identifier ?? "")-\(node.value ?? "")-\(node.rect.x)-\(node.rect.y)-\(node.rect.width)-\(node.rect.height)"
+      if seen.contains(key) { return false }
+      seen.insert(key)
+      return true
+    }
+    if uniqueCandidates.count < 2 { return [] }
+
+    return uniqueCandidates.enumerated().map { offset, node in
+      RawAXNode(
+        index: startingIndex + offset,
+        type: node.type,
+        label: node.label,
+        identifier: node.identifier,
+        value: node.value,
+        rect: node.rect,
+        enabled: node.enabled,
+        focused: node.focused,
+        selected: node.selected,
+        hittable: node.hittable,
+        depth: depth,
+        parentIndex: parentIndex,
+        hiddenContentAbove: nil,
+        hiddenContentBelow: nil
+      )
+    }
+  }
+
+  private func collapsedTabCandidateNode(
+    element: XCUIElement,
+    containerSnapshot: XCUIElementSnapshot,
+    containerFrame: CGRect
+  ) -> RawAXNode? {
+    var node: RawAXNode?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      if !element.exists { return }
+      let elementType = element.elementType
+      if !Self.collapsedTabCandidateTypes.contains(elementType) { return }
+      let frame = element.frame
+      if frame.isNull || frame.isEmpty { return }
+      if frame.equalTo(containerFrame) { return }
+      let area = max(CGFloat(1), frame.width * frame.height)
+      let containerArea = max(CGFloat(1), containerFrame.width * containerFrame.height)
+      if area >= containerArea * 0.9 { return }
+      let center = CGPoint(x: frame.midX, y: frame.midY)
+      if !containerFrame.contains(center) { return }
+
+      let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
+      let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
+      let valueText = snapshotValueText(element)
+      let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil
+      if !hasContent { return }
+      if sameSemanticElement(
+        containerSnapshot: containerSnapshot,
+        elementType: elementType,
+        label: label,
+        identifier: identifier
+      ) {
+        return
+      }
+
+
+
+      node = RawAXNode(
+        index: 0,
+        type: elementTypeName(elementType),
+        label: label.isEmpty ? nil : label,
+        identifier: identifier.isEmpty ? nil : identifier,
+        value: valueText,
+        rect: SnapshotRect(frame),
+        enabled: element.isEnabled,
+        focused: elementHasFocus(element) ? true : nil,
+        selected: element.isSelected ? true : nil,
+        hittable: false,
+        depth: 0,
+        parentIndex: nil,
+        hiddenContentAbove: nil,
+        hiddenContentBelow: nil
+      )
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_SNAPSHOT_TAB_FALLBACK_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return nil
+    }
+    return node
+  }
+
+  private func snapshotHasFocus(_ snapshot: XCUIElementSnapshot) -> Bool {
+    var focused = false
+    _ = RunnerObjCExceptionCatcher.catchException({
+      if let value = (snapshot as! NSObject).value(forKey: "hasFocus") as? Bool {
+        focused = value
+      }
+    })
+    return focused
+  }
+
+  private func snapshotIsSelected(_ snapshot: XCUIElementSnapshot) -> Bool {
+    return snapshot.isSelected
+  }
+
+  private func shouldExpandCollapsedTabContainer(_ snapshot: XCUIElementSnapshot) -> Bool {
+    let frame = snapshot.frame
+    if frame.isNull || frame.isEmpty { return false }
+    if frame.width < max(CGFloat(160), frame.height * 1.75) { return false }
+    switch snapshot.elementType {
+    case .tabBar, .segmentedControl, .slider:
+      return true
+    default:
+      return false
+    }
+  }
+
+  private func snapshotValueText(_ element: XCUIElement) -> String? {
+    let text = String(describing: element.value ?? "")
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+    return text.isEmpty ? nil : text
+  }
+
+  private func sameSemanticElement(
+    containerSnapshot: XCUIElementSnapshot,
+    elementType: XCUIElement.ElementType,
+    label: String,
+    identifier: String
+  ) -> Bool {
+    if containerSnapshot.elementType != elementType { return false }
+    let containerLabel = containerSnapshot.label.trimmingCharacters(in: .whitespacesAndNewlines)
+    let containerIdentifier = containerSnapshot.identifier
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+    return containerLabel == label && containerIdentifier == identifier
+  }
+
+  func flatInteractiveElements(
+    app: XCUIApplication,
+    deadline: Date
+  ) -> (elements: [XCUIElement], truncated: Bool) {
+    let queries: [XCUIElementQuery] = [
+      app.buttons,
+      app.links,
+      app.textFields,
+      app.secureTextFields,
+      app.searchFields,
+      app.textViews,
+      app.switches,
+      app.sliders,
+      app.segmentedControls,
+      app.cells,
+      app.collectionViews,
+      app.tables,
+      app.scrollViews,
+      app.pickers,
+      app.steppers,
+      app.tabBars,
+      app.menuItems,
+      app.staticTexts,
+      app.images
+    ]
+
+    var elements: [XCUIElement] = []
+    var truncated = false
+    for query in queries {
+      if Date() >= deadline {
+        NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_FALLBACK_DEADLINE")
+        truncated = true
+        break
+      }
+      let result = snapshotElementsQuery {
+        query.allElementsBoundByIndex
+      }
+      elements.append(contentsOf: result.elements)
+      if result.axUnavailable {
+        break
+      }
+    }
+    return (elements, truncated)
+  }
+
+  func snapshotElementsQuery(
+    _ fetch: () -> [XCUIElement]
+  ) -> (elements: [XCUIElement], axUnavailable: Bool) {
+    let (elements, exceptionMessage) = catchingObjCException(fallback: [], fetch)
+    guard let exceptionMessage else {
+      return (elements, false)
+    }
+    NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_QUERY_IGNORED_EXCEPTION=%@", exceptionMessage)
+    if Self.isAxIllegalArgument(exceptionMessage) {
+      invalidateCachedTarget(reason: "ax_snapshot_query_unavailable")
+      return ([], true)
+    }
+    return ([], false)
+  }
+
+  func flatSnapshotNode(
+    element: XCUIElement,
+    index: Int,
+    parentIndex: Int?
+  ) -> RawAXNode? {
+    var node: RawAXNode?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      if !element.exists { return }
+
+
+      let frame = element.frame
+      if frame.isNull || frame.isEmpty { return }
+      let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
+      let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
+      let valueText = snapshotValueText(element)
+      let elementType = element.elementType
+      let enabled = element.isEnabled
+
+      node = RawAXNode(
+        index: index,
+        type: elementTypeName(elementType),
+        label: label.isEmpty ? nil : label,
+        identifier: identifier.isEmpty ? nil : identifier,
+        value: valueText,
+        rect: SnapshotRect(frame),
+        enabled: enabled,
+        focused: elementHasFocus(element) ? true : nil,
+        selected: element.isSelected ? true : nil,
+        hittable: false,
+        depth: 1,
+        parentIndex: parentIndex,
+        hiddenContentAbove: nil,
+        hiddenContentBelow: nil
+      )
+    })
+    if let exceptionMessage {
+      NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_FLAT_IGNORED_EXCEPTION=%@", exceptionMessage)
+      return nil
+    }
+    return node
+  }
+
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift

@@ -59,6 +59,9 @@ struct SynthesizedGesturePolicy: Equatable, Hashable {
 
 struct SynthesizedCoordinateContext {
   let referenceFrame: CGRect
+
+
+  let resolvedWindow: XCUIElement
   let keyboardPolicy: SynthesizedKeyboardPolicy
   let fallbackPolicy: SynthesizedFallbackPolicy
   let accessibilityHealth: RunnerAccessibilityHealth
@@ -66,6 +69,7 @@ struct SynthesizedCoordinateContext {
   func withReferenceFrame(_ frame: CGRect) -> SynthesizedCoordinateContext {
     SynthesizedCoordinateContext(
       referenceFrame: frame,
+      resolvedWindow: resolvedWindow,
       keyboardPolicy: keyboardPolicy,
       fallbackPolicy: fallbackPolicy,
       accessibilityHealth: accessibilityHealth
@@ -144,7 +148,7 @@ extension RunnerTests {
       return
     }
     NSLog(
-      "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=%@ axHealth=%@ frameSource=screenshot keyboardPolicy=%@ fallbackPolicy=%@ fallbackAllowed=%@ fallbackAttempted=%@",
+      "AGENT_DEVICE_RUNNER_SYNTHESIZED_GESTURE_POLICY kind=%@ axHealth=%@ frameSource=window keyboardPolicy=%@ fallbackPolicy=%@ fallbackAllowed=%@ fallbackAttempted=%@",
       kind.rawValue,
       context.accessibilityHealth.rawValue,
       context.keyboardPolicy.rawValue,