agent-device 0.17.2 → 0.17.4

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift

@@ -11,21 +11,7 @@ extension RunnerTests {
   private static let rawSnapshotMaxNodes = 5_000
   private static let rawSnapshotTooLargeHint =
     "Raw iOS snapshot exceeded the runner payload guard. Use regular snapshot for visible UI, or scope/depth-limit raw snapshot when inspecting a large accessibility tree."
-  private static let collapsedTabCandidateTypes: Set<XCUIElement.ElementType> = [
-    .button,
-    .link,
-    .menuItem,
-    .other,
-    .staticText
-  ]
-  private static let scrollContainerTypes: Set<XCUIElement.ElementType> = [
-    .collectionView,
-    .scrollView,
-    .table
-  ]
-  private static let flatInteractiveFallbackBudget: TimeInterval = 1.0
-
-  private struct SnapshotTraversalContext {
+  struct SnapshotTraversalContext {
     let queryRoot: XCUIElement
     let rootSnapshot: XCUIElementSnapshot
     let viewport: CGRect
@@ -44,12 +30,6 @@ extension RunnerTests {
     let visible: Bool
   }
 
-  private enum SnapshotTraversalCapture {
-    case context(SnapshotTraversalContext)
-    case fallback(DataPayload)
-    case empty
-  }
-
   struct SnapshotCaptureFailure: Error {
     let code: String
     let message: String
@@ -98,29 +78,45 @@ extension RunnerTests {
     }
   }
 
+  static let structuralOnlyNodeTypes: Set<String> = [
+    "Application",
+    "Window",
+    "Other",
+    "ScrollView"
+  ]
+
+  private static let collapsedTabCandidateTypes: Set<XCUIElement.ElementType> = [
+    .button,
+    .link,
+    .menuItem,
+    .other,
+    .staticText
+  ]
+
+  static let scrollContainerTypes: Set<XCUIElement.ElementType> = [
+    .collectionView,
+    .scrollView,
+    .table
+  ]
+
+  private static let flatInteractiveFallbackBudget: TimeInterval = 1.0
+
   func snapshotFast(app: XCUIApplication, options: SnapshotOptions) throws -> DataPayload {
-    if options.interactiveOnly && options.compact {
-      return snapshotFlatInteractive(app: app, options: options)
-    }
     if let blocking = blockingSystemAlertSnapshot() {
       return blocking
     }
-
-    let capture = try captureSnapshotTraversalContext(
+    return try runSnapshotCapturePlan(
+      Self.regularVisiblePlan,
       app: app,
       options: options,
-      allowInteractiveUnavailableFallback: true
+      terminal: .sparseWithFatalOnAXFailure
     )
-    let context: SnapshotTraversalContext
-    switch capture {
-    case .context(let traversalContext):
-      context = traversalContext
-    case .fallback(let fallback):
-      return fallback
-    case .empty:
-      return DataPayload(nodes: [], truncated: false)
-    }
+  }
 
+  func recursiveTreeSnapshotPayload(
+    context: SnapshotTraversalContext,
+    options: SnapshotOptions
+  ) -> DataPayload {
     var cachedDescendantElements: [XCUIElement]?
     func collapsedTabDescendants() -> [XCUIElement] {
       if let cachedDescendantElements {
@@ -253,22 +249,18 @@ extension RunnerTests {
     if let blocking = blockingSystemAlertSnapshot() {
       return blocking
     }
-
-    let capture = try captureSnapshotTraversalContext(
+    return try runSnapshotCapturePlan(
+      Self.rawDiagnosticPlan,
       app: app,
       options: options,
-      allowInteractiveUnavailableFallback: false
+      terminal: .throwOnAXFailure
     )
-    let context: SnapshotTraversalContext
-    switch capture {
-    case .context(let traversalContext):
-      context = traversalContext
-    case .fallback(let fallback):
-      return fallback
-    case .empty:
-      return DataPayload(nodes: [], truncated: false)
-    }
+  }
 
+  func rawTreeSnapshotPayload(
+    context: SnapshotTraversalContext,
+    options: SnapshotOptions
+  ) throws -> DataPayload {
     var nodes: [SnapshotNode] = []
 
     func walk(_ snapshot: XCUIElementSnapshot, depth: Int, parentIndex: Int?) throws {
@@ -310,9 +302,9 @@ extension RunnerTests {
     return DataPayload(nodes: nodes, truncated: false)
   }
 
-  private func snapshotFlatInteractive(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
+  func snapshotFlatInteractive(app: XCUIApplication, options: SnapshotOptions) -> DataPayload {
     var nodes: [SnapshotNode] = [
-      compactInteractiveRootNode(rect: .zero)
+      interactiveRootNode(rect: .zero)
     ]
     if options.depth == 0 {
       return DataPayload(nodes: nodes, truncated: false)
@@ -356,7 +348,13 @@ extension RunnerTests {
       return left.type < right.type
     }
 
-    nodes[0] = compactInteractiveRootNode(rect: compactInteractiveRootFrame(for: candidates))
+    // The synthetic root doubles as the daemon's viewport (find.ts prefers on-screen matches
+    // inside nodes[0].rect): use the real screen viewport when capture produced a finite one,
+    // so off-screen candidates can never inflate the root and masquerade as on-screen.
+    let rootRect = viewport.isInfinite || viewport.isNull || viewport.isEmpty
+      ? interactiveRootFrame(for: candidates)
+      : viewport
+    nodes[0] = interactiveRootNode(rect: rootRect)
     for candidate in candidates {
       nodes.append(
         SnapshotNode(
@@ -380,78 +378,26 @@ extension RunnerTests {
     return DataPayload(nodes: nodes, truncated: truncated)
   }
 
-  private func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload {
+  func snapshotAccessibilityUnavailable(failure: SnapshotCaptureFailure) -> DataPayload {
     NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_AX_UNAVAILABLE=%@", failure.message)
     invalidateCachedTarget(reason: Self.axSnapshotUnavailableReason)
+    // This is a planned terminal result, so it carries the structured verdict like every other
+    // planned snapshot — downstream sparse handling keys off the verdict, not node shapes.
     return sparseTruncatedSnapshotPayload(
       message: recoveredSnapshotMessage(failure),
+      snapshotQuality: SnapshotQuality(
+        state: "sparse",
+        backend: SnapshotBackendKind.recursiveTree.rawValue,
+        reason: failure.message,
+        reasonCode: "ax-rejected",
+        effectiveDepth: nil,
+        collapsedLeafIndexes: nil
+      ),
       runnerFatal: true,
       runnerFatalReason: Self.axSnapshotUnavailableReason
     )
   }
 
-  private func captureSnapshotTraversalContext(
-    app: XCUIApplication,
-    options: SnapshotOptions,
-    allowInteractiveUnavailableFallback: Bool
-  ) throws -> SnapshotTraversalCapture {
-    do {
-      guard let context = try makeSnapshotTraversalContext(app: app, options: options) else {
-        return .empty
-      }
-      return .context(context)
-    } catch let failure as SnapshotCaptureFailure {
-      if let fallback = snapshotDepthLimitedAccessibilityFallback(
-        app: app,
-        options: options,
-        failure: failure
-      ) {
-        return .fallback(fallback)
-      }
-      if allowInteractiveUnavailableFallback && options.interactiveOnly {
-        return .fallback(snapshotAccessibilityUnavailable(failure: failure))
-      }
-      throw failure
-    }
-  }
-
-  private func snapshotDepthLimitedAccessibilityFallback(
-    app: XCUIApplication,
-    options: SnapshotOptions,
-    failure: SnapshotCaptureFailure
-  ) -> DataPayload? {
-    guard let requestedDepth = options.depth else {
-      return nil
-    }
-
-    NSLog(
-      "AGENT_DEVICE_RUNNER_SNAPSHOT_DEPTH_FALLBACK=%@",
-      failure.message
-    )
-
-    if requestedDepth <= 0 {
-      return sparseTruncatedSnapshotPayload(message: recoveredSnapshotMessage(failure))
-    }
-
-    // Raw depth-limited recovery intentionally falls back to sparse interactive discovery because
-    // the raw AX tree is the failed operation.
-    let fallback = snapshotFlatInteractive(
-      app: app,
-      options: SnapshotOptions(
-        interactiveOnly: true,
-        compact: options.compact,
-        depth: requestedDepth,
-        scope: options.scope,
-        raw: false
-      )
-    )
-    return DataPayload(
-      message: recoveredSnapshotMessage(failure),
-      nodes: fallback.nodes,
-      truncated: true
-    )
-  }
-
   private func recoveredSnapshotMessage(_ failure: SnapshotCaptureFailure) -> String {
     return "\(failure.message) Hint: \(failure.hint)"
   }
@@ -464,15 +410,17 @@ extension RunnerTests {
     )
   }
 
-  private func sparseTruncatedSnapshotPayload(
-    message: String,
+  func sparseTruncatedSnapshotPayload(
+    message: String? = nil,
+    snapshotQuality: SnapshotQuality? = nil,
     runnerFatal: Bool? = nil,
     runnerFatalReason: String? = nil
   ) -> DataPayload {
     return DataPayload(
       message: message,
-      nodes: [compactInteractiveRootNode(rect: .zero)],
+      nodes: [interactiveRootNode(rect: .zero)],
       truncated: true,
+      snapshotQuality: snapshotQuality,
       runnerFatal: runnerFatal,
       runnerFatalReason: runnerFatalReason
     )
@@ -521,40 +469,7 @@ extension RunnerTests {
     XCTAssertEqual(failure.hint, Self.rawSnapshotTooLargeHint)
   }
 
-  func testDepthLimitedSnapshotFailureReturnsNonFatalFallback() {
-    currentApp = app
-    currentBundleId = "com.example.app"
-
-    let payload = snapshotDepthLimitedAccessibilityFallback(
-      app: app,
-      options: SnapshotOptions(
-        interactiveOnly: false,
-        compact: false,
-        depth: 0,
-        scope: nil,
-        raw: false
-      ),
-      failure: SnapshotCaptureFailure(
-        code: Self.axSnapshotErrorCode,
-        message: "\(Self.axSnapshotFailureMessage) kAXErrorIllegalArgument.",
-        hint: Self.axSnapshotHint
-      )
-    )
-
-    XCTAssertEqual(
-      payload?.message,
-      "\(Self.axSnapshotFailureMessage) kAXErrorIllegalArgument. Hint: \(Self.axSnapshotHint)"
-    )
-    XCTAssertEqual(payload?.nodes?.count, 1)
-    XCTAssertEqual(payload?.nodes?.first?.type, "Application")
-    XCTAssertEqual(payload?.truncated, true)
-    XCTAssertNil(payload?.runnerFatal)
-    XCTAssertNil(payload?.runnerFatalReason)
-    XCTAssertNotNil(currentApp)
-    XCTAssertEqual(currentBundleId, "com.example.app")
-  }
-
-  private func compactInteractiveRootNode(rect: CGRect) -> SnapshotNode {
+  private func interactiveRootNode(rect: CGRect) -> SnapshotNode {
     SnapshotNode(
       index: 0,
       type: "Application",
@@ -573,7 +488,7 @@ extension RunnerTests {
     )
   }
 
-  private func compactInteractiveRootFrame(for candidates: [SnapshotNode]) -> CGRect {
+  private func interactiveRootFrame(for candidates: [SnapshotNode]) -> CGRect {
     guard !candidates.isEmpty else {
       return .zero
     }
@@ -605,9 +520,6 @@ extension RunnerTests {
   ) -> Bool {
     let type = snapshot.elementType
     let hasContent = !label.isEmpty || !identifier.isEmpty || (valueText != nil)
-    if options.compact && type == .other && !hasContent && !hittable {
-      if snapshot.children.count <= 1 { return false }
-    }
     if options.interactiveOnly {
       if isScrollableContainer(snapshot, visible: visible) { return true }
       #if os(macOS)
@@ -620,9 +532,6 @@ extension RunnerTests {
       if hasContent { return true }
       return false
     }
-    if options.compact {
-      return hasContent || hittable
-    }
     if regularSnapshot {
       if type == .application || type == .window { return true }
       return visible
@@ -649,7 +558,7 @@ extension RunnerTests {
     return true
   }
 
-  private func makeSnapshotTraversalContext(
+  func makeSnapshotTraversalContext(
     app: XCUIApplication,
     options: SnapshotOptions
   ) throws -> SnapshotTraversalContext? {
@@ -692,7 +601,7 @@ extension RunnerTests {
     return nil
   }
 
-  private func safeSnapshotViewport(app: XCUIApplication) -> CGRect {
+  func safeSnapshotViewport(app: XCUIApplication) -> CGRect {
     safely("SNAPSHOT_VIEWPORT", CGRect.infinite) { snapshotViewport(app: app) }
   }
 
@@ -725,6 +634,10 @@ extension RunnerTests {
       || (normalized.contains("illegal argument") && normalized.contains("snapshot"))
   }
 
+  static func isAxSnapshotFailure(_ failure: SnapshotCaptureFailure) -> Bool {
+    failure.code == Self.axSnapshotErrorCode || isAxIllegalArgument(failure.message)
+  }
+
   private func evaluateSnapshot(
     _ snapshot: XCUIElementSnapshot,
     in context: SnapshotTraversalContext
@@ -826,15 +739,6 @@ extension RunnerTests {
   }
 
   private func snapshotViewport(app: XCUIApplication) -> CGRect {
-    let windows = app.windows.allElementsBoundByIndex
-    let windowFrames = windows
-      .filter { $0.exists && !$0.frame.isNull && !$0.frame.isEmpty }
-      .map(\.frame)
-    if let largestWindowFrame = windowFrames.max(by: { left, right in
-      left.width * left.height < right.width * right.height
-    }) {
-      return largestWindowFrame
-    }
     let appFrame = app.frame
     if !appFrame.isNull && !appFrame.isEmpty {
       return appFrame
@@ -855,7 +759,7 @@ extension RunnerTests {
     return nil
   }
 
-  private func isVisibleInViewport(_ rect: CGRect, _ viewport: CGRect) -> Bool {
+  func isVisibleInViewport(_ rect: CGRect, _ viewport: CGRect) -> Bool {
     if rect.isNull || rect.isEmpty { return false }
     return rect.intersects(viewport)
   }
@@ -1191,19 +1095,19 @@ extension RunnerTests {
       let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
       let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
       let valueText = snapshotValueText(element)
-      let hasContent = !label.isEmpty || !identifier.isEmpty || valueText != nil
       let elementType = element.elementType
       let enabled = element.isEnabled
       let hittable = visible && enabled && element.isHittable
-      if options.compact && !hasContent && !hittable && !interactiveTypes.contains(elementType) {
+      let filterNode = FlatSnapshotFilterNode(
+        isRoot: false,
+        label: label,
+        identifier: identifier,
+        valueText: valueText,
+        visible: visible
+      )
+      if !flatSnapshotFilterDecision(filterNode, options: options, insideMatchedScope: false).include {
         return
       }
-      if let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines), !scope.isEmpty {
-        let haystack = [label, identifier, valueText ?? ""].joined(separator: "\n")
-        if !haystack.localizedCaseInsensitiveContains(scope) {
-          return
-        }
-      }
 
       node = SnapshotNode(
         index: index,