agent-device 0.18.2 → 0.19.0

package/{apple-runner → dist/apple-runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift

@@ -63,6 +63,46 @@ extension RunnerTests {
   static let regularVisiblePlan: [SnapshotBackendKind] = [.recursiveTree, .querySweep, .privateAX]
   static let rawDiagnosticPlan: [SnapshotBackendKind] = [.recursiveTree, .privateAX]
 
+  // MARK: Tree-backend penalty (cross-attempt memory, #1105)
+  //
+  // On some deep/dynamic screens the XCTest bulk snapshot no longer fails fast with
+  // kAXErrorIllegalArgument (the #758 signature) — it grinds for many seconds first. One slow
+  // grind is tolerable; re-grinding on every subsequent capture of the same screen buries the
+  // main thread past the execution watchdog. After a slow or abandoned tree capture, later
+  // plans for the same bundle lead with the private AX backend until the penalty expires.
+
+  func penalizeSnapshotTreeBackend(bundleId: String?, reason: String) {
+    snapshotTreePenaltyLock.lock()
+    snapshotTreePenaltyBundleId = bundleId
+    snapshotTreePenaltyUntil = Date().addingTimeInterval(snapshotTreePenaltyDuration)
+    snapshotTreePenaltyLock.unlock()
+    NSLog(
+      "AGENT_DEVICE_RUNNER_SNAPSHOT_TREE_PENALIZED bundle=%@ reason=%@",
+      bundleId ?? "",
+      reason
+    )
+  }
+
+  func isSnapshotTreeBackendPenalized(bundleId: String?) -> Bool {
+    snapshotTreePenaltyLock.lock()
+    defer { snapshotTreePenaltyLock.unlock() }
+    guard Date() < snapshotTreePenaltyUntil else { return false }
+    // A penalty recorded without a bundle id applies to whatever target is current.
+    guard let penalized = snapshotTreePenaltyBundleId else { return true }
+    return penalized == bundleId
+  }
+
+  /// Pure plan-reorder rule: a penalized tree backend defers to the private AX backend on the
+  /// regular plan only. The raw diagnostic plan keeps tree-first errors, and unknown plans are
+  /// left untouched.
+  static func effectiveSnapshotCapturePlan(
+    _ plan: [SnapshotBackendKind],
+    treePenalized: Bool
+  ) -> (plan: [SnapshotBackendKind], treeDeferred: Bool) {
+    guard treePenalized, plan == Self.regularVisiblePlan else { return (plan, false) }
+    return ([.privateAX, .querySweep, .recursiveTree], true)
+  }
+
   // MARK: Plan runner
 
   func runSnapshotCapturePlan(
@@ -76,21 +116,56 @@ extension RunnerTests {
     var axFailure: SnapshotCaptureFailure?
     let deadline = Date().addingTimeInterval(Self.snapshotPlanBudget)
 
-    for kind in plan {
-      if kind != plan.first && Date() >= deadline {
+    // Reorder is gated to iOS simulators — the only place the private AX backend exists, so
+    // the deferred plan cannot degrade platforms where the tree backend is the strongest tier.
+    var treePenalized = false
+#if os(iOS) && targetEnvironment(simulator)
+    treePenalized = isSnapshotTreeBackendPenalized(bundleId: currentBundleId)
+#endif
+    let (effectivePlan, treeDeferred) = Self.effectiveSnapshotCapturePlan(
+      plan,
+      treePenalized: treePenalized
+    )
+    if treeDeferred {
+      firstFailure = (
+        "the XCTest tree backend was deferred after a recent slow capture on this screen",
+        "budget"
+      )
+      NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_TREE_DEFERRED bundle=%@", currentBundleId ?? "")
+    }
+
+    for kind in effectivePlan {
+      if kind != effectivePlan.first && Date() >= deadline {
         NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_PLAN_BUDGET_EXHAUSTED skipped=%@", kind.rawValue)
         if firstFailure == nil {
           firstFailure = ("the capture plan ran out of its time budget", "budget")
         }
         break
       }
+      // While an abandoned tree capture is still grinding inside testmanagerd, XCTest-backed
+      // tiers would block behind it; only the private AX backend stays responsive (#1105).
+      if kind != .privateAX, hasAbandonedTreeCapture() {
+        NSLog("AGENT_DEVICE_RUNNER_SNAPSHOT_TIER_SKIPPED_XCTEST_OCCUPIED tier=%@", kind.rawValue)
+        if firstFailure == nil {
+          firstFailure = (
+            "the XCTest capture channel is occupied by an abandoned tree capture",
+            "budget"
+          )
+        }
+        continue
+      }
       let capture: SnapshotBackendCapture
+      let backendStartedAt = Date()
       do {
-        guard let result = try captureWithBackend(kind, app: app, options: options) else {
+        guard let result = try captureWithBackend(kind, app: app, options: options, deadline: deadline)
+        else {
+          recordSlowTreeBackendIfNeeded(kind, startedAt: backendStartedAt)
           continue
         }
         capture = result
+        recordSlowTreeBackendIfNeeded(kind, startedAt: backendStartedAt)
       } catch let failure as SnapshotCaptureFailure {
+        recordSlowTreeBackendIfNeeded(kind, startedAt: backendStartedAt)
         if Self.isAxSnapshotFailure(failure) { axFailure = failure }
         if firstFailure == nil {
           firstFailure = (failure.message, Self.isAxSnapshotFailure(failure) ? "ax-rejected" : "capture-failed")
@@ -111,7 +186,7 @@ extension RunnerTests {
         continue
       }
 
-      let recovered = kind != plan.first
+      let recovered = kind != effectivePlan.first || treeDeferred
       if recovered {
         NSLog(
           "AGENT_DEVICE_RUNNER_SNAPSHOT_RECOVERED backend=%@ reason=%@",
@@ -157,14 +232,33 @@ extension RunnerTests {
     return fallbackPayload
   }
 
+  /// Marks the tree backend as penalized when a capture attempt ground past the slow-capture
+  /// threshold — even a successful one: the next capture of this screen must not re-grind.
+  private func recordSlowTreeBackendIfNeeded(_ kind: SnapshotBackendKind, startedAt: Date) {
+    guard kind == .recursiveTree else { return }
+    let elapsed = Date().timeIntervalSince(startedAt)
+    guard elapsed > snapshotTreeSlowCaptureThreshold else { return }
+    penalizeSnapshotTreeBackend(
+      bundleId: currentBundleId,
+      reason: "slow_tree_capture_\(Int(elapsed * 1000))ms"
+    )
+  }
+
   private func captureWithBackend(
     _ kind: SnapshotBackendKind,
     app: XCUIApplication,
-    options: SnapshotOptions
+    options: SnapshotOptions,
+    deadline: Date
   ) throws -> SnapshotBackendCapture? {
     switch kind {
     case .recursiveTree:
-      guard let context = try makeSnapshotTraversalContext(app: app, options: options) else {
+      guard
+        let context = try makeSnapshotTraversalContext(
+          app: app,
+          options: options,
+          captureDeadline: deadline
+        )
+      else {
         return nil
       }
       let payload = options.raw
@@ -173,11 +267,11 @@ extension RunnerTests {
       return SnapshotBackendCapture(payload: payload, effectiveDepth: nil)
     case .querySweep:
       return SnapshotBackendCapture(
-        payload: snapshotFlatInteractive(app: app, options: options),
+        payload: snapshotFlatInteractive(app: app, options: options, planDeadline: deadline),
         effectiveDepth: nil
       )
     case .privateAX:
-      return privateAXSnapshotCapture(app: app, options: options)
+      return privateAXSnapshotCapture(app: app, options: options, deadline: deadline)
     }
   }
 
@@ -304,124 +398,3 @@ extension RunnerTests {
   }
 }
 
-#if AGENT_DEVICE_RUNNER_UNIT_TESTS
-// MARK: - In-bundle unit tests
-
-extension RunnerTests {
-  private func planTestNode(
-    index: Int,
-    type: String,
-    label: String? = nil,
-    identifier: String? = nil,
-    hittable: Bool = false,
-    parentIndex: Int? = nil
-  ) -> SnapshotNode {
-    SnapshotNode(
-      index: index,
-      type: type,
-      label: label,
-      identifier: identifier,
-      value: nil,
-      rect: snapshotRect(from: .zero),
-      enabled: true,
-      focused: nil,
-      selected: nil,
-      hittable: hittable,
-      depth: parentIndex == nil ? 0 : 1,
-      parentIndex: parentIndex,
-      hiddenContentAbove: nil,
-      hiddenContentBelow: nil
-    )
-  }
-
-  func testSparsePayloadReasonMatrix() {
-    let root = planTestNode(index: 0, type: "Application", label: "Example App", hittable: true)
-    let window = planTestNode(index: 1, type: "Window", parentIndex: 0)
-    let button = planTestNode(index: 1, type: "Button", label: "Ok", hittable: true, parentIndex: 0)
-
-    // Labeled, hittable root over a bare window is still sparse.
-    XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, window], truncated: false)))
-    // Deadline-truncated near-empty sweep needs recovery even with one real control.
-    XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: true)))
-    // The same tiny tree from a completed sweep is a legitimately minimal screen.
-    XCTAssertNil(Self.sparsePayloadReason(DataPayload(nodes: [root, button], truncated: false)))
-    // Empty payloads are degraded.
-    XCTAssertNotNil(Self.sparsePayloadReason(DataPayload(nodes: [], truncated: false)))
-  }
-
-  func testCollapsedLeafIndexesFlagsMergedContainersOnly() {
-    let root = planTestNode(index: 0, type: "Application", label: "App")
-    let merged = planTestNode(
-      index: 1,
-      type: "Other",
-      label: (0...30).map { "Row \($0), Tap" }.joined(separator: ", "),
-      parentIndex: 0
-    )
-    let prose = planTestNode(
-      index: 2,
-      type: "StaticText",
-      label: (0...30).map { "clause \($0)" }.joined(separator: ", "),
-      parentIndex: 0
-    )
-    XCTAssertEqual(Self.collapsedLeafIndexes([root, merged, prose]), [1])
-    XCTAssertNil(Self.collapsedLeafIndexes([root, prose]))
-  }
-
-  func testLegacyQualityMessageStatesFallbackMeaning() {
-    let recovered = SnapshotQuality(
-      state: "recovered",
-      backend: "queries",
-      reason: "snapshot returned only structural application/window nodes",
-      reasonCode: "sparse-tree",
-      effectiveDepth: nil,
-      collapsedLeafIndexes: nil
-    )
-    let message = Self.legacyQualityMessage(recovered)
-    XCTAssertTrue(message?.contains("queries accessibility backend") == true)
-    XCTAssertTrue(message?.contains("fixing the app's accessibility") == true)
-    XCTAssertTrue(message?.contains("screenshot as visual truth") == true)
-    XCTAssertNil(
-      Self.legacyQualityMessage(
-        SnapshotQuality(
-          state: "healthy", backend: "tree", reason: nil, reasonCode: nil, effectiveDepth: nil,
-          collapsedLeafIndexes: nil)
-      )
-    )
-  }
-  func testTerminalFailsClosedOnInteractiveAxFailureRegardlessOfSparseBest() {
-    // Interactive AX failure must invalidate + fail closed; a later tier's sparse synthetic-root
-    // "best" must never downgrade this to a returned-sparse payload (regression: best == nil guard).
-    XCTAssertEqual(
-      Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: true),
-      .failClosed
-    )
-    XCTAssertEqual(
-      Self.resolveSnapshotPlanTerminal(terminal: .sparseWithFatalOnAXFailure, interactiveOnly: false),
-      .sparseBest
-    )
-    XCTAssertEqual(
-      Self.resolveSnapshotPlanTerminal(terminal: .throwOnAXFailure, interactiveOnly: true),
-      .throwAxFailure
-    )
-  }
-
-  func testSnapshotAccessibilityUnavailableCarriesSparseVerdict() {
-    currentApp = app
-    currentBundleId = "com.example.app"
-    defer {
-      currentApp = nil
-      currentBundleId = nil
-    }
-    let payload = snapshotAccessibilityUnavailable(
-      failure: SnapshotCaptureFailure(
-        code: "IOS_AX_SNAPSHOT_FAILED",
-        message: "kAXErrorIllegalArgument",
-        hint: "use screenshot"
-      )
-    )
-    XCTAssertEqual(payload.runnerFatal, true)
-    XCTAssertEqual(payload.snapshotQuality?.state, "sparse")
-    XCTAssertEqual(payload.snapshotQuality?.reasonCode, "ax-rejected")
-  }
-}
-#endif