agent-device 0.21.12 → 0.21.14

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -1,30 +1,70 @@
+import AgentDeviceSnapshotPresentation
 import XCTest
 
 extension RunnerTests {
   static let navigationBackKeywords = ["back", "close", "cancel"]
   static let navigationFallbackVerificationDelay: TimeInterval = 0.25
 
-  func tapInAppBackControl(app: XCUIApplication) -> Bool {
+
+
+
+  enum InAppBackOutcome {
+    case performed
+    case unavailable
+    case unverified(ErrorPayload)
+  }
+
+
+
+  enum NavigationVisualObservation: Equatable {
+    case changed
+    case unchanged
+    case unobserved
+
+    var logToken: String {
+      switch self {
+      case .changed: return "yes"
+      case .unchanged: return "no"
+      case .unobserved: return "unknown"
+      }
+    }
+  }
+
+
+
+  struct NavigationVisualSample {
+    let data: Data?
+    let refusalCode: String?
+    let refusalHint: String?
+
+    init(data: Data?, refusalCode: String? = nil, refusalHint: String? = nil) {
+      self.data = data
+      self.refusalCode = refusalCode
+      self.refusalHint = refusalHint
+    }
+  }
+
+  func tapInAppBackControl(app: XCUIApplication) -> InAppBackOutcome {
 #if os(macOS)
     if let back = macOSNavigationBackElement(app: app) {
       tapElementCenter(app: app, element: back)
-      return true
+      return .performed
     }
-    return false
+    return .unavailable
 #elseif os(tvOS)
     _ = pressTvRemote(.menu)
-    return true
+    return .performed
 #else
     let buttons = app.navigationBars.buttons.allElementsBoundByIndex
     if let back = buttons.first(where: { $0.isHittable }) {
       back.tap()
-      return true
+      return .performed
     }
     if isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) {
       NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", currentBundleId ?? "")
     } else if let back = topNavigationBackElement(app: app) {
       tapElementCenter(app: app, element: back)
-      return true
+      return .performed
     }
     return tapTopLeadingNavigationFallback(app: app)
 #endif
@@ -81,16 +121,9 @@ extension RunnerTests {
     return navigationBackKeywords.firstIndex { text.contains($0) }
   }
 
-
-  static func isUsableNavigationFrame(_ frame: CGRect) -> Bool {
-    guard frame.width.isFinite, frame.height.isFinite, frame.width > 0, frame.height > 0 else {
-      return false
-    }
-    return !frame.isInfinite
-  }
-
   static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool {
-    guard isUsableNavigationFrame(candidate), isUsableNavigationFrame(window) else {
+    guard SnapshotGeometry.isPositiveFinite(candidate), SnapshotGeometry.isPositiveFinite(window)
+    else {
       return false
     }
 
@@ -99,7 +132,7 @@ extension RunnerTests {
   }
 
   static func topLeadingNavigationFallbackPoint(in frame: CGRect) -> CGPoint? {
-    guard isUsableNavigationFrame(frame) else {
+    guard SnapshotGeometry.isPositiveFinite(frame) else {
       return nil
     }
 
@@ -109,122 +142,48 @@ extension RunnerTests {
     return CGPoint(x: frame.minX + xOffset, y: frame.minY + yOffset)
   }
 
-  private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> Bool {
+  private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> InAppBackOutcome {
 #if os(iOS)
     let frame = onScreenWindowFrame(app: app)
     guard let point = Self.topLeadingNavigationFallbackPoint(in: frame) else {
-      return false
+      return .unavailable
     }
     let before = captureNavigationFallbackVisualState(app: app)
     let context = synthesizedCoordinateContext(
       app: app,
       policy: synthesizedGesturePolicy(.coordinateTap)
     )?.withReferenceFrame(frame)
-    let synthesized = performGesture(app, idleTimeout: false) {
+    switch performSynthesizedGesture(app, kind: .coordinateTap, context: context, synthesize: {
       synthesizedTapAt(app: app, x: point.x, y: point.y, context: context)
-    }
-    if case .performed = synthesized.outcome {
-      return didNavigationFallbackChangeVisualState(app: app, before: before)
-    }
-    let fallback = performGesture(app) {
-      tapAt(app: app, x: point.x, y: point.y)
-    }
-    if case .performed = fallback.outcome {
-      return didNavigationFallbackChangeVisualState(app: app, before: before)
+    }) {
+    case .performed:
+      return verifyNavigationFallbackOutcome(app: app, before: before)
+    case .refused:
+      return .unavailable
+    case .xctestFallback:
+      let fallback = performGesture(app) {
+        tapAt(app: app, x: point.x, y: point.y)
+      }
+      if case .performed = fallback.outcome {
+        return verifyNavigationFallbackOutcome(app: app, before: before)
+      }
     }
 #endif
-    return false
+    return .unavailable
   }
 
-  private func captureNavigationFallbackVisualState(app: XCUIApplication) -> Data? {
+  private func captureNavigationFallbackVisualState(app: XCUIApplication) -> NavigationVisualSample {
 #if os(iOS)
-
-
-
-
-    guard case .success(let captured) = captureObservedScreen(app: app) else {
-      return nil
-    }
-    return runnerPngData(for: captured.image)
+    return Self.navigationFallbackSample(
+      resolvingApp: { self.captureResolvedAppScreen(app: app) },
+      systemSurface: { self.captureResolvedAppScreen(app: self.springboard) },
+      encoding: { runnerPngData(for: $0.image) }
+    )
 #else
-    return nil
+    return NavigationVisualSample(data: nil)
 #endif
   }
 
-  private func didNavigationFallbackChangeVisualState(
-    app: XCUIApplication,
-    before: Data?
-  ) -> Bool {
-    sleepFor(Self.navigationFallbackVerificationDelay)
-    let after = captureNavigationFallbackVisualState(app: app)
-    let changed = Self.didNavigationFallbackChangeVisualState(before: before, after: after)
-
-
-    NSLog(
-      "AGENT_DEVICE_RUNNER_IN_APP_BACK_VISUAL_VERIFICATION beforeBytes=%ld afterBytes=%ld changed=%@",
-      before?.count ?? -1,
-      after?.count ?? -1,
-      changed ? "yes" : "no"
-    )
-    return changed
-  }
-
-  static func didNavigationFallbackChangeVisualState(before: Data?, after: Data?) -> Bool {
-    guard let before, let after else { return false }
-    return before != after
-  }
-
-  private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
-    let predicate = NSPredicate(
-      format: "identifier == %@ OR label == %@",
-      "go back",
-      "Back"
-    )
-    let element = app.descendants(matching: .any).matching(predicate).firstMatch
-    return element.exists ? element : nil
-  }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 
 
 
@@ -232,23 +191,113 @@ extension RunnerTests {
 
 
 
+  static func navigationFallbackSample(
+    resolvingApp: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
+    systemSurface: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
+    encoding: (CapturedAppScreen) -> Data?
+  ) -> NavigationVisualSample {
+    _ = systemSurface
+    return navigationVisualSample(from: resolvingApp(), encoding: encoding)
+  }
 
 
 
 
 
+  static func navigationVisualSample(
+    from outcome: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
+    encoding: (CapturedAppScreen) -> Data?
+  ) -> NavigationVisualSample {
+    switch outcome {
+    case .success(let captured):
+      guard let png = encoding(captured) else {
+        let refusal = RunnerAppScreenCaptureFailure.unrenderableImage
+        return NavigationVisualSample(
+          data: nil,
+          refusalCode: refusal.rawValue,
+          refusalHint: refusal.hint
+        )
+      }
+      return NavigationVisualSample(data: png)
+    case .failure(let failure):
+      return NavigationVisualSample(
+        data: nil,
+        refusalCode: failure.rawValue,
+        refusalHint: failure.hint
+      )
+    }
+  }
 
+  private func verifyNavigationFallbackOutcome(
+    app: XCUIApplication,
+    before: NavigationVisualSample
+  ) -> InAppBackOutcome {
+    sleepFor(Self.navigationFallbackVerificationDelay)
+    let after = captureNavigationFallbackVisualState(app: app)
+    let observation = Self.navigationVisualObservation(before: before.data, after: after.data)
 
 
+    NSLog(
+      "AGENT_DEVICE_RUNNER_IN_APP_BACK_VISUAL_VERIFICATION beforeBytes=%ld afterBytes=%ld changed=%@",
+      before.data?.count ?? -1,
+      after.data?.count ?? -1,
+      observation.logToken
+    )
+    return Self.inAppBackOutcome(observation: observation, before: before, after: after)
+  }
 
 
 
+  static func inAppBackOutcome(
+    observation: NavigationVisualObservation,
+    before: NavigationVisualSample,
+    after: NavigationVisualSample
+  ) -> InAppBackOutcome {
+    switch observation {
+    case .changed:
+      return .performed
+    case .unchanged:
+      return .unavailable
+    case .unobserved:
 
 
+      return .unverified(Self.navigationFallbackErrorPayload(after: after, before: before))
+    }
+  }
 
+  static func navigationVisualObservation(
+    before: Data?,
+    after: Data?
+  ) -> NavigationVisualObservation {
+    guard let before, let after else { return .unobserved }
+    return before != after ? .changed : .unchanged
+  }
 
 
 
 
+  static func navigationFallbackErrorPayload(
+    after: NavigationVisualSample,
+    before: NavigationVisualSample
+  ) -> ErrorPayload {
+    ErrorPayload(
+      code:
+        after.refusalCode
+        ?? before.refusalCode
+        ?? RunnerAppScreenCaptureFailure.unresolvedScreen.rawValue,
+      message:
+        "The in-app back fallback was dispatched, but no display could be sampled to confirm the result. This is an unknown outcome, not evidence that a back control is absent.",
+      hint: after.refusalHint ?? before.refusalHint
+    )
+  }
 
+  private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
+    let predicate = NSPredicate(
+      format: "identifier == %@ OR label == %@",
+      "go back",
+      "Back"
+    )
+    let element = app.descendants(matching: .any).matching(predicate).firstMatch
+    return element.exists ? element : nil
+  }
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift

@@ -89,212 +89,3 @@ extension RunnerTests {
     (value as? Bool) ?? (value as? NSNumber)?.boolValue
   }
 }
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift

@@ -25,12 +25,20 @@ extension RunnerTests {
     private var startedSession = false
     private var startError: Error?
 
+
+
+
     init(outputPath: String, fps: Int32?) {
       self.outputPath = outputPath
       self.fps = fps
     }
 
-    func start(captureFrame: @escaping () -> RunnerImage?) throws {
+
+
+    func start(
+      bootstrap: @escaping () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
+      frame: @escaping () -> RunnerImage?
+    ) throws {
       let url = URL(fileURLWithPath: outputPath)
       let directory = url.deletingLastPathComponent()
       try FileManager.default.createDirectory(
@@ -44,21 +52,34 @@ extension RunnerTests {
 
       var dimensions: CGSize = .zero
       var bootstrapImage: RunnerImage?
+      var lastFailure: RunnerAppScreenCaptureFailure?
       let bootstrapDeadline = Date().addingTimeInterval(2.0)
       while Date() < bootstrapDeadline {
-        if let image = captureFrame(), let cgImage = runnerCGImage(from: image) {
-          bootstrapImage = image
-          dimensions = CGSize(width: cgImage.width, height: cgImage.height)
+        switch bootstrap() {
+        case .success(let captured):
+          bootstrapImage = captured.image
+          dimensions = CGSize(width: captured.pixelWidth, height: captured.pixelHeight)
+        case .failure(let failure):
+          lastFailure = failure
+        }
+        if dimensions.width > 0, dimensions.height > 0 {
           break
         }
         Thread.sleep(forTimeInterval: 0.05)
       }
       guard dimensions.width > 0, dimensions.height > 0 else {
-        throw NSError(
-          domain: "AgentDeviceRunner.Record",
-          code: 1,
-          userInfo: [NSLocalizedDescriptionKey: "failed to capture initial frame"]
-        )
+
+
+
+
+        #if os(iOS)
+        throw RunnerTests.recordingBootstrapError(from: lastFailure)
+        #else
+
+
+        _ = lastFailure
+        throw RunnerTests.recordingBootstrapError(from: nil)
+        #endif
       }
 
       let writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
@@ -114,7 +135,7 @@ extension RunnerTests {
       timer.setEventHandler { [weak self] in
         guard let self else { return }
         if self.shouldStop() { return }
-        guard let image = captureFrame() else { return }
+        guard let image = frame() else { return }
         self.append(image: image)
       }
       self.timer = timer
@@ -212,6 +233,9 @@ extension RunnerTests {
         return
       }
       lastTimestampValue = timestampValue
+
+
+
     }
 
     private func timestampCandidateValue(for nowUptime: TimeInterval) -> Int64 {
@@ -269,6 +293,58 @@ extension RunnerTests {
   }
 }
 
+extension RunnerTests {
+
+
+
+
+
+  func startRecording(
+    _ recorder: ScreenRecorder,
+    capture: @escaping () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
+  ) throws {
+    try recorder.start(bootstrap: capture) { [weak self] in
+      guard let self else { return nil }
+      return try? self.runMainThreadWorkIfIdle(
+        "recording_frame",
+        timeout: self.recordingFrameCaptureTimeout,
+        timeoutError: self.mainThreadExecutionTimeoutError
+      ) {
+        try capture().get().image
+      }
+    }
+  }
+
+
+
+
+
+  static func recordingBootstrapError(from lastFailure: RunnerAppScreenCaptureFailure?) -> Error {
+    lastFailure
+      ?? NSError(
+        domain: "AgentDeviceRunner.Record",
+        code: 1,
+        userInfo: [NSLocalizedDescriptionKey: "failed to capture initial frame"]
+      )
+  }
+
+
+
+
+  static func recordingStartErrorPayload(for error: Error) -> ErrorPayload {
+    if let failure = error as? RunnerAppScreenCaptureFailure {
+      return ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)
+    }
+    return ErrorPayload(message: "failed to start recording: \(error.localizedDescription)")
+  }
+}
+
+
+
+
+
+
+