agent-device 0.21.7 → 0.21.9

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift

@@ -212,6 +212,7 @@ extension RunnerTests {
     if tapKeyboardDismissControl(app: app) {
       _ = keyboard.waitForNonExistence(timeout: KeyboardDismissObservationTiming.timeout)
       waitForScreenshotStability(
+        app: app,
         timeout: KeyboardDismissObservationTiming.settleTimeout,
         sampleInterval: KeyboardDismissObservationTiming.settleSampleInterval,
         requiredConsecutiveMatches: KeyboardDismissObservationTiming.settleRequiredConsecutiveMatches
@@ -240,23 +241,38 @@ extension RunnerTests {
 
 
   private func waitForScreenshotStability(
+    app: XCUIApplication,
     timeout: TimeInterval,
     sampleInterval: TimeInterval,
     requiredConsecutiveMatches: Int
   ) {
     let deadline = Date().addingTimeInterval(timeout)
-    var samples: [Data?] = [screenshotFingerprintForStabilityCheck()]
+    var samples: [Data?] = [screenshotFingerprintForStabilityCheck(app: app)]
     while Date() < deadline {
       sleepFor(sampleInterval)
-      samples.append(screenshotFingerprintForStabilityCheck())
+      samples.append(screenshotFingerprintForStabilityCheck(app: app))
       if runnerScreenshotStabilitySettled(samples, requiredConsecutiveMatches: requiredConsecutiveMatches) {
+        logScreenshotStability(samples: samples, settled: true)
         return
       }
     }
+    logScreenshotStability(samples: samples, settled: false)
+  }
+
+
+
+
+  private func logScreenshotStability(samples: [Data?], settled: Bool) {
+    NSLog(
+      "AGENT_DEVICE_RUNNER_KEYBOARD_STABILITY samples=%ld captured=%ld settled=%@",
+      samples.count,
+      samples.compactMap { $0 }.count,
+      settled ? "yes" : "no"
+    )
   }
 
-  private func screenshotFingerprintForStabilityCheck() -> Data? {
-    guard let image = captureRunnerFrame() else { return nil }
+  private func screenshotFingerprintForStabilityCheck(app: XCUIApplication) -> Data? {
+    guard let image = captureRunnerFrame(app: app) else { return nil }
     return runnerPngData(for: image)
   }
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -42,7 +42,20 @@ enum RunnerInteractionIdleWaits {
 extension RunnerTests {
 
 
-  func captureRunnerFrame() -> RunnerImage? {
+
+
+
+
+
+
+
+  func captureRunnerFrame(app: XCUIApplication) -> RunnerImage? {
+#if os(iOS)
+    guard case .success(let captured) = captureObservedScreen(app: app) else {
+      return nil
+    }
+    return captured.image
+#else
     var image: RunnerImage?
     let capture = {
       let screenshot = XCUIScreen.main.screenshot()
@@ -54,6 +67,7 @@ extension RunnerTests {
       DispatchQueue.main.sync(execute: capture)
     }
     return image
+#endif
   }
 
   func screenshotRoot(app: XCUIApplication) -> XCUIElement {
@@ -66,6 +80,54 @@ extension RunnerTests {
     return app
   }
 
+
+
+
+
+  func screenshotResponse(
+    pngData: Data,
+    inlineScreenshot: Bool,
+    metadata: ScreenshotMetadataPayload? = nil
+  ) -> Response {
+    if inlineScreenshot {
+      return Response(
+        ok: true,
+        data: DataPayload(imageBase64: pngData.base64EncodedString(), screenshotMetadata: metadata)
+      )
+    }
+    let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png"
+    let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName)
+    do {
+      try pngData.write(to: URL(fileURLWithPath: filePath))
+    } catch {
+      return Response(
+        ok: false,
+        error: ErrorPayload(message: "Failed to write screenshot: \(error.localizedDescription)")
+      )
+    }
+#if os(macOS)
+    return Response(ok: true, data: DataPayload(message: filePath, screenshotMetadata: metadata))
+#else
+
+    return Response(
+      ok: true,
+      data: DataPayload(message: "tmp/\(fileName)", screenshotMetadata: metadata)
+    )
+#endif
+  }
+
+
+  func screenshotResponse(
+    image: RunnerImage,
+    inlineScreenshot: Bool,
+    metadata: ScreenshotMetadataPayload? = nil
+  ) -> Response {
+    guard let pngData = runnerPngData(for: image) else {
+      return Response(ok: false, error: ErrorPayload(message: "Failed to encode screenshot as PNG"))
+    }
+    return screenshotResponse(pngData: pngData, inlineScreenshot: inlineScreenshot, metadata: metadata)
+  }
+
   func stopRecordingIfNeeded() {
     guard let recorder = activeRecording else { return }
     do {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -263,6 +263,16 @@ extension Response {
 
 
 
+struct ScreenshotMetadataPayload: Codable {
+  let displayID: UInt
+  let pixelWidth: Int
+  let pixelHeight: Int
+  let pixelsPerPoint: Double
+}
+
+
+
+
 
 
 
@@ -330,6 +340,9 @@ struct DataPayload: Codable {
   var failedStepIndex: Int?
   var sequenceResults: [SequenceStepResult]?
   var targetActivation: TargetActivationFactPayload?
+
+
+  var screenshotMetadata: ScreenshotMetadataPayload?
 }
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -115,7 +115,7 @@ extension RunnerTests {
     guard let point = Self.topLeadingNavigationFallbackPoint(in: frame) else {
       return false
     }
-    let before = captureNavigationFallbackVisualState()
+    let before = captureNavigationFallbackVisualState(app: app)
     let context = synthesizedCoordinateContext(
       app: app,
       policy: synthesizedGesturePolicy(.coordinateTap)
@@ -124,33 +124,48 @@ extension RunnerTests {
       synthesizedTapAt(app: app, x: point.x, y: point.y, context: context)
     }
     if case .performed = synthesized.outcome {
-      return didNavigationFallbackChangeVisualState(before: before)
+      return didNavigationFallbackChangeVisualState(app: app, before: before)
     }
     let fallback = performGesture(app) {
       tapAt(app: app, x: point.x, y: point.y)
     }
     if case .performed = fallback.outcome {
-      return didNavigationFallbackChangeVisualState(before: before)
+      return didNavigationFallbackChangeVisualState(app: app, before: before)
     }
 #endif
     return false
   }
 
-  private func captureNavigationFallbackVisualState() -> Data? {
+  private func captureNavigationFallbackVisualState(app: XCUIApplication) -> Data? {
 #if os(iOS)
-    runnerPngData(for: XCUIScreen.main.screenshot().image)
+
+
+
+
+    guard case .success(let captured) = captureObservedScreen(app: app) else {
+      return nil
+    }
+    return runnerPngData(for: captured.image)
 #else
     return nil
 #endif
   }
 
-  private func didNavigationFallbackChangeVisualState(before: Data?) -> Bool {
+  private func didNavigationFallbackChangeVisualState(
+    app: XCUIApplication,
+    before: Data?
+  ) -> Bool {
     sleepFor(Self.navigationFallbackVerificationDelay)
-    let after = captureNavigationFallbackVisualState()
+    let after = captureNavigationFallbackVisualState(app: app)
     let changed = Self.didNavigationFallbackChangeVisualState(before: before, after: after)
-    if !changed {
-      NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_FALLBACK_NO_STATE_CHANGE")
-    }
+
+
+    NSLog(
+      "AGENT_DEVICE_RUNNER_IN_APP_BACK_VISUAL_VERIFICATION beforeBytes=%ld afterBytes=%ld changed=%@",
+      before?.count ?? -1,
+      after?.count ?? -1,
+      changed ? "yes" : "no"
+    )
     return changed
   }