agent-device 0.21.8 → 0.21.9

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift

@@ -0,0 +1,175 @@
+import XCTest
+#if canImport(UIKit)
+import UIKit
+#endif
+
+
+
+
+struct CapturedAppScreen {
+  let image: RunnerImage
+  let displayID: UInt
+  let pixelWidth: Int
+  let pixelHeight: Int
+  let pixelsPerPoint: Double
+}
+
+
+
+
+enum RunnerAppScreenCaptureFailure: String, Error {
+  case unresolvedWindow = "APP_SCREEN_WINDOW_UNRESOLVED"
+  case unresolvedScreen = "APP_SCREEN_UNRESOLVED"
+  case unrenderableImage = "APP_SCREEN_CAPTURE_UNRENDERABLE"
+
+  init(_ failure: RunnerApplicationScreenFailure) {
+    switch failure {
+    case .unresolvedWindow:
+      self = .unresolvedWindow
+    default:
+      self = .unresolvedScreen
+    }
+  }
+
+  var message: String {
+    switch self {
+    case .unresolvedWindow:
+      return "Neither the target app nor the system surface resolved a window, so no display could be named."
+    case .unresolvedScreen:
+      return "The resolved app window did not report the display it is shown on."
+    case .unrenderableImage:
+      return "The resolved display handed back an image that could not be drawn upright."
+    }
+  }
+
+  var hint: String {
+    switch self {
+    case .unresolvedWindow:
+      return "The capture follows the display that owns a window, so this is a runtime that is answering no window for any process; runner.log names the window frame and the query answer it refused."
+    case .unresolvedScreen:
+      return "Retry the capture. If it persists, the XCTest runtime is not reporting the window's display; runner.log names the window frame it resolved."
+    case .unrenderableImage:
+      return "Retry the capture. The display was resolved, so this is the capture itself failing, not a panel question."
+    }
+  }
+}
+
+#if canImport(UIKit) && os(iOS)
+extension RunnerTests {
+
+
+
+
+
+
+
+
+
+
+
+  func captureResolvedAppScreen(
+    app: XCUIApplication
+  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
+    var outcome: Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> = .failure(
+      .unresolvedScreen
+    )
+    let capture = {
+      outcome = Self.resolveCapturedAppScreen(app: app)
+    }
+    if Thread.isMainThread {
+      capture()
+    } else {
+      DispatchQueue.main.sync(execute: capture)
+    }
+    return outcome
+  }
+
+
+
+
+
+
+
+
+
+
+
+  func captureObservedScreen(app: XCUIApplication) -> Result<
+    CapturedAppScreen, RunnerAppScreenCaptureFailure
+  > {
+    selectObservedScreenCapture(
+      resolving: { captureResolvedAppScreen(app: app) },
+      fallingBack: { captureResolvedAppScreen(app: springboard) }
+    )
+  }
+
+
+
+  func selectObservedScreenCapture(
+    resolving: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>,
+    fallingBack: () -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure>
+  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
+    let outcome = resolving()
+    if case .failure(.unresolvedWindow) = outcome {
+      return fallingBack()
+    }
+    return outcome
+  }
+
+  private static func resolveCapturedAppScreen(
+    app: XCUIApplication
+  ) -> Result<CapturedAppScreen, RunnerAppScreenCaptureFailure> {
+    var screen: AnyObject?
+    var displayID: UInt = 0
+    var resolution: RunnerApplicationScreenFailure = .unresolvedScreen
+    guard RunnerResolveApplicationScreen(app, &screen, &displayID, &resolution),
+      let captureScreen = screen as? XCUIScreen
+    else {
+      return .failure(RunnerAppScreenCaptureFailure(resolution))
+    }
+    let image = captureScreen.screenshot().image
+    guard let upright = runnerUprightCaptureImage(image) else {
+      return .failure(.unrenderableImage)
+    }
+
+
+
+    guard upright.scale.isFinite, upright.scale > 0 else {
+      return .failure(.unrenderableImage)
+    }
+    guard let cgImage = runnerCGImage(from: upright) else {
+      return .failure(.unrenderableImage)
+    }
+    return .success(
+      CapturedAppScreen(
+        image: upright,
+        displayID: displayID,
+        pixelWidth: cgImage.width,
+        pixelHeight: cgImage.height,
+        pixelsPerPoint: Double(upright.scale)
+      )
+    )
+  }
+
+
+
+
+
+
+
+  private static func runnerUprightCaptureImage(_ image: UIImage) -> UIImage? {
+    if image.imageOrientation == .up {
+      return image
+    }
+    guard image.size.width > 0, image.size.height > 0, image.scale.isFinite, image.scale > 0 else {
+      return nil
+    }
+    let format = UIGraphicsImageRendererFormat()
+    format.scale = image.scale
+    let renderer = UIGraphicsImageRenderer(size: image.size, format: format)
+    return renderer.image { _ in
+      image.draw(in: CGRect(origin: .zero, size: image.size))
+    }
+  }
+}
+#endif

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h

@@ -8,7 +8,11 @@ FOUNDATION_EXPORT double RunnerControlledScrollProgress(double t);
 
 @interface RunnerSynthesizedGesture : NSObject
 
+// `resolvedWindow` is the app window the caller already resolved for the gesture's reference
+// frame. The record's display ID is read from that same window so geometry and routing can never
+// name different windows; pass nil to fall back to the application's first window.
 + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
+                                        resolvedWindow:(id _Nullable)resolvedWindow
                                                     x:(double)x
                                                     y:(double)y
                                                    x2:(double)x2
@@ -16,26 +20,30 @@ FOUNDATION_EXPORT double RunnerControlledScrollProgress(double t);
                                             durationMs:(double)durationMs;
 
 + (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application
+                                                resolvedWindow:(id _Nullable)resolvedWindow
                                                              x:(double)x
                                                              y:(double)y
                                                             x2:(double)x2
-                                                             y2:(double)y2
+                                                            y2:(double)y2
                                                      durationMs:(double)durationMs;
 
 + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application
+                                                 resolvedWindow:(id _Nullable)resolvedWindow
                                                                 x:(double)x
                                                                 y:(double)y
                                                                x2:(double)x2
-                                                                y2:(double)y2
+                                                               y2:(double)y2
                                                         durationMs:(double)durationMs;
 
 + (NSString * _Nullable)synthesizeTapWithApplication:(id)application
+                                      resolvedWindow:(id _Nullable)resolvedWindow
                                                    x:(double)x
                                                    y:(double)y;
 
 // Each pointer is an ordered array of { x, y, offsetMs } samples. The first sample
 // starts contact; subsequent samples move it; all pointers lift at their final offset.
 + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application
+                                          resolvedWindow:(id _Nullable)resolvedWindow
                                           pointerSamples:(NSArray<NSArray<NSDictionary<NSString *, NSNumber *> *> *> *)pointerSamples;
 
 // UIInterfaceOrientation of the app (1 portrait, 2 upsideDown, 3 landscapeRight,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m

@@ -6,13 +6,13 @@
 
 static NSString *const RunnerGestureSynthesisSurface = @"event";
 
-typedef id (*RunnerMsgSendInitRecord)(id, SEL, NSString *, NSInteger);
+typedef id (*RunnerMsgSendInitRecord)(id, SEL, NSString *, NSUInteger, NSInteger);
 typedef id (*RunnerMsgSendInitPath)(id, SEL, CGPoint, NSTimeInterval);
 typedef void (*RunnerMsgSendPathMove)(id, SEL, CGPoint, NSTimeInterval);
 typedef void (*RunnerMsgSendPathOffset)(id, SEL, NSTimeInterval);
 
-// Gesture-specific extension of the shared bridge: the 2-arg
-// `initWithName:interfaceOrientation:` record initializer and the touch-path
+// Gesture-specific extension of the shared bridge: the display-aware
+// `initWithName:displayID:interfaceOrientation:` record initializer and the touch-path
 // factory/mutator selectors, none of which text-entry synthesis needs.
 typedef struct {
   RunnerXCTestEventBridge core;
@@ -37,6 +37,7 @@ static NSString * _Nullable RunnerResolveGestureEventBridge(
 );
 static NSString * _Nullable RunnerCreateEventRecord(
   id application,
+  id _Nullable resolvedWindow,
   NSString *recordName,
   RunnerGestureEventBridge *bridge,
   id *record
@@ -74,13 +75,14 @@ static id RunnerSampledDragPointerPath(
 static double RunnerSmoothstepProgress(double t);
 static NSString * _Nullable RunnerTrySynthesizeDrag(
   id application,
+  id _Nullable resolvedWindow,
   CGPoint start,
   CGPoint end,
   double durationMs,
   NSString *recordName,
   RunnerDragPointerPathFactory pathFactory
 );
-static NSString * _Nullable RunnerTrySynthesizeTap(id application, CGPoint point);
+static NSString * _Nullable RunnerTrySynthesizeTap(id application, id _Nullable resolvedWindow, CGPoint point);
 // XCTest's proven swipe profile reaches the endpoint in 100 ms, then holds for the planned
 // fling duration. Fast movement is what lets UIKit distinguish a fling from a timed pan.
 static const NSTimeInterval RunnerSwipeMovementDurationSeconds = 0.1;
@@ -94,14 +96,16 @@ static id RunnerTapPointerPath(
 @implementation RunnerSynthesizedGesture
 
 + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
-                                                    x:(double)x
-                                                    y:(double)y
-                                                   x2:(double)x2
-                                                   y2:(double)y2
+                                        resolvedWindow:(id _Nullable)resolvedWindow
+                                                     x:(double)x
+                                                     y:(double)y
+                                                    x2:(double)x2
+                                                    y2:(double)y2
                                             durationMs:(double)durationMs {
   @try {
     return RunnerTrySynthesizeDrag(
       application,
+      resolvedWindow,
       CGPointMake(x, y),
       CGPointMake(x2, y2),
       durationMs,
@@ -114,6 +118,7 @@ static id RunnerTapPointerPath(
 }
 
 + (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application
+                                                resolvedWindow:(id _Nullable)resolvedWindow
                                                              x:(double)x
                                                              y:(double)y
                                                             x2:(double)x2
@@ -122,6 +127,7 @@ static id RunnerTapPointerPath(
   @try {
     return RunnerTrySynthesizeDrag(
       application,
+      resolvedWindow,
       CGPointMake(x, y),
       CGPointMake(x2, y2),
       durationMs,
@@ -134,14 +140,16 @@ static id RunnerTapPointerPath(
 }
 
 + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application
+                                                 resolvedWindow:(id _Nullable)resolvedWindow
                                                                 x:(double)x
                                                                 y:(double)y
                                                                x2:(double)x2
-                                                                y2:(double)y2
-                                                        durationMs:(double)durationMs {
+                                                               y2:(double)y2
+                                                       durationMs:(double)durationMs {
   @try {
     return RunnerTrySynthesizeDrag(
       application,
+      resolvedWindow,
       CGPointMake(x, y),
       CGPointMake(x2, y2),
       durationMs,
@@ -154,22 +162,25 @@ static id RunnerTapPointerPath(
 }
 
 + (NSString * _Nullable)synthesizeTapWithApplication:(id)application
+                                      resolvedWindow:(id _Nullable)resolvedWindow
                                                    x:(double)x
                                                    y:(double)y {
   @try {
-    return RunnerTrySynthesizeTap(application, CGPointMake(x, y));
+    return RunnerTrySynthesizeTap(application, resolvedWindow, CGPointMake(x, y));
   } @catch (NSException *exception) {
     return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed");
   }
 }
 
 + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application
+                                          resolvedWindow:(id _Nullable)resolvedWindow
                                           pointerSamples:(NSArray<NSArray<NSDictionary<NSString *, NSNumber *> *> *> *)pointerSamples {
   @try {
     RunnerGestureEventBridge bridge;
     id record = nil;
     NSString *error = RunnerCreateEventRecord(
       application,
+      resolvedWindow,
       @"agent-device-gesture-plan",
       &bridge,
       &record
@@ -224,6 +235,7 @@ static id RunnerTapPointerPath(
 
 static NSString * _Nullable RunnerTrySynthesizeDrag(
   id application,
+  id _Nullable resolvedWindow,
   CGPoint start,
   CGPoint end,
   double durationMs,
@@ -232,7 +244,7 @@ static NSString * _Nullable RunnerTrySynthesizeDrag(
 ) {
   RunnerGestureEventBridge bridge;
   id record = nil;
-  NSString *error = RunnerCreateEventRecord(application, recordName, &bridge, &record);
+  NSString *error = RunnerCreateEventRecord(application, resolvedWindow, recordName, &bridge, &record);
   if (error != nil) return error;
 
   id path = pathFactory(&bridge, start, end, durationMs);
@@ -244,11 +256,12 @@ static NSString * _Nullable RunnerTrySynthesizeDrag(
   return RunnerSynthesizeEventRecord(&bridge, record);
 }
 
-static NSString * _Nullable RunnerTrySynthesizeTap(id application, CGPoint point) {
+static NSString * _Nullable RunnerTrySynthesizeTap(id application, id _Nullable resolvedWindow, CGPoint point) {
   RunnerGestureEventBridge bridge;
   id record = nil;
   NSString *error = RunnerCreateEventRecord(
     application,
+    resolvedWindow,
     @"agent-device-tap",
     &bridge,
     &record
@@ -271,14 +284,14 @@ static NSString * _Nullable RunnerResolveGestureEventBridge(
   NSString *missing = RunnerResolveXCTestEventBridge(application, RunnerGestureSynthesisSurface, &core);
   if (missing != nil) return missing;
 
-  SEL initRecordSelector = NSSelectorFromString(@"initWithName:interfaceOrientation:");
+  SEL initRecordSelector = NSSelectorFromString(@"initWithName:displayID:interfaceOrientation:");
   SEL interfaceOrientationSelector = NSSelectorFromString(@"interfaceOrientation");
   SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:");
   SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:");
   SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:");
 
   missing = RunnerRequireSelector(
-    core.recordClass, initRecordSelector, @"initWithName:interfaceOrientation:", RunnerGestureSynthesisSurface
+    core.recordClass, initRecordSelector, @"initWithName:displayID:interfaceOrientation:", RunnerGestureSynthesisSurface
   );
   if (missing != nil) return missing;
   missing = RunnerRequireSelector(
@@ -311,6 +324,7 @@ static NSString * _Nullable RunnerResolveGestureEventBridge(
 
 static NSString * _Nullable RunnerCreateEventRecord(
   id application,
+  id _Nullable resolvedWindow,
   NSString *recordName,
   RunnerGestureEventBridge *bridge,
   id *record
@@ -318,6 +332,12 @@ static NSString * _Nullable RunnerCreateEventRecord(
   NSString *missing = RunnerResolveGestureEventBridge(application, bridge);
   if (missing != nil) return missing;
 
+  NSUInteger displayID = 0;
+  if (resolvedWindow == nil) {
+    return @"private XCTest event synthesis unavailable: no resolved application window";
+  }
+  NSString *displayError = RunnerResolveWindowDisplayID(resolvedWindow, &displayID);
+  if (displayError != nil) return displayError;
   NSInteger interfaceOrientation =
     ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->interfaceOrientationSelector);
   NSInteger targetProcessID =
@@ -330,11 +350,18 @@ static NSString * _Nullable RunnerCreateEventRecord(
     [bridge->core.recordClass alloc],
     bridge->initRecordSelector,
     recordName,
+    displayID,
     interfaceOrientation
   );
   if (eventRecord == nil) {
     return @"private XCTest event synthesis failed: could not create event record";
   }
+  NSLog(
+    @"AGENT_DEVICE_RUNNER_SYNTHESIZED_RECORD name=%@ displayID=%lu interfaceOrientation=%ld",
+    recordName,
+    (unsigned long)displayID,
+    (long)interfaceOrientation
+  );
   ((RunnerMsgSendSetInteger)objc_msgSend)(
     eventRecord,
     bridge->core.setTargetProcessIDSelector,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift

@@ -68,6 +68,14 @@ extension RunnerTests {
       guard waitUntilAlertButtonHittable(button, deadline: deadline) else {
         return alertVerificationResponse(.timedOut, action: action, activated: false)
       }
+      let buttonFrame = button.frame
+      NSLog(
+        "AGENT_DEVICE_RUNNER_ALERT_ACTIVATION action=%@ label=%@ frame=(%.1f,%.1f,%.1f,%.1f) point=(%.1f,%.1f)",
+        action,
+        button.label,
+        buttonFrame.origin.x, buttonFrame.origin.y, buttonFrame.size.width, buttonFrame.size.height,
+        buttonFrame.midX, buttonFrame.midY
+      )
 
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -24,6 +24,7 @@ import AgentDeviceSnapshotPresentation
 
 
 
+
 
 
 extension RunnerTests {
@@ -1465,7 +1466,7 @@ extension RunnerTests {
           fps: command.fps.map { Int32($0) }
         )
         try recorder.start { [weak self] in
-          return self?.captureRunnerFrame()
+          return self?.captureRunnerFrame(app: activeApp)
         }
         activeRecording = recorder
         return Response(ok: true, data: DataPayload(message: "recording started"))
@@ -2054,7 +2055,6 @@ extension RunnerTests {
     case .snapshot:
       return try executeSnapshotPrepared(command: command, activeApp: activeApp)
     case .screenshot:
-      let screenshot: XCUIScreenshot
 #if os(macOS)
 
       if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
@@ -2064,6 +2064,7 @@ extension RunnerTests {
 
         sleepFor(0.5)
       }
+      let screenshot: XCUIScreenshot
       if command.fullscreen == true {
         screenshot = XCUIScreen.main.screenshot()
       } else if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
@@ -2071,30 +2072,52 @@ extension RunnerTests {
       } else {
         screenshot = XCUIScreen.main.screenshot()
       }
-#else
-      screenshot = XCUIScreen.main.screenshot()
-#endif
-      guard let pngData = runnerPngData(for: screenshot.image) else {
-        return Response(ok: false, error: ErrorPayload(message: "Failed to encode screenshot as PNG"))
-      }
-      if command.inlineScreenshot == true {
+      return screenshotResponse(
+        image: screenshot.image,
+        inlineScreenshot: command.inlineScreenshot == true
+      )
+    #elseif os(iOS)
+
+
+
+
+
+
+
+
+
+      switch captureObservedScreen(app: resolveAppWithoutActivation(command: command)) {
+      case .failure(let failure):
         return Response(
-          ok: true,
-          data: DataPayload(imageBase64: pngData.base64EncodedString())
+          ok: false,
+          error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)
+        )
+      case .success(let captured):
+
+
+        NSLog(
+          "AGENT_DEVICE_RUNNER_SCREEN_CAPTURE display=%lu pixels=%ldx%ld pixelsPerPoint=%g",
+          captured.displayID,
+          captured.pixelWidth,
+          captured.pixelHeight,
+          captured.pixelsPerPoint
+        )
+        return screenshotResponse(
+          image: captured.image,
+          inlineScreenshot: command.inlineScreenshot == true,
+          metadata: ScreenshotMetadataPayload(
+            displayID: captured.displayID,
+            pixelWidth: captured.pixelWidth,
+            pixelHeight: captured.pixelHeight,
+            pixelsPerPoint: captured.pixelsPerPoint
+          )
         )
       }
-      let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png"
-      let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName)
-      do {
-        try pngData.write(to: URL(fileURLWithPath: filePath))
-      } catch {
-        return Response(ok: false, error: ErrorPayload(message: "Failed to write screenshot: \(error.localizedDescription)"))
-      }
-#if os(macOS)
-      return Response(ok: true, data: DataPayload(message: filePath))
 #else
-
-      return Response(ok: true, data: DataPayload(message: "tmp/\(fileName)"))
+      return screenshotResponse(
+        image: XCUIScreen.main.screenshot().image,
+        inlineScreenshot: command.inlineScreenshot == true
+      )
 #endif
     case .back, .backInApp:
       if tapInAppBackControl(app: activeApp) {