agent-device 0.21.7 → 0.21.9

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m

@@ -6,13 +6,13 @@
 
 static NSString *const RunnerGestureSynthesisSurface = @"event";
 
-typedef id (*RunnerMsgSendInitRecord)(id, SEL, NSString *, NSInteger);
+typedef id (*RunnerMsgSendInitRecord)(id, SEL, NSString *, NSUInteger, NSInteger);
 typedef id (*RunnerMsgSendInitPath)(id, SEL, CGPoint, NSTimeInterval);
 typedef void (*RunnerMsgSendPathMove)(id, SEL, CGPoint, NSTimeInterval);
 typedef void (*RunnerMsgSendPathOffset)(id, SEL, NSTimeInterval);
 
-// Gesture-specific extension of the shared bridge: the 2-arg
-// `initWithName:interfaceOrientation:` record initializer and the touch-path
+// Gesture-specific extension of the shared bridge: the display-aware
+// `initWithName:displayID:interfaceOrientation:` record initializer and the touch-path
 // factory/mutator selectors, none of which text-entry synthesis needs.
 typedef struct {
   RunnerXCTestEventBridge core;
@@ -37,6 +37,7 @@ static NSString * _Nullable RunnerResolveGestureEventBridge(
 );
 static NSString * _Nullable RunnerCreateEventRecord(
   id application,
+  id _Nullable resolvedWindow,
   NSString *recordName,
   RunnerGestureEventBridge *bridge,
   id *record
@@ -74,13 +75,14 @@ static id RunnerSampledDragPointerPath(
 static double RunnerSmoothstepProgress(double t);
 static NSString * _Nullable RunnerTrySynthesizeDrag(
   id application,
+  id _Nullable resolvedWindow,
   CGPoint start,
   CGPoint end,
   double durationMs,
   NSString *recordName,
   RunnerDragPointerPathFactory pathFactory
 );
-static NSString * _Nullable RunnerTrySynthesizeTap(id application, CGPoint point);
+static NSString * _Nullable RunnerTrySynthesizeTap(id application, id _Nullable resolvedWindow, CGPoint point);
 // XCTest's proven swipe profile reaches the endpoint in 100 ms, then holds for the planned
 // fling duration. Fast movement is what lets UIKit distinguish a fling from a timed pan.
 static const NSTimeInterval RunnerSwipeMovementDurationSeconds = 0.1;
@@ -94,14 +96,16 @@ static id RunnerTapPointerPath(
 @implementation RunnerSynthesizedGesture
 
 + (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
-                                                    x:(double)x
-                                                    y:(double)y
-                                                   x2:(double)x2
-                                                   y2:(double)y2
+                                        resolvedWindow:(id _Nullable)resolvedWindow
+                                                     x:(double)x
+                                                     y:(double)y
+                                                    x2:(double)x2
+                                                    y2:(double)y2
                                             durationMs:(double)durationMs {
   @try {
     return RunnerTrySynthesizeDrag(
       application,
+      resolvedWindow,
       CGPointMake(x, y),
       CGPointMake(x2, y2),
       durationMs,
@@ -114,6 +118,7 @@ static id RunnerTapPointerPath(
 }
 
 + (NSString * _Nullable)synthesizeContinuousDragWithApplication:(id)application
+                                                resolvedWindow:(id _Nullable)resolvedWindow
                                                              x:(double)x
                                                              y:(double)y
                                                             x2:(double)x2
@@ -122,6 +127,7 @@ static id RunnerTapPointerPath(
   @try {
     return RunnerTrySynthesizeDrag(
       application,
+      resolvedWindow,
       CGPointMake(x, y),
       CGPointMake(x2, y2),
       durationMs,
@@ -134,14 +140,16 @@ static id RunnerTapPointerPath(
 }
 
 + (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application
+                                                 resolvedWindow:(id _Nullable)resolvedWindow
                                                                 x:(double)x
                                                                 y:(double)y
                                                                x2:(double)x2
-                                                                y2:(double)y2
-                                                        durationMs:(double)durationMs {
+                                                               y2:(double)y2
+                                                       durationMs:(double)durationMs {
   @try {
     return RunnerTrySynthesizeDrag(
       application,
+      resolvedWindow,
       CGPointMake(x, y),
       CGPointMake(x2, y2),
       durationMs,
@@ -154,22 +162,25 @@ static id RunnerTapPointerPath(
 }
 
 + (NSString * _Nullable)synthesizeTapWithApplication:(id)application
+                                      resolvedWindow:(id _Nullable)resolvedWindow
                                                    x:(double)x
                                                    y:(double)y {
   @try {
-    return RunnerTrySynthesizeTap(application, CGPointMake(x, y));
+    return RunnerTrySynthesizeTap(application, resolvedWindow, CGPointMake(x, y));
   } @catch (NSException *exception) {
     return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed");
   }
 }
 
 + (NSString * _Nullable)synthesizeGestureWithApplication:(id)application
+                                          resolvedWindow:(id _Nullable)resolvedWindow
                                           pointerSamples:(NSArray<NSArray<NSDictionary<NSString *, NSNumber *> *> *> *)pointerSamples {
   @try {
     RunnerGestureEventBridge bridge;
     id record = nil;
     NSString *error = RunnerCreateEventRecord(
       application,
+      resolvedWindow,
       @"agent-device-gesture-plan",
       &bridge,
       &record
@@ -224,6 +235,7 @@ static id RunnerTapPointerPath(
 
 static NSString * _Nullable RunnerTrySynthesizeDrag(
   id application,
+  id _Nullable resolvedWindow,
   CGPoint start,
   CGPoint end,
   double durationMs,
@@ -232,7 +244,7 @@ static NSString * _Nullable RunnerTrySynthesizeDrag(
 ) {
   RunnerGestureEventBridge bridge;
   id record = nil;
-  NSString *error = RunnerCreateEventRecord(application, recordName, &bridge, &record);
+  NSString *error = RunnerCreateEventRecord(application, resolvedWindow, recordName, &bridge, &record);
   if (error != nil) return error;
 
   id path = pathFactory(&bridge, start, end, durationMs);
@@ -244,11 +256,12 @@ static NSString * _Nullable RunnerTrySynthesizeDrag(
   return RunnerSynthesizeEventRecord(&bridge, record);
 }
 
-static NSString * _Nullable RunnerTrySynthesizeTap(id application, CGPoint point) {
+static NSString * _Nullable RunnerTrySynthesizeTap(id application, id _Nullable resolvedWindow, CGPoint point) {
   RunnerGestureEventBridge bridge;
   id record = nil;
   NSString *error = RunnerCreateEventRecord(
     application,
+    resolvedWindow,
     @"agent-device-tap",
     &bridge,
     &record
@@ -271,14 +284,14 @@ static NSString * _Nullable RunnerResolveGestureEventBridge(
   NSString *missing = RunnerResolveXCTestEventBridge(application, RunnerGestureSynthesisSurface, &core);
   if (missing != nil) return missing;
 
-  SEL initRecordSelector = NSSelectorFromString(@"initWithName:interfaceOrientation:");
+  SEL initRecordSelector = NSSelectorFromString(@"initWithName:displayID:interfaceOrientation:");
   SEL interfaceOrientationSelector = NSSelectorFromString(@"interfaceOrientation");
   SEL initPathSelector = NSSelectorFromString(@"initForTouchAtPoint:offset:");
   SEL moveSelector = NSSelectorFromString(@"moveToPoint:atOffset:");
   SEL liftSelector = NSSelectorFromString(@"liftUpAtOffset:");
 
   missing = RunnerRequireSelector(
-    core.recordClass, initRecordSelector, @"initWithName:interfaceOrientation:", RunnerGestureSynthesisSurface
+    core.recordClass, initRecordSelector, @"initWithName:displayID:interfaceOrientation:", RunnerGestureSynthesisSurface
   );
   if (missing != nil) return missing;
   missing = RunnerRequireSelector(
@@ -311,6 +324,7 @@ static NSString * _Nullable RunnerResolveGestureEventBridge(
 
 static NSString * _Nullable RunnerCreateEventRecord(
   id application,
+  id _Nullable resolvedWindow,
   NSString *recordName,
   RunnerGestureEventBridge *bridge,
   id *record
@@ -318,6 +332,12 @@ static NSString * _Nullable RunnerCreateEventRecord(
   NSString *missing = RunnerResolveGestureEventBridge(application, bridge);
   if (missing != nil) return missing;
 
+  NSUInteger displayID = 0;
+  if (resolvedWindow == nil) {
+    return @"private XCTest event synthesis unavailable: no resolved application window";
+  }
+  NSString *displayError = RunnerResolveWindowDisplayID(resolvedWindow, &displayID);
+  if (displayError != nil) return displayError;
   NSInteger interfaceOrientation =
     ((RunnerMsgSendInteger)objc_msgSend)(application, bridge->interfaceOrientationSelector);
   NSInteger targetProcessID =
@@ -330,11 +350,18 @@ static NSString * _Nullable RunnerCreateEventRecord(
     [bridge->core.recordClass alloc],
     bridge->initRecordSelector,
     recordName,
+    displayID,
     interfaceOrientation
   );
   if (eventRecord == nil) {
     return @"private XCTest event synthesis failed: could not create event record";
   }
+  NSLog(
+    @"AGENT_DEVICE_RUNNER_SYNTHESIZED_RECORD name=%@ displayID=%lu interfaceOrientation=%ld",
+    recordName,
+    (unsigned long)displayID,
+    (long)interfaceOrientation
+  );
   ((RunnerMsgSendSetInteger)objc_msgSend)(
     eventRecord,
     bridge->core.setTargetProcessIDSelector,

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift

@@ -68,6 +68,14 @@ extension RunnerTests {
       guard waitUntilAlertButtonHittable(button, deadline: deadline) else {
         return alertVerificationResponse(.timedOut, action: action, activated: false)
       }
+      let buttonFrame = button.frame
+      NSLog(
+        "AGENT_DEVICE_RUNNER_ALERT_ACTIVATION action=%@ label=%@ frame=(%.1f,%.1f,%.1f,%.1f) point=(%.1f,%.1f)",
+        action,
+        button.label,
+        buttonFrame.origin.x, buttonFrame.origin.y, buttonFrame.size.width, buttonFrame.size.height,
+        buttonFrame.midX, buttonFrame.midY
+      )
 
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -24,6 +24,7 @@ import AgentDeviceSnapshotPresentation
 
 
 
+
 
 
 extension RunnerTests {
@@ -1465,7 +1466,7 @@ extension RunnerTests {
           fps: command.fps.map { Int32($0) }
         )
         try recorder.start { [weak self] in
-          return self?.captureRunnerFrame()
+          return self?.captureRunnerFrame(app: activeApp)
         }
         activeRecording = recorder
         return Response(ok: true, data: DataPayload(message: "recording started"))
@@ -2054,7 +2055,6 @@ extension RunnerTests {
     case .snapshot:
       return try executeSnapshotPrepared(command: command, activeApp: activeApp)
     case .screenshot:
-      let screenshot: XCUIScreenshot
 #if os(macOS)
 
       if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
@@ -2064,6 +2064,7 @@ extension RunnerTests {
 
         sleepFor(0.5)
       }
+      let screenshot: XCUIScreenshot
       if command.fullscreen == true {
         screenshot = XCUIScreen.main.screenshot()
       } else if let bundleId = command.appBundleId, !bundleId.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty {
@@ -2071,30 +2072,52 @@ extension RunnerTests {
       } else {
         screenshot = XCUIScreen.main.screenshot()
       }
-#else
-      screenshot = XCUIScreen.main.screenshot()
-#endif
-      guard let pngData = runnerPngData(for: screenshot.image) else {
-        return Response(ok: false, error: ErrorPayload(message: "Failed to encode screenshot as PNG"))
-      }
-      if command.inlineScreenshot == true {
+      return screenshotResponse(
+        image: screenshot.image,
+        inlineScreenshot: command.inlineScreenshot == true
+      )
+    #elseif os(iOS)
+
+
+
+
+
+
+
+
+
+      switch captureObservedScreen(app: resolveAppWithoutActivation(command: command)) {
+      case .failure(let failure):
         return Response(
-          ok: true,
-          data: DataPayload(imageBase64: pngData.base64EncodedString())
+          ok: false,
+          error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)
+        )
+      case .success(let captured):
+
+
+        NSLog(
+          "AGENT_DEVICE_RUNNER_SCREEN_CAPTURE display=%lu pixels=%ldx%ld pixelsPerPoint=%g",
+          captured.displayID,
+          captured.pixelWidth,
+          captured.pixelHeight,
+          captured.pixelsPerPoint
+        )
+        return screenshotResponse(
+          image: captured.image,
+          inlineScreenshot: command.inlineScreenshot == true,
+          metadata: ScreenshotMetadataPayload(
+            displayID: captured.displayID,
+            pixelWidth: captured.pixelWidth,
+            pixelHeight: captured.pixelHeight,
+            pixelsPerPoint: captured.pixelsPerPoint
+          )
         )
       }
-      let fileName = "screenshot-\(Int(Date().timeIntervalSince1970 * 1000)).png"
-      let filePath = (NSTemporaryDirectory() as NSString).appendingPathComponent(fileName)
-      do {
-        try pngData.write(to: URL(fileURLWithPath: filePath))
-      } catch {
-        return Response(ok: false, error: ErrorPayload(message: "Failed to write screenshot: \(error.localizedDescription)"))
-      }
-#if os(macOS)
-      return Response(ok: true, data: DataPayload(message: filePath))
 #else
-
-      return Response(ok: true, data: DataPayload(message: "tmp/\(fileName)"))
+      return screenshotResponse(
+        image: XCUIScreen.main.screenshot().image,
+        inlineScreenshot: command.inlineScreenshot == true
+      )
 #endif
     case .back, .backInApp:
       if tapInAppBackControl(app: activeApp) {