appium-webdriveragent 16.14.1 → 16.15.0

package/WebDriverAgentLib/Utilities/FBW3CActionsSynthesizer.m

@@ -15,6 +15,7 @@
 #import "FBMacros.h"
 #import "FBMathUtils.h"
 #import "FBProtocolHelpers.h"
+#import "FBScreen.h"
 #import "FBW3CActionsHelpers.h"
 #import "FBXCodeCompatibility.h"
 #import "FBXCTestDaemonsProxy.h"
@@ -28,6 +29,7 @@
 #import "XCSynthesizedEventRecord.h"
 #import "XCPointerEventPath.h"
 #import "XCPointerEvent.h"
+#import "XCUIScreen.h"
 
 
 static NSString *const FB_KEY_TYPE = @"type";
@@ -66,7 +68,10 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
 #if !TARGET_OS_TV && !TARGET_OS_WATCH
 @interface FBW3CGestureItem : FBBaseGestureItem
 
-@property (nullable, readonly, nonatomic) FBBaseGestureItem *previousItem;
+@property (nullable, readonly, nonatomic) FBW3CGestureItem *previousItem;
+@property (nonatomic) CGVector displayCorrection;
+@property (nonatomic) BOOL usesDisplayCorrection;
+@property (readonly, nonatomic) CGPoint screenPoint;
 
 @end
 
@@ -117,7 +122,7 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
 
 - (nullable instancetype)initWithActionItem:(NSDictionary<NSString *, id> *)actionItem
                                 application:(XCUIApplication *)application
-                               previousItem:(nullable FBBaseGestureItem *)previousItem
+                               previousItem:(nullable FBW3CGestureItem *)previousItem
                                      offset:(double)offset
                                       error:(NSError **)error
 {
@@ -127,6 +132,7 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
     self.application = application;
     self.offset = offset;
     _previousItem = previousItem;
+    _usesDisplayCorrection = previousItem.usesDisplayCorrection;
     NSNumber *durationObj = FBOptDuration(actionItem, @0, error);
     if (nil == durationObj) {
       return nil;
@@ -142,6 +148,14 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
   return self;
 }
 
+- (CGPoint)screenPoint
+{
+  CGPoint point = self.atPosition.screenPoint;
+  return self.usesDisplayCorrection
+    ? CGPointMake(point.x + self.displayCorrection.dx, point.y + self.displayCorrection.dy)
+    : point;
+}
+
 - (nullable XCUICoordinate *)positionWithError:(NSError **)error
 {
   if (nil == self.previousItem || nil == self.previousItem.atPosition) {
@@ -158,6 +172,27 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
                                   positionOffset:(nullable NSValue *)positionOffset
                                            error:(NSError **)error
 {
+  XCUIScreen *screen = nil;
+  if (element.elementType == XCUIElementTypeApplication) {
+    screen = [FBScreen currentScreenWithError:error];
+    if (nil == screen) {
+      return nil;
+    }
+  }
+  if (nil != screen && screen.displayID != XCUIScreen.mainScreen.displayID) {
+    // Unlike ordinary elements, application-root coordinates rotate using the
+    // main display. Resolve the client-visible rect's center into viewport
+    // coordinates, then let screenPoint apply the selected-display correction.
+    // Do not use a normalized app center: XCTest may report a portrait app rect
+    // for a landscape window, so its normalized center uses a different space.
+    CGRect frame = element.wdFrame;
+    if (CGRectIsEmpty(frame)) {
+      return [super hitpointWithElement:element positionOffset:positionOffset error:error];
+    }
+    CGPoint offset = nil == positionOffset ? CGPointZero : positionOffset.CGPointValue;
+    CGPoint point = CGPointMake(CGRectGetMidX(frame) + offset.x, CGRectGetMidY(frame) + offset.y);
+    return [super hitpointWithElement:nil positionOffset:[NSValue valueWithCGPoint:point] error:error];
+  }
   if (nil == element || nil == positionOffset) {
     return [super hitpointWithElement:element positionOffset:positionOffset error:error];
   }
@@ -223,7 +258,7 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
     }
   }
   if (nil == self.pressure) {
-    XCPointerEventPath *result = [[XCPointerEventPath alloc] initForTouchAtPoint:self.atPosition.screenPoint
+    XCPointerEventPath *result = [[XCPointerEventPath alloc] initForTouchAtPoint:self.screenPoint
                                                                           offset:FBMillisToSeconds(self.offset)];
     return @[result];
   }
@@ -280,6 +315,10 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
   }
   
   if (nil != element) {
+    // Application origins need viewport display correction; other elements use their own display.
+    // This correction does not fix XCTest's legacy-SDK compatibility issue:
+    // see docs/multiple-displays.md#legacy-sdk-compatibility-windows.
+    self.usesDisplayCorrection = element.elementType == XCUIElementTypeApplication;
     if (nil == x && nil == y) {
       return [self hitpointWithElement:element positionOffset:nil error:error];
     }
@@ -287,6 +326,7 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
   }
   
   if ([origin isKindOfClass:NSString.class] && [origin isEqualToString:FB_ORIGIN_TYPE_VIEWPORT]) {
+    self.usesDisplayCorrection = YES;
     return [self hitpointWithElement:nil positionOffset:[NSValue valueWithCGPoint:CGPointMake(x.floatValue, y.floatValue)] error:error];
   }
   
@@ -300,6 +340,7 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
   }
   XCUICoordinate *recentPosition = self.previousItem.atPosition;
   CGVector offsetRelativeToRecentPosition = (nil == x && nil == y) ? CGVectorMake(0, 0) : CGVectorMake(x.floatValue, y.floatValue);
+  // Pointer-relative moves keep the coordinate space of their preceding position.
   return [recentPosition coordinateWithOffset:offsetRelativeToRecentPosition];
 }
 
@@ -314,10 +355,10 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
                                             error:(NSError **)error
 {
   if (nil == eventPath) {
-    return @[[[XCPointerEventPath alloc] initForTouchAtPoint:self.atPosition.screenPoint
+    return @[[[XCPointerEventPath alloc] initForTouchAtPoint:self.screenPoint
                                                       offset:FBMillisToSeconds(self.offset + self.duration)]];
   }
-  [eventPath moveToPoint:self.atPosition.screenPoint
+  [eventPath moveToPoint:self.screenPoint
                 atOffset:FBMillisToSeconds(self.offset + self.duration)];
   return @[];
 }
@@ -637,6 +678,10 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
 @end
 
 
+@interface FBW3CActionsSynthesizer ()
+@property (nonatomic) CGVector displayCorrection;
+@end
+
 @implementation FBW3CActionsSynthesizer
 
 - (NSArray<NSDictionary<NSString *, id> *> *)preprocessedActionItemsWith:(NSArray<NSDictionary<NSString *, id> *> *)actionItems
@@ -821,6 +866,7 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
       return nil;
     }
 
+    gestureItem.displayCorrection = self.displayCorrection;
     [chain addItem:gestureItem];
   }
 
@@ -847,13 +893,10 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
   return [self eventPathsWithKeyAction:actionDescription forActionId:actionId error:error];
 }
 
-- (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error
+- (nullable NSArray<NSDictionary<NSString *, id> *> *)validatedActionsWithError:(NSError **)error
 {
-  XCSynthesizedEventRecord *eventRecord = [[XCSynthesizedEventRecord alloc]
-                                           initWithName:@"W3C Touch Action"
-                                           interfaceOrientation:self.application.interfaceOrientation];
   NSMutableDictionary<NSString *, NSDictionary<NSString *, id> *> *actionsMapping = [NSMutableDictionary new];
-  NSMutableArray<NSString *> *actionIds = [NSMutableArray new];
+  NSMutableArray<NSDictionary<NSString *, id> *> *validatedActions = [NSMutableArray new];
   for (NSDictionary<NSString *, id> *action in self.actions) {
     id actionId = [action objectForKey:FB_KEY_ID];
     if (![actionId isKindOfClass:NSString.class] || 0 == [actionId length]) {
@@ -883,11 +926,76 @@ static NSString *const FB_KEY_ACTIONS = @"actions";
       continue;
     }
 
-    [actionIds addObject:actionId];
+    [validatedActions addObject:action];
     [actionsMapping setObject:action forKey:actionId];
   }
-  for (NSString *actionId in actionIds.copy) {
-    NSDictionary<NSString *, id> *actionDescription = [actionsMapping objectForKey:actionId];
+  return validatedActions.copy;
+}
+
+- (BOOL)actionsRequireDisplay:(NSArray<NSDictionary<NSString *, id> *> *)actions
+{
+  for (NSDictionary<NSString *, id> *action in actions) {
+    if (![action[FB_KEY_TYPE] isEqual:FB_ACTION_TYPE_POINTER]) {
+      continue;
+    }
+    for (NSDictionary *item in action[FB_KEY_ACTIONS]) {
+      if (![item[FB_ACTION_ITEM_KEY_TYPE] isEqual:FB_ACTION_ITEM_TYPE_PAUSE]) {
+        return YES;
+      }
+    }
+  }
+  return NO;
+}
+
+- (nullable XCSynthesizedEventRecord *)eventRecordForActions:(NSArray<NSDictionary<NSString *, id> *> *)actions
+                                                     error:(NSError **)error
+{
+  // Keyboard and pause-only sequences have no display coordinates. A display
+  // disappearing must not prevent these input sources from being synthesized.
+  XCUIScreen *screen = [self actionsRequireDisplay:actions] ? [FBScreen currentScreenWithError:error] : XCUIScreen.mainScreen;
+  if (nil == screen) {
+    return nil;
+  }
+  XCSynthesizedEventRecord *eventRecord;
+  self.displayCorrection = CGVectorMake(0, 0);
+  if (screen.displayID == XCUIScreen.mainScreen.displayID) {
+    eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action"
+                                            interfaceOrientation:self.application.interfaceOrientation];
+  } else {
+    if (![XCSynthesizedEventRecord instancesRespondToSelector:@selector(initWithName:displayID:interfaceOrientation:)]) {
+      if (error) {
+        *error = [[FBErrorBuilder.builder
+                   withDescription:@"Actions on a display other than the main one are not supported by this XCTest version"] build];
+      }
+      return nil;
+    }
+    eventRecord = [[XCSynthesizedEventRecord alloc] initWithName:@"W3C Touch Action"
+                                                       displayID:(unsigned long long)screen.displayID
+                                            interfaceOrientation:self.application.interfaceOrientation];
+    // Viewport coordinates use the main display's size when rotating screenPoint.
+    // Element coordinates already use their own display. Apply this correction
+    // per gesture item so mixed origins and pointer-relative moves stay aligned.
+    XCUIScreen *mainScreen = XCUIScreen.mainScreen;
+    self.displayCorrection = FBDisplayCoordinateOffset(
+      CGSizeMake(mainScreen.bounds.size.width / mainScreen.scale, mainScreen.bounds.size.height / mainScreen.scale),
+      CGSizeMake(screen.bounds.size.width / screen.scale, screen.bounds.size.height / screen.scale),
+      eventRecord.interfaceOrientation);
+  }
+  return eventRecord;
+}
+
+- (nullable XCSynthesizedEventRecord *)synthesizeWithError:(NSError **)error
+{
+  NSArray<NSDictionary<NSString *, id> *> *actions = [self validatedActionsWithError:error];
+  if (nil == actions) {
+    return nil;
+  }
+  XCSynthesizedEventRecord *eventRecord = [self eventRecordForActions:actions error:error];
+  if (nil == eventRecord) {
+    return nil;
+  }
+  for (NSDictionary<NSString *, id> *actionDescription in actions) {
+    NSString *actionId = actionDescription[FB_KEY_ID];
     NSArray<XCPointerEventPath *> *eventPaths = [self eventPathsWithActionDescription:actionDescription forActionId:actionId error:error];
     if (nil == eventPaths) {
       return nil;

package/WebDriverAgentTests/IntegrationApp/COORDINATE_PROBE.md

@@ -21,7 +21,9 @@ shows 80 numbered rows, each 44 points high.
 
 The `probe-status` label exposes `start`, `last`, `phase`, and `count` after a
 touch. `start` and `last` are **canvas-local app points**. It also exposes
-`canvasBounds`, `canvasWindowRect`, `windowSize`, `screenSize`, and `scrollY`.
+`canvasBounds`, `canvasWindowRect`, `windowSize`, `screenSize`, `scrollY`, and
+`fullyVisibleRows` (row indices whose full rectangles lie inside the table bounds).
+`rotationInProgress` stays true until the fixture's rotation transition completes.
 Geometry is refreshed when the view lays out, including after rotation or window
 resizing. Touch-related fields are absent before the first touch.
 

package/WebDriverAgentTests/IntegrationApp/Classes/FBCoordinateProbeViewController.m

@@ -106,6 +106,17 @@
   [self.view addSubview:self.table];
 }
 
+- (void)viewWillTransitionToSize:(CGSize)size withTransitionCoordinator:(id<UIViewControllerTransitionCoordinator>)coordinator
+{
+  [super viewWillTransitionToSize:size withTransitionCoordinator:coordinator];
+  self.measurement[@"rotationInProgress"] = @YES;
+  [self publishMeasurement];
+  [coordinator animateAlongsideTransition:nil completion:^(id<UIViewControllerTransitionCoordinatorContext> context) {
+    self.measurement[@"rotationInProgress"] = @NO;
+    [self publishMeasurement];
+  }];
+}
+
 - (void)viewDidLayoutSubviews
 {
   [super viewDidLayoutSubviews];
@@ -139,6 +150,13 @@
   self.measurement[@"windowSize"] = @[@(window.bounds.size.width), @(window.bounds.size.height)];
   self.measurement[@"screenSize"] = @[@(window.screen.bounds.size.width), @(window.screen.bounds.size.height)];
   self.measurement[@"scrollY"] = @(self.table.contentOffset.y);
+  NSMutableArray<NSNumber *> *fullyVisibleRows = [NSMutableArray new];
+  for (NSIndexPath *indexPath in self.table.indexPathsForVisibleRows) {
+    if (CGRectContainsRect(self.table.bounds, [self.table rectForRowAtIndexPath:indexPath])) {
+      [fullyVisibleRows addObject:@(indexPath.row)];
+    }
+  }
+  self.measurement[@"fullyVisibleRows"] = fullyVisibleRows;
   NSData *data = [NSJSONSerialization dataWithJSONObject:self.measurement
                                                options:NSJSONWritingSortedKeys
                                                  error:nil];