agent-device 0.20.10 → 0.21.0

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m

@@ -29,6 +29,7 @@ typedef id (*RunnerDragPointerPathFactory)(
   CGPoint,
   double
 );
+typedef double (*RunnerDragProgressFunction)(double);
 
 static NSString * _Nullable RunnerResolveGestureEventBridge(
   id application,
@@ -56,6 +57,21 @@ static id RunnerContinuousDragPointerPath(
   CGPoint end,
   double durationMs
 );
+static id RunnerControlledScrollPointerPath(
+  const RunnerGestureEventBridge *bridge,
+  CGPoint start,
+  CGPoint end,
+  double durationMs
+);
+static id RunnerSampledDragPointerPath(
+  const RunnerGestureEventBridge *bridge,
+  CGPoint start,
+  CGPoint end,
+  double durationMs,
+  RunnerDragProgressFunction progress,
+  NSInteger frameCount
+);
+static double RunnerSmoothstepProgress(double t);
 static NSString * _Nullable RunnerTrySynthesizeDrag(
   id application,
   CGPoint start,
@@ -68,6 +84,8 @@ static NSString * _Nullable RunnerTrySynthesizeTap(id application, CGPoint point
 // XCTest's proven swipe profile reaches the endpoint in 100 ms, then holds for the planned
 // fling duration. Fast movement is what lets UIKit distinguish a fling from a timed pan.
 static const NSTimeInterval RunnerSwipeMovementDurationSeconds = 0.1;
+static const double RunnerDragSampleIntervalMs = 16.0;
+static const NSInteger RunnerControlledScrollMaxFrameCount = 30;
 static id RunnerTapPointerPath(
   const RunnerGestureEventBridge *bridge,
   CGPoint point
@@ -115,6 +133,26 @@ static id RunnerTapPointerPath(
   }
 }
 
++ (NSString * _Nullable)synthesizeControlledScrollWithApplication:(id)application
+                                                                x:(double)x
+                                                                y:(double)y
+                                                               x2:(double)x2
+                                                                y2:(double)y2
+                                                        durationMs:(double)durationMs {
+  @try {
+    return RunnerTrySynthesizeDrag(
+      application,
+      CGPointMake(x, y),
+      CGPointMake(x2, y2),
+      durationMs,
+      @"agent-device-controlled-scroll",
+      RunnerControlledScrollPointerPath
+    );
+  } @catch (NSException *exception) {
+    return RunnerFormatXCTestException(exception, @"private XCTest event synthesis failed");
+  }
+}
+
 + (NSString * _Nullable)synthesizeTapWithApplication:(id)application
                                                    x:(double)x
                                                    y:(double)y {
@@ -351,6 +389,46 @@ static id RunnerContinuousDragPointerPath(
   CGPoint start,
   CGPoint end,
   double durationMs
+) {
+  // This is velocity shaping, not just interpolation density: smoothstep's endpoint slope is zero,
+  // while a planned linear segment reaches lift with nonzero velocity unless a destination hold
+  // follows it. UIKit uses finger-up velocity for scroll deceleration. See ADR 0013 and issue #1586.
+  return RunnerSampledDragPointerPath(
+    bridge,
+    start,
+    end,
+    durationMs,
+    RunnerSmoothstepProgress,
+    RunnerContinuousDragFrameCount(durationMs)
+  );
+}
+
+static id RunnerControlledScrollPointerPath(
+  const RunnerGestureEventBridge *bridge,
+  CGPoint start,
+  CGPoint end,
+  double durationMs
+) {
+  // A stationary hold is not a reliable zero-velocity release in XCTest. Cubic ease-out decreases
+  // velocity from the first movement and reaches near zero before lift. Profiles that accelerate
+  // after pan recognition can still be classified as flings despite a slow final sample.
+  return RunnerSampledDragPointerPath(
+    bridge,
+    start,
+    end,
+    durationMs,
+    RunnerControlledScrollProgress,
+    RunnerControlledScrollFrameCount(durationMs)
+  );
+}
+
+static id RunnerSampledDragPointerPath(
+  const RunnerGestureEventBridge *bridge,
+  CGPoint start,
+  CGPoint end,
+  double durationMs,
+  RunnerDragProgressFunction progress,
+  NSInteger frameCount
 ) {
   id path =
     ((RunnerMsgSendInitPath)objc_msgSend)([bridge->core.pathClass alloc], bridge->initPathSelector, start, 0.0);
@@ -358,14 +436,10 @@ static id RunnerContinuousDragPointerPath(
     return nil;
   }
 
-  // This is velocity shaping, not just interpolation density: smoothstep's endpoint slope is zero,
-  // while a planned linear segment reaches lift with nonzero velocity unless a destination hold
-  // follows it. UIKit uses finger-up velocity for scroll deceleration. See ADR 0013 and issue #1586.
-  int frameCount = MAX(3, (int)(durationMs / 16.0));
   NSTimeInterval durationSeconds = durationMs / 1000.0;
-  for (int index = 1; index <= frameCount; index += 1) {
+  for (NSInteger index = 1; index <= frameCount; index += 1) {
     double t = (double)index / (double)frameCount;
-    double easedT = t * t * (3.0 - 2.0 * t);
+    double easedT = progress(t);
     CGPoint point = CGPointMake(
       start.x + (end.x - start.x) * easedT,
       start.y + (end.y - start.y) * easedT
@@ -377,6 +451,26 @@ static id RunnerContinuousDragPointerPath(
   return path;
 }
 
+static double RunnerSmoothstepProgress(double t) {
+  return t * t * (3.0 - 2.0 * t);
+}
+
+NSInteger RunnerContinuousDragFrameCount(double durationMs) {
+  return MAX(3, (NSInteger)(durationMs / RunnerDragSampleIntervalMs));
+}
+
+NSInteger RunnerControlledScrollFrameCount(double durationMs) {
+  return MIN(
+    RunnerControlledScrollMaxFrameCount,
+    MAX(3, (NSInteger)ceil(durationMs / RunnerDragSampleIntervalMs) - 1)
+  );
+}
+
+double RunnerControlledScrollProgress(double t) {
+  double remaining = 1.0 - t;
+  return 1.0 - remaining * remaining * remaining;
+}
+
 static id RunnerTapPointerPath(
   const RunnerGestureEventBridge *bridge,
   CGPoint point

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift

@@ -11,8 +11,8 @@ import XCTest
 // This is pure geometry on purpose — no XCUIElement — so the exact decision
 // can be proven against the golden fixture table shared with the TS twin:
 //   table:   contracts/fixtures/tap-point-policy.json
-//   TS twin: src/snapshot/mobile-snapshot-semantics.ts#isTapPointInsideViewport
-//   TS test: src/snapshot/__tests__/tap-point-policy-parity.test.ts
+//   TS twin: packages/contracts/src/snapshot-visibility.ts#isTapPointInsideViewport
+//   TS test: packages/contracts/src/snapshot-tap-point-policy.test.ts
 // Drift on either side turns CI red without needing a simulator.
 enum TapPointPolicy {
   static func isAllowed(elementFrame: CGRect, windowFrame: CGRect) -> Bool {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift

@@ -1,4 +1,5 @@
 import XCTest
+import AgentDeviceSnapshotPresentation
 
 extension RunnerTests {
   private static let privateAXSnapshotMaxNodes = 5_000
@@ -105,14 +106,14 @@ extension RunnerTests {
 
   func privateAXSnapshotAcquisition(
     app: XCUIApplication,
-    options: PresentationOptions,
+    hint: CaptureHint,
     deadline: Date = .distantFuture
   ) -> SnapshotAcquisition? {
     #if os(iOS) && targetEnvironment(simulator)
-      let requestedDepth = options.depth ?? 64
+      let requestedDepth = hint.rawTraversalDepth ?? 64
       // An explicit --depth request is honored as asked: no accepted-depth
       // memory, no frontier extension past it.
-      let exactDepthRequested = options.depth != nil
+      let exactDepthRequested = hint.rawTraversalDepth != nil
       let rememberedDepth =
         exactDepthRequested
         ? nil
@@ -135,12 +136,16 @@ extension RunnerTests {
           NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_BUDGET_EXHAUSTED depth=%ld", depth)
           break
         }
+        // Declared residue (#1797): the bridge caps the tree at 5000 nodes while serializing,
+        // BEFORE either projection exists, so a raw capture of a huge screen is bounded rather
+        // than failing the way the tree backend's own raw cap does. The cap is disclosed as
+        // `truncated`, and it applied to the acquired tree before this projection split too.
         response = RunnerAXSnapshotBridge.snapshotTree(
           for: app,
           maxDepth: depth,
           maxNodes: Self.privateAXSnapshotMaxNodes,
           deepExtensionCallLimit: exactDepthRequested ? 0 : Self.privateAXDeepExtensionCallLimit,
-          customActionLimit: options.customActions ? Self.privateAXCustomActionLimit : 0,
+          customActionLimit: hint.customActions ? Self.privateAXCustomActionLimit : 0,
           deadline: deadline
         )
         if response["ok"] as? Bool == true {
@@ -175,11 +180,13 @@ extension RunnerTests {
 
       let rootFrame = privateAXRect(root["frame"])
       let viewport = privateAXSnapshotViewport(app: app, rootFrame: rootFrame)
-      let nodes = privateAXPresentation(
+      let nodes = privateAXAcquisition(
         rawRoot: root,
-        options: options,
+        hint: hint,
         viewport: viewport
       )
+      // Serialization-level emptiness only: an acquired-but-fully-clipped tree is presentation's
+      // verdict now, surfaced by the plan's sparse classifier on the presented payload (#1797).
       if nodes.count <= 1 {
         NSLog("AGENT_DEVICE_RUNNER_PRIVATE_AX_SNAPSHOT_SPARSE=%ld", nodes.count)
         return nil
@@ -203,12 +210,14 @@ extension RunnerTests {
         deepExtension?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int ?? 0
       )
       return SnapshotAcquisition(
+        hint: hint,
         nodes: nodes,
         truncated: (response["truncated"] as? Bool) == true,
         effectiveDepth: depthLimited ? effectiveDepth : nil,
         customActions: Self.privateAXCustomActionCoverage(
           response[RunnerAXSnapshotCustomActionsKey]
-        )
+        ),
+        viewport: viewport
       )
     #else
       return nil
@@ -230,7 +239,6 @@ extension RunnerTests {
     }
     do {
       let viewport = try runMainThreadWork(
-        command: nil,
         timeout: 1,
         timeoutError: snapshotMainThreadTimeoutError("reading private AX viewport")
       ) {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift

@@ -50,50 +50,34 @@ extension RunnerTests {
       guard let button = chooseAlertButton(alert.buttons, action: action) else {
         return Response(ok: false, error: ErrorPayload(message: "alert \(action) button not found"))
       }
-      let previousTitle = preferredAlertTitle(alert.root, buttons: alert.buttons)
-      let actionButtonLabel = button.label.trimmingCharacters(in: .whitespacesAndNewlines)
-      let actionButtonFrame = button.frame
+      guard Date() < deadline else {
+        return alertVerificationResponse(.timedOut, action: action, activated: false)
+      }
+      guard let original = captureAlertPresentation(alert.root) else {
+        return alertVerificationResponse(.unconfirmed, action: action, activated: false)
+      }
+      let observesApplicationRoot = alert.root.elementType == .application
+      guard Date() < deadline else {
+        return alertVerificationResponse(.timedOut, action: action, activated: false)
+      }
       let outcome = activateElement(app: alert.ownerApp, element: button, action: "alert \(action)")
       if let response = unsupportedResponse(for: outcome) {
         return response
       }
-      sleepFor(0.2)
-      if alertStillVisible(
-        in: alert.ownerApp,
-        source: alert.source,
-        previousTitle: previousTitle,
-        actionButtonLabel: actionButtonLabel,
-        deadline: deadline
-      ) {
-        if !actionButtonFrame.isNull && !actionButtonFrame.isEmpty {
-          let coordinateOutcome = tapAt(
-            app: alert.ownerApp,
-            x: actionButtonFrame.midX,
-            y: actionButtonFrame.midY
-          )
-          if let response = unsupportedResponse(for: coordinateOutcome) {
-            return response
-          }
-          sleepFor(0.2)
-        }
-      }
-      if alertStillVisible(
-        in: alert.ownerApp,
-        source: alert.source,
-        previousTitle: previousTitle,
-        actionButtonLabel: actionButtonLabel,
-        deadline: deadline
-      ) {
-        return Response(
-          ok: false,
-          error: ErrorPayload(
-            code: "INTERACTION_FAILED",
-            message: "alert \(action) did not dismiss the visible alert",
-            hint: "The alert button was found but the system still reports the alert after tapping it."
+      while true {
+        sleepFor(min(0.2, max(0, deadline.timeIntervalSinceNow)))
+        let verification = RunnerAlertVerification.verify(
+          original: original,
+          observation: observeAlert(
+            in: alert.ownerApp,
+            source: alert.source,
+            observesApplicationRoot: observesApplicationRoot,
+            deadline: deadline
           )
         )
+        if verification == .stillVisible { continue }
+        return alertVerificationResponse(verification, action: action, activated: true)
       }
-      return Response(ok: true, data: DataPayload(message: action == "accept" ? "accepted" : "dismissed"))
     }
 
     return Response(
@@ -130,57 +114,6 @@ extension RunnerTests {
     return RunnerAlert(root: root, ownerApp: ownerApp, buttons: buttons, source: source)
   }
 
-  private func alertStillVisible(
-    in ownerApp: XCUIApplication,
-    source: RunnerAlertSource,
-    previousTitle: String,
-    actionButtonLabel: String,
-    deadline: Date
-  ) -> Bool {
-    guard Date() < deadline,
-          let current = resolveAlert(source: source, app: ownerApp, deadline: deadline)
-    else {
-      return false
-    }
-    let currentTitle = preferredAlertTitle(current.root, buttons: current.buttons)
-    if previousTitle == currentTitle {
-      return true
-    }
-    return current.buttons.contains { button in
-      button.label.trimmingCharacters(in: .whitespacesAndNewlines) == actionButtonLabel
-    }
-  }
-
-  private func resolveAlert(
-    source: RunnerAlertSource,
-    app: XCUIApplication,
-    deadline: Date
-  ) -> RunnerAlert? {
-    switch source {
-    case .blockingSystemModal:
-#if os(macOS)
-      return nil
-#else
-      guard case .resolved(let modal) = resolveBlockingSystemModal(deadline: deadline) else {
-        return nil
-      }
-      return runnerAlert(modal)
-#endif
-    case .appAlert:
-      guard let alert = firstExistingElement(
-        in: safeElementsQuery { app.alerts.allElementsBoundByIndex }
-      ) else {
-        return nil
-      }
-      return runnerAlert(root: alert, ownerApp: app, source: .appAlert)
-    case .dismissPopup:
-      guard let popup = firstDismissPopupWindow(in: app) else {
-        return nil
-      }
-      return runnerAlert(root: popup, ownerApp: app, source: .dismissPopup)
-    }
-  }
-
   private func firstExistingElement(in elements: [XCUIElement]) -> XCUIElement? {
     elements.first { isVisibleElement($0) }
   }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift

@@ -0,0 +1,124 @@
+import XCTest
+
+extension RunnerTests {
+  func captureAlertPresentation(_ element: XCUIElement) -> RunnerAlertPresentation? {
+    var presentation: RunnerAlertPresentation?
+    _ = RunnerObjCExceptionCatcher.catchException {
+      guard let snapshot = try? element.snapshot() else { return }
+      presentation = self.alertPresentation(snapshot)
+    }
+    return presentation
+  }
+
+  func observeAlert(
+    in ownerApp: XCUIApplication,
+    source: RunnerAlertSource,
+    observesApplicationRoot: Bool,
+    deadline: Date
+  ) -> RunnerAlertObservation {
+    guard Date() < deadline else { return .deadlineExceeded }
+    var observation = RunnerAlertObservation.unavailable
+    _ = RunnerObjCExceptionCatcher.catchException {
+      switch ownerApp.state {
+      case .notRunning:
+        observation = .absent
+        return
+      case .runningForeground, .runningBackground:
+        break
+#if !os(macOS)
+      case .runningBackgroundSuspended:
+        break
+#endif
+      case .unknown:
+        return
+      @unknown default:
+        return
+      }
+      guard Date() < deadline, let snapshot = try? ownerApp.snapshot(),
+            !snapshot.children.isEmpty, !snapshot.frame.isNull, !snapshot.frame.isEmpty else { return }
+      if observesApplicationRoot {
+        observation = .visible(self.alertPresentation(snapshot))
+        return
+      }
+      let candidates = self.alertSnapshots(in: snapshot, source: source, viewport: snapshot.frame)
+      guard candidates.count <= 1 else { return }
+      observation = candidates.first.map { .visible(self.alertPresentation($0)) } ?? .absent
+    }
+    return Date() < deadline ? observation : .deadlineExceeded
+  }
+
+  private func alertSnapshots(
+    in snapshot: XCUIElementSnapshot,
+    source: RunnerAlertSource,
+    viewport: CGRect
+  ) -> [XCUIElementSnapshot] {
+    let frame = snapshot.frame
+    let visible = !frame.isNull && !frame.isEmpty && viewport.contains(CGPoint(x: frame.midX, y: frame.midY))
+    let matches: Bool
+    switch source {
+    case .blockingSystemModal:
+      matches = snapshot.elementType == .alert || snapshot.elementType == .sheet
+    case .appAlert:
+      matches = snapshot.elementType == .alert
+    case .dismissPopup:
+      matches = snapshot.elementType == .window && containsDismissPopupMarker(snapshot)
+    }
+    if matches && visible { return [snapshot] }
+    return snapshot.children.flatMap { alertSnapshots(in: $0, source: source, viewport: viewport) }
+  }
+
+  private func containsDismissPopupMarker(_ snapshot: XCUIElementSnapshot) -> Bool {
+    [snapshot.label, snapshot.identifier].contains {
+      $0.trimmingCharacters(in: .whitespacesAndNewlines).caseInsensitiveCompare("dismiss popup") == .orderedSame
+    } || snapshot.children.contains { containsDismissPopupMarker($0) }
+  }
+
+  private func alertPresentation(_ snapshot: XCUIElementSnapshot) -> RunnerAlertPresentation {
+    var content: [String] = []
+    var buttons: [String] = []
+    func collect(_ node: XCUIElementSnapshot) {
+      let label = node.label.trimmingCharacters(in: .whitespacesAndNewlines)
+      if actionableTypes.contains(node.elementType) {
+        buttons.append(label)
+      } else if !label.isEmpty {
+        content.append(label)
+      }
+      node.children.forEach(collect)
+    }
+    snapshot.children.forEach(collect)
+    return RunnerAlertPresentation(
+      title: snapshot.label.trimmingCharacters(in: .whitespacesAndNewlines),
+      content: content,
+      buttons: buttons
+    )
+  }
+
+  func alertVerificationResponse(
+    _ verification: RunnerAlertVerification,
+    action: String,
+    activated: Bool
+  ) -> Response {
+    let code: String
+    let message: String
+    switch verification {
+    case .disappeared, .presentationChanged:
+      return Response(ok: true, data: DataPayload(message: action == "accept" ? "accepted" : "dismissed"))
+    case .timedOut:
+      code = "ALERT_DEADLINE_EXCEEDED"
+      message = "alert \(action) exhausted its deadline"
+    case .stillVisible:
+      code = "INTERACTION_FAILED"
+      message = "alert \(action) still observes an unchanged alert presentation"
+    case .unconfirmed:
+      code = "ALERT_CONFIRMATION_UNAVAILABLE"
+      message = "alert \(action) could not read the alert presentation"
+    }
+    return Response(ok: false, error: ErrorPayload(
+      code: code,
+      message: message,
+      hint: activated
+        ? "The button was activated once. Inspect the current alert before deciding on another action."
+        : "No alert button was activated. Inspect the current alert before deciding on an action."
+    ))
+  }
+}