agent-device 0.19.3 → 0.20.1

package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift

@@ -146,8 +146,8 @@ final class RunnerCommandJournal {
     case .tap, .mouseClick, .longPress, .drag,
          .remotePress, .type, .swipe, .scroll, .desktopScroll, .findText, .querySelector, .readText, .back,
          .backInApp, .backSystem, .home, .rotate, .appSwitcher, .keyboardDismiss, .keyboardReturn,
-         .alert, .pinch, .sequence, .rotateGesture, .transformGesture, .recordStart, .recordStop,
-         .status, .uptime, .shutdown:
+         .alert, .sequence, .gesture, .gestureViewport, .recordStart, .recordStop,
+         .status, .uptime, .activate, .terminate, .targetReset, .shutdown:
       return true
     }
   }

package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -13,6 +13,16 @@ private enum RunnerInterfaceOrientation {
 }
 
 extension RunnerTests {
+  enum PlannedGestureExecution: Equatable {
+    case fastSwipe
+    case sampled
+  }
+
+  enum SynthesizedDragProfile {
+    case continuous
+    case fastSwipe
+  }
+
   struct TouchVisualizationFrame {
     let x: Double
     let y: Double
@@ -108,9 +118,9 @@ extension RunnerTests {
 #endif
   }
 
-  func rotateDevice(to orientationName: String) -> Bool {
+  func rotateDevice(to orientationName: String) -> String? {
 #if os(macOS) || os(tvOS) || os(visionOS)
-    return false
+    return nil
 #else
     switch orientationName {
     case "portrait":
@@ -122,10 +132,21 @@ extension RunnerTests {
     case "landscape-right":
       XCUIDevice.shared.orientation = .landscapeRight
     default:
-      return false
+      return nil
     }
     sleepFor(0.2)
-    return true
+    switch XCUIDevice.shared.orientation {
+    case .portrait:
+      return "portrait"
+    case .portraitUpsideDown:
+      return "portrait-upside-down"
+    case .landscapeLeft:
+      return "landscape-left"
+    case .landscapeRight:
+      return "landscape-right"
+    default:
+      return nil
+    }
 #endif
   }
 
@@ -139,7 +160,8 @@ extension RunnerTests {
     app: XCUIApplication,
     selectorKey: String,
     selectorValue: String,
-    allowNonHittableFallback: Bool = false
+    allowNonHittableFallback: Bool = false,
+    expectedPoint: CGPoint? = nil
   ) -> SelectorElementMatch {
     let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines)
     guard !value.isEmpty else {
@@ -163,6 +185,9 @@ extension RunnerTests {
     var nonHittableElement: XCUIElement?
     let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex
     for element in matches where element.exists {
+      if let expectedPoint, !element.frame.contains(expectedPoint) {
+        continue
+      }
       if !element.isHittable {
         if allowNonHittableFallback && hasTappableFrame(app: app, element: element) {
           guard nonHittableElement == nil else {
@@ -775,7 +800,7 @@ extension RunnerTests {
     x2: Double,
     y2: Double,
     durationMs: Double,
-    semantics: SynthesizedDragSemantics?,
+    profile: SynthesizedDragProfile = .continuous,
     context: SynthesizedCoordinateContext? = nil
   ) -> RunnerInteractionOutcome {
 #if os(iOS)
@@ -795,8 +820,9 @@ extension RunnerTests {
     let frame = context.referenceFrame
     let start = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: frame, interfaceOrientation: orientation)
     let end = nativeSynthesizedPoint(orientedX: x2, orientedY: y2, in: frame, interfaceOrientation: orientation)
-    let message = if semantics == .swipe {
-      RunnerSynthesizedGesture.synthesizeSwipe(
+    let message = switch profile {
+    case .continuous:
+      RunnerSynthesizedGesture.synthesizeContinuousDrag(
         withApplication: app,
         x: Double(start.x),
         y: Double(start.y),
@@ -804,8 +830,8 @@ extension RunnerTests {
         y2: Double(end.y),
         durationMs: durationMs
       )
-    } else {
-      RunnerSynthesizedGesture.synthesizeContinuousDrag(
+    case .fastSwipe:
+      RunnerSynthesizedGesture.synthesizeSwipe(
         withApplication: app,
         x: Double(start.x),
         y: Double(start.y),
@@ -1166,125 +1192,143 @@ extension RunnerTests {
     }
   }
 
-  func pinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
-#if os(iOS)
-    // A coordinate tap+drag is a single-finger gesture: React Native reads it as a pan
-    // and the pinch scale never changes (#629). Drive the two-finger XCTest synthesis
-    // path (the same one transformGesture uses) with zero translation/rotation so RN's
-    // pinch recognizer actually fires.
-    let frame = interactionRoot(app: app).frame
-    let centerX = x ?? Double(frame.midX)
-    let centerY = y ?? Double(frame.midY)
-    return transformGesture(
-      app: app,
-      x: centerX,
-      y: centerY,
-      dx: 0,
-      dy: 0,
-      scale: scale,
-      degrees: 0,
-      durationMs: 300
-    )
-#elseif os(tvOS)
-    return .unsupported(
-      message: "pinch is not supported on tvOS",
-      hint: "tvOS has no touch input; pinch requires a touchscreen (run on iOS)."
-    )
-#else
-    return .unsupported(
-      message: "pinch is not supported on macOS",
-      hint: "macOS automation has no multi-touch input; pinch requires a touchscreen (run on iOS)."
-    )
-#endif
+  func plannedGestureValidationError(_ plan: RunnerGesturePlan) -> String? {
+    guard plan.topology == "single" || plan.topology == "two" else {
+      return "planned gesture topology must be single or two"
+    }
+    let supportedIntent = plan.topology == "single"
+      ? plan.intent == "fling" || plan.intent == "pan"
+      : plan.intent == "pan" || plan.intent == "pinch" || plan.intent == "rotate"
+        || plan.intent == "transform"
+    guard supportedIntent else { return "planned gesture has unsupported intent for its topology" }
+    if plan.topology == "single" {
+      guard plan.executionProfile == "endpoint-hold" || plan.executionProfile == "timed-pan" else {
+        return "single-pointer gesture requires a supported execution profile"
+      }
+    } else if plan.executionProfile != nil {
+      return "multi-touch gesture cannot define a single-pointer execution profile"
+    }
+    guard plan.durationMs.isFinite, plan.durationMs >= 16, plan.durationMs <= 10_000 else {
+      return "planned gesture durationMs must be between 16 and 10000"
+    }
+    let viewport = plan.viewport
+    guard viewport.x.isFinite, viewport.y.isFinite, viewport.width.isFinite,
+      viewport.height.isFinite, viewport.width > 0, viewport.height > 0
+    else {
+      return "planned gesture viewport must be finite and positive"
+    }
+    let expectedPointerCount = plan.topology == "single" ? 1 : 2
+    guard plan.pointers.count == expectedPointerCount else {
+      return "planned gesture pointer count does not match topology"
+    }
+    for (index, pointer) in plan.pointers.enumerated() where pointer.pointerId != index {
+      return "planned gesture requires ordered pointer ids"
+    }
+    let firstSamples = plan.pointers[0].samples
+    guard firstSamples.count >= 2 else { return "planned pointer paths require at least two samples" }
+    for pointer in plan.pointers {
+      guard pointer.samples.count == firstSamples.count else {
+        return "planned pointer paths require matching samples"
+      }
+      var previousOffset = -1.0
+      for (index, sample) in pointer.samples.enumerated() {
+        guard sample.offsetMs.isFinite,
+          sample.offsetMs == firstSamples[index].offsetMs,
+          sample.offsetMs > previousOffset
+        else {
+          return "planned pointer sample offsets must match and strictly increase"
+        }
+        let point = sample.point
+        guard point.x.isFinite, point.y.isFinite,
+          point.x >= viewport.x,
+          point.x <= viewport.x + viewport.width,
+          point.y >= viewport.y,
+          point.y <= viewport.y + viewport.height
+        else {
+          return "planned pointer sample lies outside the viewport"
+        }
+        previousOffset = sample.offsetMs
+      }
+      guard pointer.samples.first?.offsetMs == 0,
+        pointer.samples.last?.offsetMs == plan.durationMs
+      else {
+        return "planned pointer paths must start at 0 and end at durationMs"
+      }
+    }
+    if plan.topology == "two" {
+      guard let firstStart = firstSamples.first?.point,
+        let secondStart = plan.pointers[1].samples.first?.point,
+        hypot(firstStart.x - secondStart.x, firstStart.y - secondStart.y) > 0
+      else {
+        return "planned pointer paths require a positive initial span"
+      }
+    }
+    return nil
   }
 
-  func rotateGesture(app: XCUIApplication, degrees: Double, x: Double?, y: Double?, velocity: Double) -> RunnerInteractionOutcome {
-#if os(iOS)
-    // Drive the two-finger XCTest synthesis path (the same one pinch/transformGesture use, #634)
-    // with zero translation/scale so React Native's rotation recognizer actually fires. The native
-    // XCUIElement.rotate(withVelocity:) injects a single synthetic rotation that RN's gesture
-    // handler does not read reliably — the same class of problem #629/#634 fixed for pinch.
-    // velocity is unused on iOS (synthesis speed is governed by durationMs); the wire contract
-    // keeps it for compatibility and direction is carried entirely by the sign of `degrees`.
-    let frame = interactionRoot(app: app).frame
-    let centerX = x ?? Double(frame.midX)
-    let centerY = y ?? Double(frame.midY)
-    return transformGesture(
-      app: app,
-      x: centerX,
-      y: centerY,
-      dx: 0,
-      dy: 0,
-      scale: 1,
-      degrees: degrees,
-      durationMs: 300
-    )
-#elseif os(tvOS)
-    return .unsupported(
-      message: "rotate-gesture is not supported on tvOS",
-      hint: "tvOS has no touch input; rotation gestures require a touchscreen (run on iOS)."
-    )
-#else
-    return .unsupported(
-      message: "rotate-gesture is not supported on macOS",
-      hint: "macOS automation has no multi-touch input; rotation gestures require a touchscreen (run on iOS)."
-    )
-#endif
+  func plannedGestureExecution(for plan: RunnerGesturePlan) -> PlannedGestureExecution {
+    plan.topology == "single" && plan.executionProfile == "endpoint-hold"
+      ? .fastSwipe
+      : .sampled
   }
 
-  func transformGesture(
+  func sampledPlannedGesture(
     app: XCUIApplication,
-    x: Double,
-    y: Double,
-    dx: Double,
-    dy: Double,
-    scale: Double,
-    degrees: Double,
-    durationMs: Double
+    plan: RunnerGesturePlan
   ) -> RunnerInteractionOutcome {
 #if os(iOS)
-    let target = interactionRoot(app: app)
     let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
-    let point = nativeSynthesizedPoint(orientedX: x, orientedY: y, in: app.frame, interfaceOrientation: orientation)
-    let vector = nativeSynthesizedVector(orientedDx: dx, orientedDy: dy, interfaceOrientation: orientation)
-    if let message = RunnerSynthesizedGesture.synthesizeTransform(
+    // The portable planner and validation use this exact viewport. Using app.frame here can
+    // diverge when XCTest unions transformed/off-screen descendants into the application frame.
+    let frame = CGRect(
+      x: plan.viewport.x,
+      y: plan.viewport.y,
+      width: plan.viewport.width,
+      height: plan.viewport.height
+    )
+    let pointerSamples: [[[String: NSNumber]]] = plan.pointers.map { pointer in
+      pointer.samples.map { sample in
+        let point = nativeSynthesizedPoint(
+          orientedX: sample.point.x,
+          orientedY: sample.point.y,
+          in: frame,
+          interfaceOrientation: orientation
+        )
+        return [
+          "x": NSNumber(value: Double(point.x)),
+          "y": NSNumber(value: Double(point.y)),
+          "offsetMs": NSNumber(value: sample.offsetMs),
+        ]
+      }
+    }
+    if let message = RunnerSynthesizedGesture.synthesizeGesture(
       withApplication: app,
-      x: Double(point.x),
-      y: Double(point.y),
-      dx: Double(vector.dx),
-      dy: Double(vector.dy),
-      scale: scale,
-      degrees: degrees,
-      radius: transformGestureRadius(frame: target.frame, scale: scale),
-      durationMs: durationMs
+      pointerSamples: pointerSamples
     ) {
       return .unsupported(
         message: message,
-        hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode (these APIs can change across Xcode versions)."
+        hint: "This gesture uses private XCTest event-synthesis APIs; rebuild the runner with a supported Xcode if this persists."
       )
     }
     return .performed
 #elseif os(tvOS)
     return .unsupported(
-      message: "transformGesture is not supported on tvOS",
-      hint: "tvOS has no touch input; transform gestures require a touchscreen (run on iOS)."
+      message: "two-finger gestures are not supported on tvOS",
+      hint: "tvOS has no touch input; use remote-driven navigation."
+    )
+#elseif os(visionOS)
+    return .unsupported(
+      message: "two-finger touch gestures are not supported on visionOS",
+      hint: "The current XCTest synthesizer supports iOS and iPadOS touch simulators only."
     )
 #else
     return .unsupported(
-      message: "transformGesture is not supported on macOS",
-      hint: "macOS automation has no multi-touch input; transform gestures require a touchscreen (run on iOS)."
+      message: "two-finger gestures are not supported on macOS",
+      hint: "macOS automation has no multi-touch input; run on an iOS simulator."
     )
 #endif
   }
 
-  private func transformGestureRadius(frame: CGRect, scale: Double) -> Double {
-    let shorterSide = Double(min(frame.width, frame.height))
-    let frameRadius = shorterSide * 0.20
-    let minimumEndRadius = shorterSide * 0.08
-    let scaleAdjustedRadius = scale < 1.0 ? max(frameRadius, minimumEndRadius / scale) : frameRadius
-    return min(max(scaleAdjustedRadius, 48.0), shorterSide * 0.35)
-  }
-
   private func interactionRoot(app: XCUIApplication) -> XCUIElement {
     let windows = app.windows.allElementsBoundByIndex
     if let window = windows.first(where: { $0.exists && !$0.frame.isEmpty }) {

package/dist/{apple-runner → apple/runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -85,6 +85,8 @@ extension RunnerTests {
     }
     currentApp = app
     currentBundleId = nil
+    currentAppProcessIdentifier = nil
+    snapshotXCTestPenaltyWarmupExemptionPending = false
   }
 
   func invalidateCachedTarget(reason: String) {
@@ -93,6 +95,48 @@ extension RunnerTests {
     }
     currentApp = nil
     currentBundleId = nil
+    currentAppProcessIdentifier = nil
+    snapshotXCTestPenaltyWarmupExemptionPending = false
+  }
+
+  func resetTargetAfterExternalRelaunch() -> Response {
+    invalidateCachedTarget(reason: "external_app_relaunch")
+    needsFirstInteractionDelay = true
+    return Response(ok: true, data: DataPayload(message: "target reset"))
+  }
+
+  func refreshCachedTargetIfProcessChanged(bundleId: String) {
+    guard currentBundleId == bundleId, currentApp != nil else { return }
+    let candidate = XCUIApplication(bundleIdentifier: bundleId)
+    let observedProcessIdentifier = Self.processIdentifier(of: candidate)
+    guard Self.shouldRefreshCachedTarget(
+      cachedProcessIdentifier: currentAppProcessIdentifier,
+      observedProcessIdentifier: observedProcessIdentifier
+    ) else { return }
+    NSLog(
+      "AGENT_DEVICE_RUNNER_TARGET_CACHE_REFRESH bundle=%@ previousPid=%d currentPid=%d",
+      bundleId,
+      currentAppProcessIdentifier ?? 0,
+      observedProcessIdentifier ?? 0
+    )
+    currentApp = candidate
+    currentAppProcessIdentifier = observedProcessIdentifier
+    clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
+    snapshotXCTestPenaltyWarmupExemptionPending = true
+    needsFirstInteractionDelay = true
+  }
+
+  static func processIdentifier(of target: XCUIApplication) -> Int? {
+    let value = RunnerAXSnapshotBridge.processIdentifier(for: target)
+    return value > 0 ? value : nil
+  }
+
+  static func shouldRefreshCachedTarget(
+    cachedProcessIdentifier: Int?,
+    observedProcessIdentifier: Int?
+  ) -> Bool {
+    guard let cachedProcessIdentifier, let observedProcessIdentifier else { return false }
+    return cachedProcessIdentifier != observedProcessIdentifier
   }
 
   func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
@@ -131,16 +175,26 @@ extension RunnerTests {
 
   func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
     let target = XCUIApplication(bundleIdentifier: bundleId)
+    let initialState = target.state
     NSLog(
       "AGENT_DEVICE_RUNNER_ACTIVATE bundle=%@ state=%d reason=%@",
       bundleId,
-      target.state.rawValue,
+      initialState.rawValue,
       reason
     )
     // activate avoids terminating and relaunching the target app
-    target.activate()
+    if initialState == .runningForeground {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_ACTIVATE_SKIPPED bundle=%@ reason=already_foreground",
+        bundleId
+      )
+    } else {
+      target.activate()
+    }
     currentApp = target
     currentBundleId = bundleId
+    currentAppProcessIdentifier = Self.processIdentifier(of: target)
+    snapshotXCTestPenaltyWarmupExemptionPending = false
     needsFirstInteractionDelay = true
     return target
   }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift

@@ -0,0 +1,73 @@
+import XCTest
+
+
+extension RunnerTests {
+
+  func testCachedTargetRefreshRequiresChangedPositiveProcessIdentity() {
+    XCTAssertFalse(
+      Self.shouldRefreshCachedTarget(
+        cachedProcessIdentifier: nil,
+        observedProcessIdentifier: 42
+      )
+    )
+    XCTAssertFalse(
+      Self.shouldRefreshCachedTarget(
+        cachedProcessIdentifier: 42,
+        observedProcessIdentifier: 42
+      )
+    )
+    XCTAssertTrue(
+      Self.shouldRefreshCachedTarget(
+        cachedProcessIdentifier: 41,
+        observedProcessIdentifier: 42
+      )
+    )
+  }
+
+  func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() {
+    snapshotXCTestPenaltyWarmupExemptionPending = true
+
+    XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption())
+    XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption())
+  }
+
+  func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() {
+    penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test")
+    XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
+
+    clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
+
+    XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
+  }
+
+  func testCachedTargetInvalidationClearsProcessBoundState() {
+    currentApp = app
+    currentBundleId = "com.example.app"
+    currentAppProcessIdentifier = 42
+    snapshotXCTestPenaltyWarmupExemptionPending = true
+
+    invalidateCachedTarget(reason: "unit_test")
+
+    XCTAssertNil(currentApp)
+    XCTAssertNil(currentBundleId)
+    XCTAssertNil(currentAppProcessIdentifier)
+    XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
+  }
+
+  func testTargetResetInvalidatesProcessBoundStateWithoutRestartingRunner() {
+    currentApp = app
+    currentBundleId = "com.example.app"
+    currentAppProcessIdentifier = 42
+    snapshotXCTestPenaltyWarmupExemptionPending = true
+    needsFirstInteractionDelay = false
+
+    let response = resetTargetAfterExternalRelaunch()
+
+    XCTAssertTrue(response.ok)
+    XCTAssertNil(currentApp)
+    XCTAssertNil(currentBundleId)
+    XCTAssertNil(currentAppProcessIdentifier)
+    XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
+    XCTAssertTrue(needsFirstInteractionDelay)
+  }
+}