agent-device 0.18.3 → 0.19.1

package/dist/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift

@@ -0,0 +1,172 @@
+import XCTest
+
+extension RunnerTests {
+  static let navigationBackKeywords = ["back", "close", "cancel"]
+  static let navigationFallbackVerificationDelay: TimeInterval = 0.25
+
+  func tapInAppBackControl(app: XCUIApplication) -> Bool {
+#if os(macOS)
+    if let back = macOSNavigationBackElement(app: app) {
+      tapElementCenter(app: app, element: back)
+      return true
+    }
+    return false
+#elseif os(tvOS)
+    _ = pressTvRemote(.menu)
+    return true
+#else
+    let buttons = app.navigationBars.buttons.allElementsBoundByIndex
+    if let back = buttons.first(where: { $0.isHittable }) {
+      back.tap()
+      return true
+    }
+    if isSnapshotXCTestChannelPenalized(bundleId: currentBundleId) {
+      NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_SKIPPED_XCTEST_ENUMERATION bundle=%@", currentBundleId ?? "")
+    } else if let back = topNavigationBackElement(app: app) {
+      tapElementCenter(app: app, element: back)
+      return true
+    }
+    return tapTopLeadingNavigationFallback(app: app)
+#endif
+  }
+
+  private func tapElementCenter(app: XCUIApplication, element: XCUIElement) {
+    let frame = element.frame
+    if !frame.isEmpty {
+      _ = tapAt(app: app, x: frame.midX, y: frame.midY)
+      return
+    }
+#if !os(tvOS)
+    element.tap()
+#endif
+  }
+
+  private func topNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
+#if os(iOS)
+    let frame = onScreenWindowFrame(app: app)
+    let candidates = app.buttons.matching(Self.navigationBackPredicate()).allElementsBoundByIndex.compactMap {
+      element -> (XCUIElement, Int)? in
+      guard element.exists, element.isHittable else { return nil }
+      guard Self.isTopNavigationControlFrame(element.frame, in: frame) else { return nil }
+      guard let rank = Self.navigationBackControlRank(
+        label: element.label,
+        identifier: element.identifier
+      ) else {
+        return nil
+      }
+      return (element, rank)
+    }
+    return candidates.sorted { lhs, rhs in
+      if lhs.1 != rhs.1 { return lhs.1 < rhs.1 }
+      let leftFrame = lhs.0.frame
+      let rightFrame = rhs.0.frame
+      if leftFrame.minY != rightFrame.minY { return leftFrame.minY < rightFrame.minY }
+      return leftFrame.minX < rightFrame.minX
+    }.first?.0
+#else
+    return nil
+#endif
+  }
+
+  static func navigationBackPredicate() -> NSPredicate {
+    let clauses = navigationBackKeywords.flatMap { _ in
+      ["label CONTAINS[c] %@", "identifier CONTAINS[c] %@"]
+    }.joined(separator: " OR ")
+    let arguments = navigationBackKeywords.flatMap { [$0, $0] }
+    return NSPredicate(format: clauses, argumentArray: arguments)
+  }
+
+  static func navigationBackControlRank(label: String, identifier: String) -> Int? {
+    let text = "\(label) \(identifier)".lowercased()
+    return navigationBackKeywords.firstIndex { text.contains($0) }
+  }
+
+  static func isTopNavigationControlFrame(_ candidate: CGRect, in window: CGRect) -> Bool {
+    guard
+      candidate.width.isFinite,
+      candidate.height.isFinite,
+      window.width.isFinite,
+      window.height.isFinite,
+      candidate.width > 0,
+      candidate.height > 0,
+      window.width > 0,
+      window.height > 0
+    else {
+      return false
+    }
+    // Accept the compact navigation/search header band without matching deep content controls.
+    let maxY = window.minY + min(max(window.height * 0.22, 96), 180)
+    return candidate.midY >= window.minY && candidate.midY <= maxY
+  }
+
+  static func topLeadingNavigationFallbackPoint(in frame: CGRect) -> CGPoint? {
+    guard frame.width.isFinite, frame.height.isFinite, frame.width > 0, frame.height > 0 else {
+      return nil
+    }
+    // Aim at the standard leading navigation slot, bounded for compact and tablet widths.
+    let xOffset = min(max(frame.width * 0.08, 28), 44)
+    // Sit below the status/dynamic-island region and inside common custom RN search headers.
+    let yOffset = min(max(frame.height * 0.155, 56), 132)
+    return CGPoint(x: frame.minX + xOffset, y: frame.minY + yOffset)
+  }
+
+  private func tapTopLeadingNavigationFallback(app: XCUIApplication) -> Bool {
+#if os(iOS)
+    let frame = onScreenWindowFrame(app: app)
+    guard let point = Self.topLeadingNavigationFallbackPoint(in: frame) else {
+      return false
+    }
+    let before = captureNavigationFallbackVisualState()
+    let context = synthesizedCoordinateContext(
+      policy: synthesizedGesturePolicy(.coordinateTap)
+    )?.withReferenceFrame(frame)
+    let synthesized = performGesture(app, idleTimeout: false) {
+      synthesizedTapAt(app: app, x: point.x, y: point.y, context: context)
+    }
+    if case .performed = synthesized.outcome {
+      return didNavigationFallbackChangeVisualState(before: before)
+    }
+    let fallback = performGesture(app) {
+      tapAt(app: app, x: point.x, y: point.y)
+    }
+    if case .performed = fallback.outcome {
+      return didNavigationFallbackChangeVisualState(before: before)
+    }
+#endif
+    return false
+  }
+
+  private func captureNavigationFallbackVisualState() -> Data? {
+#if os(iOS)
+    runnerPngData(for: XCUIScreen.main.screenshot().image)
+#else
+    return nil
+#endif
+  }
+
+  private func didNavigationFallbackChangeVisualState(before: Data?) -> Bool {
+    sleepFor(Self.navigationFallbackVerificationDelay)
+    let after = captureNavigationFallbackVisualState()
+    let changed = Self.didNavigationFallbackChangeVisualState(before: before, after: after)
+    if !changed {
+      NSLog("AGENT_DEVICE_RUNNER_IN_APP_BACK_FALLBACK_NO_STATE_CHANGE")
+    }
+    return changed
+  }
+
+  static func didNavigationFallbackChangeVisualState(before: Data?, after: Data?) -> Bool {
+    guard let before, let after else { return false }
+    return before != after
+  }
+
+  private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {
+    let predicate = NSPredicate(
+      format: "identifier == %@ OR label == %@",
+      "go back",
+      "Back"
+    )
+    let element = app.descendants(matching: .any).matching(predicate).firstMatch
+    return element.exists ? element : nil
+  }
+
+}

package/dist/apple-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift

@@ -0,0 +1,64 @@
+import XCTest
+
+// Swift port of buildScrollGesturePlan from src/core/scroll-gesture.ts.
+//
+// This is a deliberate two-place invariant: the daemon keeps the TS implementation (for Android,
+// recording, and reported-pixels), and the runner places the gesture with this Swift copy. The
+// parity test vectors at the bottom of this file mirror src/core/__tests__/scroll-gesture.test.ts —
+// if you change the math in either language, update the other and both vector sets.
+//
+// All inputs here are positive (reference dims, travel, center), so Swift's `.rounded()`
+// (half away from zero) matches JS `Math.round` (half up) on every value computed below.
+
+struct RunnerScrollGesturePlan {
+  let x1: Double
+  let y1: Double
+  let x2: Double
+  let y2: Double
+  let travelPixels: Double
+}
+
+private let runnerDefaultScrollAmount = 0.6
+private let runnerDefaultEdgePaddingFraction = 0.05
+
+func runnerScrollGesturePlan(
+  direction: String,
+  amount: Double?,
+  pixels: Double?,
+  referenceWidth: Double,
+  referenceHeight: Double
+) -> RunnerScrollGesturePlan? {
+  // Mirror the TS INVALID_ARGS contract: non-positive or non-finite amount/pixels are rejected
+  // rather than clamped into a journaled 1px scroll. The daemon validates before sending, so
+  // this only triggers for non-daemon wire clients.
+  if let amount, !(amount.isFinite && amount > 0) { return nil }
+  if let pixels, !(pixels.isFinite && pixels > 0) { return nil }
+  let axisLength = (direction == "up" || direction == "down") ? referenceHeight : referenceWidth
+  let requestedAmount = amount ?? runnerDefaultScrollAmount
+  let requestedPixels: Double =
+    pixels.map { max(1, $0.rounded()) } ?? (axisLength * requestedAmount).rounded()
+  let edgePadding = max(1, (axisLength * runnerDefaultEdgePaddingFraction).rounded())
+  let maxTravelPixels = max(1, axisLength - edgePadding * 2)
+  let travelPixels = max(1, min(requestedPixels, maxTravelPixels))
+  let halfTravel = (travelPixels / 2).rounded()
+  let centerX = (referenceWidth / 2).rounded()
+  let centerY = (referenceHeight / 2).rounded()
+
+  func plan(_ x1: Double, _ y1: Double, _ x2: Double, _ y2: Double) -> RunnerScrollGesturePlan {
+    RunnerScrollGesturePlan(x1: x1, y1: y1, x2: x2, y2: y2, travelPixels: travelPixels)
+  }
+
+  switch direction {
+  case "up":
+    return plan(centerX, centerY - halfTravel, centerX, centerY + halfTravel)
+  case "down":
+    return plan(centerX, centerY + halfTravel, centerX, centerY - halfTravel)
+  case "left":
+    return plan(centerX - halfTravel, centerY, centerX + halfTravel, centerY)
+  case "right":
+    return plan(centerX + halfTravel, centerY, centerX - halfTravel, centerY)
+  default:
+    return nil
+  }
+}
+

package/{apple-runner → dist/apple-runner}/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift

@@ -40,8 +40,10 @@ extension RunnerTests {
       ? resolvedTouchVisualizationFrame(app: activeApp, x: firstStep.x!, y: firstStep.y!)
       : nil
 
+    let synthesizedContext = synthesizedSequenceCoordinateContext(steps: steps)
+
     let execution = assembleSequenceExecution(steps: steps) { _, step in
-      performSequenceStep(step, activeApp: activeApp)
+      performSequenceStep(step, activeApp: activeApp, synthesizedContext: synthesizedContext)
     }
     return sequenceResponse(execution: execution, touchFrame: firstFrame)
   }
@@ -141,7 +143,8 @@ extension RunnerTests {
 
   private func performSequenceStep(
     _ step: SequenceStep,
-    activeApp: XCUIApplication
+    activeApp: XCUIApplication,
+    synthesizedContext: SynthesizedCoordinateContext? = nil
   ) -> SequenceStepOutcome {
     let x = step.x ?? 0
     let y = step.y ?? 0
@@ -149,10 +152,26 @@ extension RunnerTests {
     // a tapAt fallback when synthesis is unsupported), so fusing a jittered tap series does not
     // change the touch mechanism for these inputs.
     if step.kind == "tap", step.synthesized == true {
+      let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag
+#if os(iOS)
+      guard let synthesizedContext else {
+        let nowMs = ProcessInfo.processInfo.systemUptime * 1000
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: nil, fallbackAttempted: false)
+        return SequenceStepOutcome(
+          outcome: .unsupported(
+            message: "synthesized coordinate tap could not resolve a finite coordinate frame",
+            hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
+          ),
+          gestureStartUptimeMs: nowMs,
+          gestureEndUptimeMs: nowMs
+        )
+      }
+#endif
       let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
-        synthesizedTapAt(app: activeApp, x: x, y: y)
+        synthesizedTapAt(app: activeApp, x: x, y: y, context: synthesizedContext)
       }
       if case .performed = outcome {
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
         if let pauseMs = step.pauseMs, pauseMs > 0 {
           sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
         }
@@ -162,11 +181,50 @@ extension RunnerTests {
           gestureEndUptimeMs: timing.gestureEndUptimeMs
         )
       }
+#if os(iOS)
+      guard synthesizedContext.allowsXCTestCoordinateFallback else {
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
+        return SequenceStepOutcome(
+          outcome: outcome,
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs
+        )
+      }
+      logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: true)
+#endif
       // Synthesis unsupported (e.g. macOS) — fall through to the drag-based tapAt below.
     }
     if step.kind == "drag", step.synthesized == true {
-      let dragPoints = keyboardAvoidingDragPoints(
+      let policyKind = SynthesizedGesturePolicyKind.synthesizedDrag
+      let dragPoints: DragPoints
+      let dragContext: SynthesizedCoordinateContext?
+#if os(iOS)
+      guard let dragPlan = axFreeSynthesizedDragPlan(
+        app: activeApp,
+        x: x,
+        y: y,
+        x2: step.x2 ?? x,
+        y2: step.y2 ?? y,
+        context: synthesizedContext
+      ) else {
+        let nowMs = ProcessInfo.processInfo.systemUptime * 1000
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: synthesizedContext, fallbackAttempted: false)
+        return SequenceStepOutcome(
+          outcome: .unsupported(
+            message: "synthesized coordinate drag could not resolve a finite coordinate frame",
+            hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
+          ),
+          gestureStartUptimeMs: nowMs,
+          gestureEndUptimeMs: nowMs
+        )
+      }
+      dragPoints = dragPlan.points
+      dragContext = dragPlan.context
+#else
+      dragPoints = keyboardAvoidingDragPoints(
         app: activeApp, x: x, y: y, x2: step.x2 ?? x, y2: step.y2 ?? y)
+      dragContext = nil
+#endif
       let durationMs = min(max(step.durationMs ?? 250, 16), 10000)
       let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
         synthesizedDragAt(
@@ -175,10 +233,12 @@ extension RunnerTests {
           y: dragPoints.y,
           x2: dragPoints.x2,
           y2: dragPoints.y2,
-          durationMs: durationMs
+          durationMs: durationMs,
+          context: dragContext
         )
       }
       if case .performed = outcome {
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: false)
         if let pauseMs = step.pauseMs, pauseMs > 0 {
           sleepFor(min(max(pauseMs, 0), 10000) / 1000.0)
         }
@@ -188,6 +248,17 @@ extension RunnerTests {
           gestureEndUptimeMs: timing.gestureEndUptimeMs
         )
       }
+#if os(iOS)
+      guard dragContext?.allowsXCTestCoordinateFallback == true else {
+        logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: false)
+        return SequenceStepOutcome(
+          outcome: outcome,
+          gestureStartUptimeMs: timing.gestureStartUptimeMs,
+          gestureEndUptimeMs: timing.gestureEndUptimeMs
+        )
+      }
+#endif
+      logSynthesizedGesturePolicyDecision(kind: policyKind, context: dragContext, fallbackAttempted: true)
       let fallbackHoldDuration = synthesizedSwipeFallbackHoldDuration(durationMs: step.durationMs ?? 250)
       let (fallbackTiming, fallbackOutcome) = performGesture(activeApp) {
         dragAt(
@@ -273,180 +344,3 @@ extension RunnerTests {
   }
 }
 
-#if AGENT_DEVICE_RUNNER_UNIT_TESTS
-// MARK: - In-bundle unit tests (device-free)
-
-extension RunnerTests {
-  func testSequenceDecodesStepsFromWire() throws {
-    let json = """
-    {"command":"sequence","commandId":"seq-1","steps":[
-      {"kind":"tap","x":100,"y":200},
-      {"kind":"doubleTap","x":101,"y":200},
-      {"kind":"longPress","x":102,"y":200,"durationMs":300},
-      {"kind":"drag","x":10,"y":600,"x2":10,"y2":200,"durationMs":250,"pauseMs":50}
-    ]}
-    """
-    let command = try JSONDecoder().decode(Command.self, from: Data(json.utf8))
-    XCTAssertEqual(command.command, .sequence)
-    XCTAssertEqual(command.steps?.count, 4)
-    XCTAssertEqual(command.steps?[0].kind, "tap")
-    XCTAssertEqual(command.steps?[1].kind, "doubleTap")
-    XCTAssertEqual(command.steps?[3].x2, 10)
-    XCTAssertEqual(command.steps?[3].pauseMs, 50)
-  }
-
-  func testSequenceAcceptsDoubleTapKind() {
-    // A doubleTap step missing coords must fail on the coords check, not the kind allowlist —
-    // proving "doubleTap" passes validateSequenceStep without needing a device to execute on.
-    let response = executeSequenceForTest(steps: [
-      sequenceStep(kind: "doubleTap", x: nil)
-    ])
-    XCTAssertEqual(response.ok, false)
-    XCTAssertEqual(response.error?.code, "INVALID_ARGS")
-    XCTAssertTrue(response.error?.message.contains("requires finite x and y") ?? false)
-    XCTAssertFalse(response.error?.message.contains("unsupported kind") ?? true)
-  }
-
-  func testSequenceRejectsUnknownKind() throws {
-    let response = executeSequenceForTest(steps: [
-      sequenceStep(kind: "tap", x: 1, y: 2),
-      sequenceStep(kind: "pinch", x: 3, y: 4),
-    ])
-    XCTAssertEqual(response.ok, false)
-    XCTAssertEqual(response.error?.code, "INVALID_ARGS")
-    XCTAssertTrue(response.error?.message.contains("step 1") ?? false)
-    XCTAssertTrue(response.error?.message.contains("pinch") ?? false)
-  }
-
-  func testSequenceRejectsEmpty() {
-    let response = executeSequenceForTest(steps: [])
-    XCTAssertEqual(response.ok, false)
-    XCTAssertEqual(response.error?.code, "INVALID_ARGS")
-  }
-
-  func testSequenceRejectsTooManySteps() {
-    let steps = (0..<21).map { _ in sequenceStep(kind: "tap", x: 1, y: 2) }
-    let response = executeSequenceForTest(steps: steps)
-    XCTAssertEqual(response.ok, false)
-    XCTAssertEqual(response.error?.code, "INVALID_ARGS")
-    XCTAssertTrue(response.error?.message.contains("at most 20") ?? false)
-  }
-
-  func testSequenceRejectsDragMissingSecondPoint() {
-    let response = executeSequenceForTest(steps: [
-      sequenceStep(kind: "tap", x: 1, y: 2),
-      sequenceStep(kind: "drag", x: 3, y: 4),
-    ])
-    XCTAssertEqual(response.ok, false)
-    XCTAssertEqual(response.error?.code, "INVALID_ARGS")
-    XCTAssertTrue(response.error?.message.contains("step 1") ?? false)
-  }
-
-  func testAssembleSequencePreservesOrderOnSuccess() {
-    let steps = [
-      sequenceStep(kind: "tap", x: 1, y: 1),
-      sequenceStep(kind: "longPress", x: 2, y: 2),
-      sequenceStep(kind: "tap", x: 3, y: 3),
-    ]
-    var calls: [Int] = []
-    let execution = assembleSequenceExecution(steps: steps) { index, _ in
-      calls.append(index)
-      return SequenceStepOutcome(
-        outcome: .performed,
-        gestureStartUptimeMs: Double(index * 10),
-        gestureEndUptimeMs: Double(index * 10 + 5)
-      )
-    }
-    XCTAssertEqual(calls, [0, 1, 2])
-    XCTAssertEqual(execution.completedSteps, 3)
-    XCTAssertNil(execution.failedStepIndex)
-    XCTAssertEqual(execution.results.map { $0.kind }, ["tap", "longPress", "tap"])
-    XCTAssertEqual(execution.gestureStartUptimeMs, 0)
-    XCTAssertEqual(execution.gestureEndUptimeMs, 25)
-  }
-
-  func testAssembleSequenceStopsAtFirstFailure() {
-    let steps = [
-      sequenceStep(kind: "tap", x: 1, y: 1),
-      sequenceStep(kind: "drag", x: 2, y: 2),
-      sequenceStep(kind: "tap", x: 3, y: 3),
-    ]
-    var calls: [Int] = []
-    let execution = assembleSequenceExecution(steps: steps) { index, _ in
-      calls.append(index)
-      if index == 1 {
-        return SequenceStepOutcome(
-          outcome: .unsupported(message: "drag unsupported", hint: nil),
-          gestureStartUptimeMs: 10,
-          gestureEndUptimeMs: 15
-        )
-      }
-      return SequenceStepOutcome(outcome: .performed, gestureStartUptimeMs: 0, gestureEndUptimeMs: 5)
-    }
-    // Step 2 is never invoked.
-    XCTAssertEqual(calls, [0, 1])
-    XCTAssertEqual(execution.completedSteps, 1)
-    XCTAssertEqual(execution.failedStepIndex, 1)
-    // results.count == completedSteps + 1 (the failed step).
-    XCTAssertEqual(execution.results.count, 2)
-    XCTAssertEqual(execution.results[1].ok, false)
-    XCTAssertEqual(execution.results[1].errorCode, "UNSUPPORTED_OPERATION")
-    XCTAssertEqual(execution.results[1].errorMessage, "drag unsupported")
-  }
-
-  func testSequenceWorstCaseResponseStaysUnderJournalCap() throws {
-    let longMessage = String(repeating: "e", count: 200)
-    let results = (0..<20).map { index in
-      SequenceStepResult(
-        ok: index < 19,
-        kind: "drag",
-        errorCode: index < 19 ? nil : "UNSUPPORTED_OPERATION",
-        errorMessage: index < 19 ? nil : longMessage,
-        gestureStartUptimeMs: 123456.789,
-        gestureEndUptimeMs: 123466.789
-      )
-    }
-    let response = Response(
-      ok: true,
-      data: DataPayload(
-        message: "sequence",
-        completedSteps: 19,
-        failedStepIndex: 19,
-        sequenceResults: results
-      )
-    )
-    let encoded = try JSONEncoder().encode(response)
-    XCTAssertLessThan(encoded.count, 16 * 1024)
-  }
-
-  private func sequenceStep(
-    kind: String,
-    x: Double?,
-    y: Double? = nil,
-    x2: Double? = nil,
-    y2: Double? = nil
-  ) -> SequenceStep {
-    SequenceStep(
-      kind: kind, x: x, y: y, x2: x2, y2: y2, durationMs: nil, pauseMs: nil, synthesized: nil)
-  }
-
-  /// Validation runs before any executor call, so the INVALID_ARGS paths are exercised without
-  /// reaching the device executor (which is never invoked when validation rejects).
-  private func executeSequenceForTest(steps: [SequenceStep]) -> Response {
-    let command = makeSequenceCommand(steps: steps)
-    return executeSequence(command: command, activeApp: app)
-  }
-
-  /// Build a sequence Command via JSON so the test does not depend on the memberwise init's
-  /// parameter order.
-  private func makeSequenceCommand(steps: [SequenceStep]) -> Command {
-    struct SequenceCommandFixture: Encodable {
-      let command = "sequence"
-      let commandId = "seq-test"
-      let steps: [SequenceStep]
-    }
-    let data = try! JSONEncoder().encode(SequenceCommandFixture(steps: steps))
-    return try! JSONDecoder().decode(Command.self, from: data)
-  }
-}
-#endif