agent-device 0.17.2 → 0.17.4

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift

@@ -0,0 +1,171 @@
+import XCTest
+
+struct FlatSnapshotFilterNode {
+  let isRoot: Bool
+  let label: String
+  let identifier: String
+  let valueText: String?
+  let visible: Bool
+
+  var hasContent: Bool {
+    return !label.isEmpty || !identifier.isEmpty || valueText != nil
+  }
+
+  func matchesScope(_ scope: String) -> Bool {
+    let haystack = [label, identifier, valueText ?? ""].joined(separator: "\n")
+    return haystack.localizedCaseInsensitiveContains(scope)
+  }
+}
+
+struct FlatSnapshotFilterDecision {
+  let include: Bool
+  let insideMatchedScope: Bool
+}
+
+extension RunnerTests {
+  func flatSnapshotFilterDecision(
+    _ node: FlatSnapshotFilterNode,
+    options: SnapshotOptions,
+    insideMatchedScope: Bool
+  ) -> FlatSnapshotFilterDecision {
+    let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines)
+    let scopeActive = scope?.isEmpty == false
+    let matchesScope: Bool
+    if scopeActive, let scope {
+      matchesScope = node.matchesScope(scope)
+    } else {
+      matchesScope = false
+    }
+    let nowInsideScope = insideMatchedScope || matchesScope
+
+    let include: Bool
+    if node.isRoot {
+      include = true
+    } else if scopeActive && !nowInsideScope {
+      include = false
+    } else if options.interactiveOnly && !node.visible {
+      include = false
+    } else {
+      include = true
+    }
+
+    return FlatSnapshotFilterDecision(include: include, insideMatchedScope: nowInsideScope)
+  }
+
+  func privateAXInteractiveCandidate(rawElementType: Int) -> Bool {
+    guard let type = flatSnapshotElementType(rawElementType: rawElementType) else {
+      return false
+    }
+    return interactiveTypes.contains(type) || Self.scrollContainerTypes.contains(type)
+  }
+
+  func flatSnapshotElementType(rawElementType: Int) -> XCUIElement.ElementType? {
+    guard let raw = UInt(exactly: rawElementType),
+      let type = XCUIElement.ElementType(rawValue: raw)
+    else {
+      return nil
+    }
+    return type
+  }
+
+  func testFlatSnapshotFilterDecisionMatrixCoversOptions() {
+    let visibleContent = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "Welcome back",
+      identifier: "",
+      valueText: nil,
+      visible: true
+    )
+    let hiddenInteractive = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "Hidden menu",
+      identifier: "",
+      valueText: nil,
+      visible: false
+    )
+    let decorative = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "",
+      identifier: "",
+      valueText: nil,
+      visible: true
+    )
+
+    XCTAssertTrue(
+      flatSnapshotFilterDecision(
+        visibleContent,
+        options: SnapshotOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false),
+        insideMatchedScope: false
+      ).include
+    )
+    XCTAssertFalse(
+      flatSnapshotFilterDecision(
+        hiddenInteractive,
+        options: SnapshotOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false),
+        insideMatchedScope: false
+      ).include
+    )
+    XCTAssertTrue(
+      flatSnapshotFilterDecision(
+        decorative,
+        options: SnapshotOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false),
+        insideMatchedScope: false
+      ).include
+    )
+    XCTAssertTrue(
+      flatSnapshotFilterDecision(
+        decorative,
+        options: SnapshotOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false),
+        insideMatchedScope: false
+      ).include
+    )
+  }
+
+  func testFlatSnapshotFilterDecisionCarriesSubtreeScopeState() {
+    let scopeRoot = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "",
+      identifier: "homeScreen",
+      valueText: nil,
+      visible: true
+    )
+    let unmatchedDescendant = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "Post body without the scope text",
+      identifier: "",
+      valueText: nil,
+      visible: true
+    )
+    let options = SnapshotOptions(interactiveOnly: false, depth: nil, scope: "homeScreen", raw: false)
+
+    let rootDecision = flatSnapshotFilterDecision(
+      scopeRoot,
+      options: options,
+      insideMatchedScope: false
+    )
+    XCTAssertTrue(rootDecision.include)
+    XCTAssertTrue(rootDecision.insideMatchedScope)
+
+    XCTAssertTrue(
+      flatSnapshotFilterDecision(
+        unmatchedDescendant,
+        options: options,
+        insideMatchedScope: rootDecision.insideMatchedScope
+      ).include
+    )
+    XCTAssertFalse(
+      flatSnapshotFilterDecision(
+        unmatchedDescendant,
+        options: options,
+        insideMatchedScope: false
+      ).include
+    )
+  }
+
+  func testPrivateAXInteractiveCandidatesPreserveBackendInputs() {
+    XCTAssertTrue(
+      privateAXInteractiveCandidate(rawElementType: Int(XCUIElement.ElementType.scrollView.rawValue)),
+      "private AX marks scroll containers as interactive candidates"
+    )
+  }
+}

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -841,16 +841,6 @@ extension RunnerTests {
 #endif
   }
 
-  func runSeries(count: Int, pauseMs: Double, operation: (Int) -> Void) {
-    let total = max(count, 1)
-    let pause = max(pauseMs, 0)
-    for idx in 0..<total {
-      operation(idx)
-      if idx < total - 1 && pause > 0 {
-        sleepFor(pause / 1000.0)
-      }
-    }
-  }
 
   func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
     if performTvRemoteSwipeIfAvailable(direction: direction) {

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -3,14 +3,12 @@
 enum CommandType: String, Codable {
   case tap
   case mouseClick
-  case tapSeries
   case longPress
-  case interactionFrame
   case drag
-  case dragSeries
   case remotePress
   case type
   case swipe
+  case scroll
   case findText
   case querySelector
   case readText
@@ -26,6 +24,7 @@ enum CommandType: String, Codable {
   case keyboardReturn
   case alert
   case pinch
+  case sequence
   case rotateGesture
   case transformGesture
   case recordStart
@@ -68,16 +67,16 @@ extension CommandType {
   var traits: CommandTraits {
     switch self {
     // Interaction commands: require the foreground-guard + stabilization preflight.
-    // tapSeries/dragSeries are the series forms of tap/drag; keyboardReturn is the sibling
-    // of keyboardDismiss — all three were missing from the historical switch (drift the
-    // table now prevents) and are classified as interactions here.
-    case .tap, .tapSeries, .longPress, .drag, .dragSeries, .remotePress, .type, .swipe,
+    // keyboardReturn is the sibling of keyboardDismiss (missing from the historical switch —
+    // drift the table now prevents). .scroll is the fused frame-resolve + drag scroll; same
+    // classification as .drag. .sequence is the fused multi-step gesture batch.
+    case .tap, .longPress, .drag, .remotePress, .type, .swipe, .scroll,
          .back, .backInApp, .backSystem, .rotate, .appSwitcher,
-         .keyboardDismiss, .keyboardReturn, .pinch, .rotateGesture, .transformGesture:
+         .keyboardDismiss, .keyboardReturn, .pinch, .sequence, .rotateGesture, .transformGesture:
       return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false)
 
     // Read-only reads: eligible for the session-invalidating retry.
-    case .interactionFrame, .findText, .readText, .snapshot:
+    case .findText, .readText, .snapshot:
       return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: false)
 
     // Screenshot is both a read and a runner-lifecycle command (skips app-activation preflight).
@@ -123,17 +122,14 @@ struct Command: Codable {
   let y: Double?
   let button: String?
   let remoteButton: String?
-  let count: Double?
-  let intervalMs: Double?
-  let doubleTap: Bool?
-  let pauseMs: Double?
-  let pattern: String?
   let x2: Double?
   let y2: Double?
   let dx: Double?
   let dy: Double?
   let durationMs: Double?
   let direction: String?
+  let amount: Double?
+  let pixels: Double?
   let orientation: String?
   let scale: Double?
   let degrees: Double?
@@ -142,12 +138,39 @@ struct Command: Codable {
   let fps: Int?
   let quality: Int?
   let interactiveOnly: Bool?
-  let compact: Bool?
   let depth: Int?
   let scope: String?
   let raw: Bool?
   let fullscreen: Bool?
   let synthesized: Bool?
+  let steps: [SequenceStep]?
+}
+
+/// One allowlisted coordinate gesture step inside a fused `sequence` command.
+/// `kind` is decoded as a raw String (not an enum) so the runner can return a clear
+/// INVALID_ARGS for an unknown kind instead of a generic decode failure.
+struct SequenceStep: Codable {
+  let kind: String
+  let x: Double?
+  let y: Double?
+  let x2: Double?
+  let y2: Double?
+  let durationMs: Double?
+  let pauseMs: Double?
+  /// For `tap`/`drag` steps on iOS non-tv: use the synthesized HID fast path instead of the
+  /// drag-based XCUICoordinate path, matching the individual command behavior.
+  let synthesized: Bool?
+}
+
+/// Per-step result for a `sequence` response. `ok:false` carries the failing step's
+/// errorCode/errorMessage; execution stops at the first failed step.
+struct SequenceStepResult: Codable {
+  let ok: Bool
+  let kind: String
+  let errorCode: String?
+  let errorMessage: String?
+  let gestureStartUptimeMs: Double?
+  let gestureEndUptimeMs: Double?
 }
 
 struct Response: Codable {
@@ -162,6 +185,18 @@ struct Response: Codable {
   }
 }
 
+extension Response {
+  // The daemon pairs this gesture-clock anchor with its own receipt time to map
+  // gesture uptimes onto wall-clock for the recording touch overlay. Error responses
+  // carry no anchor so the daemon falls back instead of pairing a stale value.
+  func stampingCurrentUptimeMs(_ value: Double) -> Response {
+    guard ok else { return self }
+    var payload = data ?? DataPayload()
+    payload.currentUptimeMs = value
+    return Response(ok: ok, data: payload, error: error)
+  }
+}
+
 struct DataPayload: Codable {
   let message: String?
   let text: String?
@@ -169,6 +204,7 @@ struct DataPayload: Codable {
   let items: [String]?
   let nodes: [SnapshotNode]?
   let truncated: Bool?
+  let snapshotQuality: SnapshotQuality?
   let gestureStartUptimeMs: Double?
   let gestureEndUptimeMs: Double?
   let x: Double?
@@ -177,7 +213,7 @@ struct DataPayload: Codable {
   let y2: Double?
   let referenceWidth: Double?
   let referenceHeight: Double?
-  let currentUptimeMs: Double?
+  var currentUptimeMs: Double?
   let commandId: String?
   let lifecycleState: String?
   let lifecycleCommand: String?
@@ -195,6 +231,9 @@ struct DataPayload: Codable {
   let gestureFallbackHint: String?
   let runnerFatal: Bool?
   let runnerFatalReason: String?
+  let completedSteps: Int?
+  let failedStepIndex: Int?
+  let sequenceResults: [SequenceStepResult]?
 
   init(
     message: String? = nil,
@@ -203,6 +242,7 @@ struct DataPayload: Codable {
     items: [String]? = nil,
     nodes: [SnapshotNode]? = nil,
     truncated: Bool? = nil,
+    snapshotQuality: SnapshotQuality? = nil,
     gestureStartUptimeMs: Double? = nil,
     gestureEndUptimeMs: Double? = nil,
     x: Double? = nil,
@@ -228,7 +268,10 @@ struct DataPayload: Codable {
     gestureFallbackMessage: String? = nil,
     gestureFallbackHint: String? = nil,
     runnerFatal: Bool? = nil,
-    runnerFatalReason: String? = nil
+    runnerFatalReason: String? = nil,
+    completedSteps: Int? = nil,
+    failedStepIndex: Int? = nil,
+    sequenceResults: [SequenceStepResult]? = nil
   ) {
     self.message = message
     self.text = text
@@ -236,6 +279,7 @@ struct DataPayload: Codable {
     self.items = items
     self.nodes = nodes
     self.truncated = truncated
+    self.snapshotQuality = snapshotQuality
     self.gestureStartUptimeMs = gestureStartUptimeMs
     self.gestureEndUptimeMs = gestureEndUptimeMs
     self.x = x
@@ -262,6 +306,9 @@ struct DataPayload: Codable {
     self.gestureFallbackHint = gestureFallbackHint
     self.runnerFatal = runnerFatal
     self.runnerFatalReason = runnerFatalReason
+    self.completedSteps = completedSteps
+    self.failedStepIndex = failedStepIndex
+    self.sequenceResults = sequenceResults
   }
 }
 
@@ -303,7 +350,6 @@ struct SnapshotNode: Codable {
 
 struct SnapshotOptions {
   let interactiveOnly: Bool
-  let compact: Bool
   let depth: Int?
   let scope: String?
   let raw: Bool

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift

@@ -0,0 +1,200 @@
+import XCTest
+
+// Swift port of buildScrollGesturePlan from src/core/scroll-gesture.ts.
+//
+// This is a deliberate two-place invariant: the daemon keeps the TS implementation (for Android,
+// recording, and reported-pixels), and the runner places the gesture with this Swift copy. The
+// parity test vectors at the bottom of this file mirror src/core/__tests__/scroll-gesture.test.ts —
+// if you change the math in either language, update the other and both vector sets.
+//
+// All inputs here are positive (reference dims, travel, center), so Swift's `.rounded()`
+// (half away from zero) matches JS `Math.round` (half up) on every value computed below.
+
+struct RunnerScrollGesturePlan {
+  let x1: Double
+  let y1: Double
+  let x2: Double
+  let y2: Double
+  let travelPixels: Double
+}
+
+private let runnerDefaultScrollAmount = 0.6
+private let runnerDefaultEdgePaddingFraction = 0.05
+
+func runnerScrollGesturePlan(
+  direction: String,
+  amount: Double?,
+  pixels: Double?,
+  referenceWidth: Double,
+  referenceHeight: Double
+) -> RunnerScrollGesturePlan? {
+  // Mirror the TS INVALID_ARGS contract: non-positive or non-finite amount/pixels are rejected
+  // rather than clamped into a journaled 1px scroll. The daemon validates before sending, so
+  // this only triggers for non-daemon wire clients.
+  if let amount, !(amount.isFinite && amount > 0) { return nil }
+  if let pixels, !(pixels.isFinite && pixels > 0) { return nil }
+  let axisLength = (direction == "up" || direction == "down") ? referenceHeight : referenceWidth
+  let requestedAmount = amount ?? runnerDefaultScrollAmount
+  let requestedPixels: Double =
+    pixels.map { max(1, $0.rounded()) } ?? (axisLength * requestedAmount).rounded()
+  let edgePadding = max(1, (axisLength * runnerDefaultEdgePaddingFraction).rounded())
+  let maxTravelPixels = max(1, axisLength - edgePadding * 2)
+  let travelPixels = max(1, min(requestedPixels, maxTravelPixels))
+  let halfTravel = (travelPixels / 2).rounded()
+  let centerX = (referenceWidth / 2).rounded()
+  let centerY = (referenceHeight / 2).rounded()
+
+  func plan(_ x1: Double, _ y1: Double, _ x2: Double, _ y2: Double) -> RunnerScrollGesturePlan {
+    RunnerScrollGesturePlan(x1: x1, y1: y1, x2: x2, y2: y2, travelPixels: travelPixels)
+  }
+
+  switch direction {
+  case "up":
+    return plan(centerX, centerY - halfTravel, centerX, centerY + halfTravel)
+  case "down":
+    return plan(centerX, centerY + halfTravel, centerX, centerY - halfTravel)
+  case "left":
+    return plan(centerX - halfTravel, centerY, centerX + halfTravel, centerY)
+  case "right":
+    return plan(centerX + halfTravel, centerY, centerX - halfTravel, centerY)
+  default:
+    return nil
+  }
+}
+
+extension RunnerTests {
+  // Cross-language parity vectors mirroring src/core/__tests__/scroll-gesture.test.ts. Keep these
+  // in sync with the vitest vectors so the two buildScrollGesturePlan implementations cannot drift.
+
+  func testRunnerScrollGesturePlanMapsRelativeAmount() throws {
+    let plan = try XCTUnwrap(
+      runnerScrollGesturePlan(
+        direction: "down",
+        amount: 0.5,
+        pixels: nil,
+        referenceWidth: 400,
+        referenceHeight: 800
+      )
+    )
+    XCTAssertEqual(plan.x1, 200)
+    XCTAssertEqual(plan.y1, 600)
+    XCTAssertEqual(plan.x2, 200)
+    XCTAssertEqual(plan.y2, 200)
+    XCTAssertEqual(plan.travelPixels, 400)
+  }
+
+  func testRunnerScrollGesturePlanPixelsDown() throws {
+    // 300x600, down, pixels 120 -> (150,360)->(150,240), travel 120.
+    let plan = try XCTUnwrap(
+      runnerScrollGesturePlan(
+        direction: "down",
+        amount: nil,
+        pixels: 120,
+        referenceWidth: 300,
+        referenceHeight: 600
+      )
+    )
+    XCTAssertEqual(plan.x1, 150)
+    XCTAssertEqual(plan.y1, 360)
+    XCTAssertEqual(plan.x2, 150)
+    XCTAssertEqual(plan.y2, 240)
+    XCTAssertEqual(plan.travelPixels, 120)
+  }
+
+  func testRunnerScrollGesturePlanClampsAmountAboveOne() throws {
+    // 400x800, down, amount 2 -> requested 1600 clamps to the safe band (720): (200,760)->(200,40).
+    let plan = try XCTUnwrap(
+      runnerScrollGesturePlan(
+        direction: "down",
+        amount: 2,
+        pixels: nil,
+        referenceWidth: 400,
+        referenceHeight: 800
+      )
+    )
+    XCTAssertEqual(plan.x1, 200)
+    XCTAssertEqual(plan.y1, 760)
+    XCTAssertEqual(plan.x2, 200)
+    XCTAssertEqual(plan.y2, 40)
+    XCTAssertEqual(plan.travelPixels, 720)
+  }
+
+  func testRunnerScrollGesturePlanFloorsTinyFrames() throws {
+    // 2x2, down, pixels 10 engages every max(1, ...) floor and the .5 rounding cases the two
+    // ports must agree on (halfTravel 0.5 -> 1, center 1 from 2/2): (1,2)->(1,0), travel 1.
+    let plan = try XCTUnwrap(
+      runnerScrollGesturePlan(
+        direction: "down",
+        amount: nil,
+        pixels: 10,
+        referenceWidth: 2,
+        referenceHeight: 2
+      )
+    )
+    XCTAssertEqual(plan.x1, 1)
+    XCTAssertEqual(plan.y1, 2)
+    XCTAssertEqual(plan.x2, 1)
+    XCTAssertEqual(plan.y2, 0)
+    XCTAssertEqual(plan.travelPixels, 1)
+  }
+
+  func testRunnerScrollGesturePlanClampsToSafeBand() throws {
+    // 300x600, right, pixels 500 clamps travel to the safe band (270).
+    let plan = try XCTUnwrap(
+      runnerScrollGesturePlan(
+        direction: "right",
+        amount: nil,
+        pixels: 500,
+        referenceWidth: 300,
+        referenceHeight: 600
+      )
+    )
+    XCTAssertEqual(plan.x1, 285)
+    XCTAssertEqual(plan.x2, 15)
+    XCTAssertEqual(plan.y1, 300)
+    XCTAssertEqual(plan.y2, 300)
+    XCTAssertEqual(plan.travelPixels, 270)
+  }
+
+  func testRunnerScrollGesturePlanRejectsUnknownDirection() {
+    XCTAssertNil(
+      runnerScrollGesturePlan(
+        direction: "sideways",
+        amount: nil,
+        pixels: 100,
+        referenceWidth: 300,
+        referenceHeight: 600
+      )
+    )
+  }
+
+  func testRunnerScrollGesturePlanRejectsInvalidAmountAndPixels() {
+    XCTAssertNil(
+      runnerScrollGesturePlan(
+        direction: "down",
+        amount: 0,
+        pixels: nil,
+        referenceWidth: 300,
+        referenceHeight: 600
+      )
+    )
+    XCTAssertNil(
+      runnerScrollGesturePlan(
+        direction: "down",
+        amount: nil,
+        pixels: -10,
+        referenceWidth: 300,
+        referenceHeight: 600
+      )
+    )
+    XCTAssertNil(
+      runnerScrollGesturePlan(
+        direction: "down",
+        amount: .infinity,
+        pixels: nil,
+        referenceWidth: 300,
+        referenceHeight: 600
+      )
+    )
+  }
+}