agent-device 0.17.2 → 0.17.4

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -13,11 +13,11 @@ extension RunnerTests {
     return (gestureStartUptimeMs, currentUptimeMs())
   }
 
-  private func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval {
+  func synthesizedSwipeFallbackHoldDuration(durationMs: Double) -> TimeInterval {
     min(max((durationMs / 5.0) / 1000.0, 0.016), 0.120)
   }
 
-  private func coordinateDragHoldDuration() -> TimeInterval {
+  func coordinateDragHoldDuration() -> TimeInterval {
     0.050
   }
 
@@ -55,27 +55,6 @@ extension RunnerTests {
     }
   }
 
-  private func performDragSeries(
-    count: Int,
-    pauseMs: Double,
-    pattern: String,
-    points: DragPoints,
-    _ drag: (_ x: Double, _ y: Double, _ x2: Double, _ y2: Double) -> RunnerInteractionOutcome
-  ) -> RunnerInteractionOutcome {
-    var outcome = RunnerInteractionOutcome.performed
-    runSeries(count: count, pauseMs: pauseMs) { idx in
-      guard case .performed = outcome else {
-        return
-      }
-      let reverse = pattern == "ping-pong" && (idx % 2 == 1)
-      let startX = reverse ? points.x2 : points.x
-      let startY = reverse ? points.y2 : points.y
-      let endX = reverse ? points.x : points.x2
-      let endY = reverse ? points.y : points.y2
-      outcome = drag(startX, startY, endX, endY)
-    }
-    return outcome
-  }
 
   /// Runs a gesture action with uniform timing capture. Touch gestures pass `idleTimeout: true`
   /// (the default) to run inside the scroll idle-timeout + quiescence-skip wrapper; synthesis
@@ -84,7 +63,7 @@ extension RunnerTests {
   ///
   /// NOTE: a new SYNTHESIS gesture must pass `idleTimeout: false` — the default `true` would wrap
   /// it in the scroll idle-timeout/quiescence-skip path and change its runtime behavior.
-  private func performGesture(
+  func performGesture(
     _ app: XCUIApplication,
     idleTimeout: Bool = true,
     _ action: () -> RunnerInteractionOutcome
@@ -244,6 +223,8 @@ extension RunnerTests {
   }
 
   func executeUptime() -> Response {
+    // Placeholder value: the transport layer (jsonResponse) overwrites currentUptimeMs with a
+    // fresher send-time stamp on every ok response; kept so direct callers still get a value.
     Response(
       ok: true,
       data: DataPayload(currentUptimeMs: currentUptimeMs())
@@ -580,42 +561,6 @@ extension RunnerTests {
       } catch {
         return Response(ok: false, error: ErrorPayload(message: error.localizedDescription))
       }
-    case .tapSeries:
-      guard let x = command.x, let y = command.y else {
-        return Response(ok: false, error: ErrorPayload(message: "tapSeries requires x and y"))
-      }
-      let count = max(Int(command.count ?? 1), 1)
-      let intervalMs = max(command.intervalMs ?? 0, 0)
-      let doubleTap = command.doubleTap ?? false
-      let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
-      if doubleTap {
-        let (timing, outcome) = performGesture(activeApp) {
-          var outcome = RunnerInteractionOutcome.performed
-          runSeries(count: count, pauseMs: intervalMs) { _ in
-            if case .performed = outcome {
-              outcome = doubleTapAt(app: activeApp, x: x, y: y)
-            }
-          }
-          return outcome
-        }
-        if let response = unsupportedResponse(for: outcome) {
-          return response
-        }
-        return gestureResponse(message: "tap series", timing: timing, frame: .touch(touchFrame))
-      }
-      let (timing, outcome) = performGesture(activeApp) {
-        var outcome = RunnerInteractionOutcome.performed
-        runSeries(count: count, pauseMs: intervalMs) { _ in
-          if case .performed = outcome {
-            outcome = tapAt(app: activeApp, x: x, y: y)
-          }
-        }
-        return outcome
-      }
-      if let response = unsupportedResponse(for: outcome) {
-        return response
-      }
-      return gestureResponse(message: "tap series", timing: timing, frame: .touch(touchFrame))
     case .longPress:
       guard let x = command.x, let y = command.y else {
         return Response(ok: false, error: ErrorPayload(message: "longPress requires x and y"))
@@ -633,114 +578,63 @@ extension RunnerTests {
       guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
         return Response(ok: false, error: ErrorPayload(message: "drag requires x, y, x2, and y2"))
       }
-      let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
-      let dragFrame = resolvedDragVisualizationFrame(
-        app: activeApp,
-        x: dragPoints.x,
-        y: dragPoints.y,
-        x2: dragPoints.x2,
-        y2: dragPoints.y2
+      return executeDragGesture(
+        activeApp: activeApp,
+        x: x,
+        y: y,
+        x2: x2,
+        y2: y2,
+        durationMs: command.durationMs,
+        synthesized: command.synthesized == true,
+        message: "dragged"
       )
-      var fallback: GestureFallback?
-      if command.synthesized == true {
-        let durationMs = min(max(command.durationMs ?? 250, 16), 10000)
-        let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
-          synthesizedDragAt(
-            app: activeApp,
-            x: dragPoints.x,
-            y: dragPoints.y,
-            x2: dragPoints.x2,
-            y2: dragPoints.y2,
-            durationMs: durationMs
+    case .scroll:
+      // Fused frame-resolve + drag scroll for non-tvOS. Resolves the interaction frame via
+      // resolvedTouchReferenceFrame, computes drag endpoints with the Swift port of
+      // buildScrollGesturePlan, then runs the same non-synthesized drag path scroll's drag used.
+      guard let direction = command.direction,
+        direction == "up" || direction == "down" || direction == "left" || direction == "right"
+      else {
+        return Response(
+          ok: false,
+          error: ErrorPayload(
+            code: "INVALID_ARGS",
+            message: "scroll requires direction up|down|left|right"
           )
-        }
-        if case .performed = outcome {
-          return gestureResponse(message: "dragged", timing: timing, frame: .drag(dragFrame))
-        }
-        fallback = gestureFallback(strategy: "xctest-coordinate-drag", from: outcome)
-      }
-      let holdDuration = command.synthesized == true
-        ? synthesizedSwipeFallbackHoldDuration(durationMs: command.durationMs ?? 250)
-        : coordinateDragHoldDuration()
-      let (timing, outcome) = performGesture(activeApp) {
-        dragAt(
-          app: activeApp,
-          x: dragPoints.x,
-          y: dragPoints.y,
-          x2: dragPoints.x2,
-          y2: dragPoints.y2,
-          holdDuration: holdDuration
         )
       }
-      if let response = unsupportedResponse(for: outcome) {
-        return response
-      }
-      return gestureResponse(
-        message: "dragged",
-        timing: timing,
-        frame: .drag(dragFrame),
-        fallback: fallback
-      )
-    case .dragSeries:
-      guard let x = command.x, let y = command.y, let x2 = command.x2, let y2 = command.y2 else {
-        return Response(ok: false, error: ErrorPayload(message: "dragSeries requires x, y, x2, and y2"))
-      }
-      let count = max(Int(command.count ?? 1), 1)
-      let pauseMs = max(command.pauseMs ?? 0, 0)
-      let pattern = command.pattern ?? "one-way"
-      if pattern != "one-way" && pattern != "ping-pong" {
-        return Response(ok: false, error: ErrorPayload(message: "dragSeries pattern must be one-way or ping-pong"))
-      }
-      let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
-      var fallback: GestureFallback?
-      if command.synthesized == true {
-        let durationMs = min(max(command.durationMs ?? 250, 16), 10000)
-        let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
-          performDragSeries(
-            count: count,
-            pauseMs: pauseMs,
-            pattern: pattern,
-            points: dragPoints
-          ) { startX, startY, endX, endY in
-            synthesizedDragAt(
-              app: activeApp,
-              x: startX,
-              y: startY,
-              x2: endX,
-              y2: endY,
-              durationMs: durationMs
-            )
-          }
-        }
-        if case .performed = outcome {
-          return gestureResponse(message: "drag series", timing: timing)
-        }
-        fallback = gestureFallback(strategy: "xctest-coordinate-drag-series", from: outcome)
+      let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
+      guard frame.width > 0, frame.height > 0 else {
+        return Response(
+          ok: false,
+          error: ErrorPayload(message: "scroll could not resolve a usable interaction frame")
+        )
       }
-      let holdDuration = command.synthesized == true
-        ? synthesizedSwipeFallbackHoldDuration(durationMs: command.durationMs ?? 250)
-        : coordinateDragHoldDuration()
-      let (timing, outcome) = performGesture(activeApp) {
-        performDragSeries(
-          count: count,
-          pauseMs: pauseMs,
-          pattern: pattern,
-          points: dragPoints
-        ) { startX, startY, endX, endY in
-          dragAt(
-            app: activeApp,
-            x: startX,
-            y: startY,
-            x2: endX,
-            y2: endY,
-            holdDuration: holdDuration
+      guard let plan = runnerScrollGesturePlan(
+        direction: direction,
+        amount: command.amount,
+        pixels: command.pixels,
+        referenceWidth: frame.width,
+        referenceHeight: frame.height
+      ) else {
+        return Response(
+          ok: false,
+          error: ErrorPayload(
+            code: "INVALID_ARGS",
+            message: "scroll could not compute a gesture plan"
           )
-        }
-      }
-      if let response = unsupportedResponse(for: outcome) {
-        return response
+        )
       }
-      return gestureResponse(message: "drag series", timing: timing, fallback: fallback)
+      return executeDragGesture(
+        activeApp: activeApp,
+        x: frame.minX + plan.x1,
+        y: frame.minY + plan.y1,
+        x2: frame.minX + plan.x2,
+        y2: frame.minY + plan.y2,
+        durationMs: nil,
+        synthesized: false,
+        message: "scrolled"
+      )
     case .remotePress:
       guard let button = tvRemoteButton(from: command.remoteButton) else {
         return Response(ok: false, error: ErrorPayload(message: "remotePress requires remoteButton"))
@@ -759,17 +653,6 @@ extension RunnerTests {
         response = executeTypeCommand(activeApp: activeApp, command: command)
       }
       return response ?? Response(ok: false, error: ErrorPayload(message: "type produced no response"))
-    case .interactionFrame:
-      let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
-      return Response(
-        ok: true,
-        data: DataPayload(
-          x: frame.minX,
-          y: frame.minY,
-          referenceWidth: frame.width,
-          referenceHeight: frame.height
-        )
-      )
     case .swipe:
       guard let direction = command.direction else {
         return Response(ok: false, error: ErrorPayload(message: "swipe requires direction"))
@@ -808,7 +691,6 @@ extension RunnerTests {
     case .snapshot:
       let options = SnapshotOptions(
         interactiveOnly: command.interactiveOnly ?? false,
-        compact: command.compact ?? false,
         depth: command.depth,
         scope: command.scope,
         raw: command.raw ?? false
@@ -960,6 +842,8 @@ extension RunnerTests {
         return response
       }
       return gestureResponse(message: "pinched", timing: timing)
+    case .sequence:
+      return executeSequence(command: command, activeApp: activeApp)
     case .rotateGesture:
       guard let degrees = command.degrees, degrees.isFinite else {
         return Response(ok: false, error: ErrorPayload(message: "rotateGesture requires degrees"))
@@ -1023,6 +907,71 @@ extension RunnerTests {
     }
   }
 
+  /// Shared drag execution for `.drag` and the fused `.scroll`. Mirrors the original `.drag` body
+  /// exactly: keyboardAvoidingDragPoints -> resolvedDragVisualizationFrame -> synthesized branch
+  /// (16-10000ms clamp) or non-synthesized dragAt with coordinateDragHoldDuration ->
+  /// gestureResponse(.drag). `.scroll` always passes synthesized: false, pinning the same
+  /// non-synthesized drag path scroll's drag used today.
+  private func executeDragGesture(
+    activeApp: XCUIApplication,
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double,
+    durationMs: Double?,
+    synthesized: Bool,
+    message: String
+  ) -> Response {
+    let dragPoints = keyboardAvoidingDragPoints(app: activeApp, x: x, y: y, x2: x2, y2: y2)
+    let dragFrame = resolvedDragVisualizationFrame(
+      app: activeApp,
+      x: dragPoints.x,
+      y: dragPoints.y,
+      x2: dragPoints.x2,
+      y2: dragPoints.y2
+    )
+    var fallback: GestureFallback?
+    if synthesized {
+      let durationMs = min(max(durationMs ?? 250, 16), 10000)
+      let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
+        synthesizedDragAt(
+          app: activeApp,
+          x: dragPoints.x,
+          y: dragPoints.y,
+          x2: dragPoints.x2,
+          y2: dragPoints.y2,
+          durationMs: durationMs
+        )
+      }
+      if case .performed = outcome {
+        return gestureResponse(message: message, timing: timing, frame: .drag(dragFrame))
+      }
+      fallback = gestureFallback(strategy: "xctest-coordinate-drag", from: outcome)
+    }
+    let holdDuration = synthesized
+      ? synthesizedSwipeFallbackHoldDuration(durationMs: durationMs ?? 250)
+      : coordinateDragHoldDuration()
+    let (timing, outcome) = performGesture(activeApp) {
+      dragAt(
+        app: activeApp,
+        x: dragPoints.x,
+        y: dragPoints.y,
+        x2: dragPoints.x2,
+        y2: dragPoints.y2,
+        holdDuration: holdDuration
+      )
+    }
+    if let response = unsupportedResponse(for: outcome) {
+      return response
+    }
+    return gestureResponse(
+      message: message,
+      timing: timing,
+      frame: .drag(dragFrame),
+      fallback: fallback
+    )
+  }
+
   private func currentXCTestFailureCount() -> Int {
     return testRun?.failureCount ?? 0
   }

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift

@@ -143,11 +143,11 @@ final class RunnerCommandJournal {
     switch command.command {
     case .snapshot, .screenshot:
       return false
-    case .tap, .mouseClick, .tapSeries, .longPress, .interactionFrame, .drag, .dragSeries,
-         .remotePress, .type, .swipe, .findText, .querySelector, .readText, .back, .backInApp,
-         .backSystem, .home, .rotate, .appSwitcher, .keyboardDismiss, .keyboardReturn, .alert,
-         .pinch, .rotateGesture, .transformGesture, .recordStart, .recordStop, .status, .uptime,
-         .shutdown:
+    case .tap, .mouseClick, .longPress, .drag,
+         .remotePress, .type, .swipe, .scroll, .findText, .querySelector, .readText, .back,
+         .backInApp, .backSystem, .home, .rotate, .appSwitcher, .keyboardDismiss, .keyboardReturn,
+         .alert, .pinch, .sequence, .rotateGesture, .transformGesture, .recordStart, .recordStop,
+         .status, .uptime, .shutdown:
       return true
     }
   }
@@ -165,6 +165,46 @@ extension RunnerTests {
     XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.notAccepted.rawValue)
   }
 
+  func testStampingCurrentUptimePreservesPayload() {
+    let stamped = Response(ok: true, data: DataPayload(message: "recording started"))
+      .stampingCurrentUptimeMs(123.5)
+
+    XCTAssertEqual(stamped.ok, true)
+    XCTAssertEqual(stamped.data?.message, "recording started")
+    XCTAssertEqual(stamped.data?.currentUptimeMs, 123.5)
+  }
+
+  func testStampingCurrentUptimeCreatesPayloadWhenNil() {
+    let stamped = Response(ok: true).stampingCurrentUptimeMs(456.0)
+
+    XCTAssertEqual(stamped.ok, true)
+    XCTAssertEqual(stamped.data?.currentUptimeMs, 456.0)
+  }
+
+  func testStampingCurrentUptimeSkipsErrorResponses() {
+    let response = Response(ok: false, error: ErrorPayload(message: "boom"))
+    let stamped = response.stampingCurrentUptimeMs(789.0)
+
+    XCTAssertEqual(stamped.ok, false)
+    XCTAssertNil(stamped.data)
+    XCTAssertEqual(stamped.error?.message, "boom")
+  }
+
+  func testJournalStoredResponseStaysUnstamped() throws {
+    let journal = RunnerCommandJournal()
+    let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor")
+
+    journal.accept(command: recordStart)
+    journal.finish(
+      command: recordStart,
+      response: Response(ok: true, data: DataPayload(message: "recording started"))
+    )
+
+    let status = journal.status(commandId: "record-start-anchor")
+    let responseJson = try XCTUnwrap(status.lifecycleResponseJson)
+    XCTAssertFalse(responseJson.contains("currentUptimeMs"))
+  }
+
   func testCommandJournalRetentionPolicy() throws {
     let journal = RunnerCommandJournal()
 
@@ -219,6 +259,38 @@ extension RunnerTests {
     XCTAssertEqual(screenshotStatus.lifecycleResponseOk, true)
     XCTAssertNil(screenshotStatus.lifecycleResponseJson)
 
+    let scroll = runnerJournalCommand("scroll", id: "scroll-drag")
+    journal.accept(command: scroll)
+    journal.finish(
+      command: scroll,
+      response: Response(
+        ok: true,
+        data: DataPayload(
+          message: "scrolled",
+          gestureStartUptimeMs: 1,
+          gestureEndUptimeMs: 2,
+          x: 155,
+          y: 420,
+          x2: 155,
+          y2: 301,
+          referenceWidth: 300,
+          referenceHeight: 600
+        )
+      )
+    )
+
+    let scrollStatus = journal.status(commandId: "scroll-drag")
+    XCTAssertEqual(scrollStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue)
+    XCTAssertEqual(scrollStatus.lifecycleResponseOk, true)
+    XCTAssertNotNil(scrollStatus.lifecycleResponseJson)
+    let scrollResponse = try decodeRunnerJournalResponse(scrollStatus.lifecycleResponseJson)
+    XCTAssertEqual(scrollResponse.data?.x, 155)
+    XCTAssertEqual(scrollResponse.data?.y, 420)
+    XCTAssertEqual(scrollResponse.data?.x2, 155)
+    XCTAssertEqual(scrollResponse.data?.y2, 301)
+    XCTAssertEqual(scrollResponse.data?.referenceWidth, 300)
+    XCTAssertEqual(scrollResponse.data?.referenceHeight, 600)
+
     let largeRead = runnerJournalCommand("readText", id: "large-read")
     journal.accept(command: largeRead)
     journal.finish(
@@ -262,6 +334,82 @@ extension RunnerTests {
     XCTAssertEqual(status.lifecycleErrorHint, hint)
   }
 
+  func testCommandJournalRetainsCompletedSequenceResults() throws {
+    let journal = RunnerCommandJournal()
+    let sequence = runnerJournalCommand("sequence", id: "sequence-completed")
+    let results = (0..<20).map { _ in
+      SequenceStepResult(
+        ok: true,
+        kind: "tap",
+        errorCode: nil,
+        errorMessage: nil,
+        gestureStartUptimeMs: 100,
+        gestureEndUptimeMs: 120
+      )
+    }
+
+    journal.accept(command: sequence)
+    journal.finish(
+      command: sequence,
+      response: Response(
+        ok: true,
+        data: DataPayload(
+          message: "sequence",
+          completedSteps: 20,
+          failedStepIndex: nil,
+          sequenceResults: results
+        )
+      )
+    )
+
+    let status = journal.status(commandId: "sequence-completed")
+    XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue)
+    XCTAssertEqual(status.lifecycleResponseOk, true)
+    let json = try XCTUnwrap(status.lifecycleResponseJson)
+    // Worst-case 20-step response must stay under the 16KB journal retention cap.
+    XCTAssertLessThan(json.utf8.count, 16 * 1024)
+    let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson)
+    XCTAssertEqual(decoded.data?.completedSteps, 20)
+    XCTAssertEqual(decoded.data?.sequenceResults?.count, 20)
+  }
+
+  func testCommandJournalRetainsFailedSequenceResults() throws {
+    let journal = RunnerCommandJournal()
+    let sequence = runnerJournalCommand("sequence", id: "sequence-failed")
+    let longError = String(repeating: "z", count: 200)
+    let results: [SequenceStepResult] = [
+      SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil,
+                         gestureStartUptimeMs: 100, gestureEndUptimeMs: 120),
+      SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil,
+                         gestureStartUptimeMs: 130, gestureEndUptimeMs: 150),
+      SequenceStepResult(ok: false, kind: "drag", errorCode: "UNSUPPORTED_OPERATION",
+                         errorMessage: longError, gestureStartUptimeMs: 160, gestureEndUptimeMs: 180),
+    ]
+
+    journal.accept(command: sequence)
+    journal.finish(
+      command: sequence,
+      response: Response(
+        ok: true,
+        data: DataPayload(
+          message: "sequence",
+          completedSteps: 2,
+          failedStepIndex: 2,
+          sequenceResults: results
+        )
+      )
+    )
+
+    let status = journal.status(commandId: "sequence-failed")
+    XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue)
+    let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson)
+    XCTAssertEqual(decoded.data?.completedSteps, 2)
+    XCTAssertEqual(decoded.data?.failedStepIndex, 2)
+    XCTAssertEqual(decoded.data?.sequenceResults?.count, 3)
+    XCTAssertEqual(decoded.data?.sequenceResults?[2].ok, false)
+    XCTAssertEqual(decoded.data?.sequenceResults?[2].errorCode, "UNSUPPORTED_OPERATION")
+  }
+
   private func runnerJournalCommand(_ command: String, id: String) -> Command {
     let json = #"{"command":"\#(command)","commandId":"\#(id)"}"#
     return try! JSONDecoder().decode(Command.self, from: Data(json.utf8))