agent-device 0.17.1 → 0.17.3

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift

@@ -143,11 +143,11 @@ final class RunnerCommandJournal {
     switch command.command {
     case .snapshot, .screenshot:
       return false
-    case .tap, .mouseClick, .tapSeries, .longPress, .interactionFrame, .drag, .dragSeries,
-         .remotePress, .type, .swipe, .findText, .querySelector, .readText, .back, .backInApp,
-         .backSystem, .home, .rotate, .appSwitcher, .keyboardDismiss, .keyboardReturn, .alert,
-         .pinch, .rotateGesture, .transformGesture, .recordStart, .recordStop, .status, .uptime,
-         .shutdown:
+    case .tap, .mouseClick, .longPress, .drag,
+         .remotePress, .type, .swipe, .scroll, .findText, .querySelector, .readText, .back,
+         .backInApp, .backSystem, .home, .rotate, .appSwitcher, .keyboardDismiss, .keyboardReturn,
+         .alert, .pinch, .sequence, .rotateGesture, .transformGesture, .recordStart, .recordStop,
+         .status, .uptime, .shutdown:
       return true
     }
   }
@@ -165,6 +165,46 @@ extension RunnerTests {
     XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.notAccepted.rawValue)
   }
 
+  func testStampingCurrentUptimePreservesPayload() {
+    let stamped = Response(ok: true, data: DataPayload(message: "recording started"))
+      .stampingCurrentUptimeMs(123.5)
+
+    XCTAssertEqual(stamped.ok, true)
+    XCTAssertEqual(stamped.data?.message, "recording started")
+    XCTAssertEqual(stamped.data?.currentUptimeMs, 123.5)
+  }
+
+  func testStampingCurrentUptimeCreatesPayloadWhenNil() {
+    let stamped = Response(ok: true).stampingCurrentUptimeMs(456.0)
+
+    XCTAssertEqual(stamped.ok, true)
+    XCTAssertEqual(stamped.data?.currentUptimeMs, 456.0)
+  }
+
+  func testStampingCurrentUptimeSkipsErrorResponses() {
+    let response = Response(ok: false, error: ErrorPayload(message: "boom"))
+    let stamped = response.stampingCurrentUptimeMs(789.0)
+
+    XCTAssertEqual(stamped.ok, false)
+    XCTAssertNil(stamped.data)
+    XCTAssertEqual(stamped.error?.message, "boom")
+  }
+
+  func testJournalStoredResponseStaysUnstamped() throws {
+    let journal = RunnerCommandJournal()
+    let recordStart = runnerJournalCommand("recordStart", id: "record-start-anchor")
+
+    journal.accept(command: recordStart)
+    journal.finish(
+      command: recordStart,
+      response: Response(ok: true, data: DataPayload(message: "recording started"))
+    )
+
+    let status = journal.status(commandId: "record-start-anchor")
+    let responseJson = try XCTUnwrap(status.lifecycleResponseJson)
+    XCTAssertFalse(responseJson.contains("currentUptimeMs"))
+  }
+
   func testCommandJournalRetentionPolicy() throws {
     let journal = RunnerCommandJournal()
 
@@ -219,6 +259,38 @@ extension RunnerTests {
     XCTAssertEqual(screenshotStatus.lifecycleResponseOk, true)
     XCTAssertNil(screenshotStatus.lifecycleResponseJson)
 
+    let scroll = runnerJournalCommand("scroll", id: "scroll-drag")
+    journal.accept(command: scroll)
+    journal.finish(
+      command: scroll,
+      response: Response(
+        ok: true,
+        data: DataPayload(
+          message: "scrolled",
+          gestureStartUptimeMs: 1,
+          gestureEndUptimeMs: 2,
+          x: 155,
+          y: 420,
+          x2: 155,
+          y2: 301,
+          referenceWidth: 300,
+          referenceHeight: 600
+        )
+      )
+    )
+
+    let scrollStatus = journal.status(commandId: "scroll-drag")
+    XCTAssertEqual(scrollStatus.lifecycleState, RunnerCommandLifecycleState.completed.rawValue)
+    XCTAssertEqual(scrollStatus.lifecycleResponseOk, true)
+    XCTAssertNotNil(scrollStatus.lifecycleResponseJson)
+    let scrollResponse = try decodeRunnerJournalResponse(scrollStatus.lifecycleResponseJson)
+    XCTAssertEqual(scrollResponse.data?.x, 155)
+    XCTAssertEqual(scrollResponse.data?.y, 420)
+    XCTAssertEqual(scrollResponse.data?.x2, 155)
+    XCTAssertEqual(scrollResponse.data?.y2, 301)
+    XCTAssertEqual(scrollResponse.data?.referenceWidth, 300)
+    XCTAssertEqual(scrollResponse.data?.referenceHeight, 600)
+
     let largeRead = runnerJournalCommand("readText", id: "large-read")
     journal.accept(command: largeRead)
     journal.finish(
@@ -262,6 +334,82 @@ extension RunnerTests {
     XCTAssertEqual(status.lifecycleErrorHint, hint)
   }
 
+  func testCommandJournalRetainsCompletedSequenceResults() throws {
+    let journal = RunnerCommandJournal()
+    let sequence = runnerJournalCommand("sequence", id: "sequence-completed")
+    let results = (0..<20).map { _ in
+      SequenceStepResult(
+        ok: true,
+        kind: "tap",
+        errorCode: nil,
+        errorMessage: nil,
+        gestureStartUptimeMs: 100,
+        gestureEndUptimeMs: 120
+      )
+    }
+
+    journal.accept(command: sequence)
+    journal.finish(
+      command: sequence,
+      response: Response(
+        ok: true,
+        data: DataPayload(
+          message: "sequence",
+          completedSteps: 20,
+          failedStepIndex: nil,
+          sequenceResults: results
+        )
+      )
+    )
+
+    let status = journal.status(commandId: "sequence-completed")
+    XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue)
+    XCTAssertEqual(status.lifecycleResponseOk, true)
+    let json = try XCTUnwrap(status.lifecycleResponseJson)
+    // Worst-case 20-step response must stay under the 16KB journal retention cap.
+    XCTAssertLessThan(json.utf8.count, 16 * 1024)
+    let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson)
+    XCTAssertEqual(decoded.data?.completedSteps, 20)
+    XCTAssertEqual(decoded.data?.sequenceResults?.count, 20)
+  }
+
+  func testCommandJournalRetainsFailedSequenceResults() throws {
+    let journal = RunnerCommandJournal()
+    let sequence = runnerJournalCommand("sequence", id: "sequence-failed")
+    let longError = String(repeating: "z", count: 200)
+    let results: [SequenceStepResult] = [
+      SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil,
+                         gestureStartUptimeMs: 100, gestureEndUptimeMs: 120),
+      SequenceStepResult(ok: true, kind: "tap", errorCode: nil, errorMessage: nil,
+                         gestureStartUptimeMs: 130, gestureEndUptimeMs: 150),
+      SequenceStepResult(ok: false, kind: "drag", errorCode: "UNSUPPORTED_OPERATION",
+                         errorMessage: longError, gestureStartUptimeMs: 160, gestureEndUptimeMs: 180),
+    ]
+
+    journal.accept(command: sequence)
+    journal.finish(
+      command: sequence,
+      response: Response(
+        ok: true,
+        data: DataPayload(
+          message: "sequence",
+          completedSteps: 2,
+          failedStepIndex: 2,
+          sequenceResults: results
+        )
+      )
+    )
+
+    let status = journal.status(commandId: "sequence-failed")
+    XCTAssertEqual(status.lifecycleState, RunnerCommandLifecycleState.completed.rawValue)
+    let decoded = try decodeRunnerJournalResponse(status.lifecycleResponseJson)
+    XCTAssertEqual(decoded.data?.completedSteps, 2)
+    XCTAssertEqual(decoded.data?.failedStepIndex, 2)
+    XCTAssertEqual(decoded.data?.sequenceResults?.count, 3)
+    XCTAssertEqual(decoded.data?.sequenceResults?[2].ok, false)
+    XCTAssertEqual(decoded.data?.sequenceResults?[2].errorCode, "UNSUPPORTED_OPERATION")
+  }
+
   private func runnerJournalCommand(_ command: String, id: String) -> Command {
     let json = #"{"command":"\#(command)","commandId":"\#(id)"}"#
     return try! JSONDecoder().decode(Command.self, from: Data(json.utf8))

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+FlatSnapshotFiltering.swift

@@ -0,0 +1,171 @@
+import XCTest
+
+struct FlatSnapshotFilterNode {
+  let isRoot: Bool
+  let label: String
+  let identifier: String
+  let valueText: String?
+  let visible: Bool
+
+  var hasContent: Bool {
+    return !label.isEmpty || !identifier.isEmpty || valueText != nil
+  }
+
+  func matchesScope(_ scope: String) -> Bool {
+    let haystack = [label, identifier, valueText ?? ""].joined(separator: "\n")
+    return haystack.localizedCaseInsensitiveContains(scope)
+  }
+}
+
+struct FlatSnapshotFilterDecision {
+  let include: Bool
+  let insideMatchedScope: Bool
+}
+
+extension RunnerTests {
+  func flatSnapshotFilterDecision(
+    _ node: FlatSnapshotFilterNode,
+    options: SnapshotOptions,
+    insideMatchedScope: Bool
+  ) -> FlatSnapshotFilterDecision {
+    let scope = options.scope?.trimmingCharacters(in: .whitespacesAndNewlines)
+    let scopeActive = scope?.isEmpty == false
+    let matchesScope: Bool
+    if scopeActive, let scope {
+      matchesScope = node.matchesScope(scope)
+    } else {
+      matchesScope = false
+    }
+    let nowInsideScope = insideMatchedScope || matchesScope
+
+    let include: Bool
+    if node.isRoot {
+      include = true
+    } else if scopeActive && !nowInsideScope {
+      include = false
+    } else if options.interactiveOnly && !node.visible {
+      include = false
+    } else {
+      include = true
+    }
+
+    return FlatSnapshotFilterDecision(include: include, insideMatchedScope: nowInsideScope)
+  }
+
+  func privateAXInteractiveCandidate(rawElementType: Int) -> Bool {
+    guard let type = flatSnapshotElementType(rawElementType: rawElementType) else {
+      return false
+    }
+    return interactiveTypes.contains(type) || Self.scrollContainerTypes.contains(type)
+  }
+
+  func flatSnapshotElementType(rawElementType: Int) -> XCUIElement.ElementType? {
+    guard let raw = UInt(exactly: rawElementType),
+      let type = XCUIElement.ElementType(rawValue: raw)
+    else {
+      return nil
+    }
+    return type
+  }
+
+  func testFlatSnapshotFilterDecisionMatrixCoversOptions() {
+    let visibleContent = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "Welcome back",
+      identifier: "",
+      valueText: nil,
+      visible: true
+    )
+    let hiddenInteractive = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "Hidden menu",
+      identifier: "",
+      valueText: nil,
+      visible: false
+    )
+    let decorative = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "",
+      identifier: "",
+      valueText: nil,
+      visible: true
+    )
+
+    XCTAssertTrue(
+      flatSnapshotFilterDecision(
+        visibleContent,
+        options: SnapshotOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false),
+        insideMatchedScope: false
+      ).include
+    )
+    XCTAssertFalse(
+      flatSnapshotFilterDecision(
+        hiddenInteractive,
+        options: SnapshotOptions(interactiveOnly: true, depth: nil, scope: nil, raw: false),
+        insideMatchedScope: false
+      ).include
+    )
+    XCTAssertTrue(
+      flatSnapshotFilterDecision(
+        decorative,
+        options: SnapshotOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false),
+        insideMatchedScope: false
+      ).include
+    )
+    XCTAssertTrue(
+      flatSnapshotFilterDecision(
+        decorative,
+        options: SnapshotOptions(interactiveOnly: false, depth: nil, scope: nil, raw: false),
+        insideMatchedScope: false
+      ).include
+    )
+  }
+
+  func testFlatSnapshotFilterDecisionCarriesSubtreeScopeState() {
+    let scopeRoot = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "",
+      identifier: "homeScreen",
+      valueText: nil,
+      visible: true
+    )
+    let unmatchedDescendant = FlatSnapshotFilterNode(
+      isRoot: false,
+      label: "Post body without the scope text",
+      identifier: "",
+      valueText: nil,
+      visible: true
+    )
+    let options = SnapshotOptions(interactiveOnly: false, depth: nil, scope: "homeScreen", raw: false)
+
+    let rootDecision = flatSnapshotFilterDecision(
+      scopeRoot,
+      options: options,
+      insideMatchedScope: false
+    )
+    XCTAssertTrue(rootDecision.include)
+    XCTAssertTrue(rootDecision.insideMatchedScope)
+
+    XCTAssertTrue(
+      flatSnapshotFilterDecision(
+        unmatchedDescendant,
+        options: options,
+        insideMatchedScope: rootDecision.insideMatchedScope
+      ).include
+    )
+    XCTAssertFalse(
+      flatSnapshotFilterDecision(
+        unmatchedDescendant,
+        options: options,
+        insideMatchedScope: false
+      ).include
+    )
+  }
+
+  func testPrivateAXInteractiveCandidatesPreserveBackendInputs() {
+    XCTAssertTrue(
+      privateAXInteractiveCandidate(rawElementType: Int(XCUIElement.ElementType.scrollView.rawValue)),
+      "private AX marks scroll containers as interactive candidates"
+    )
+  }
+}

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -841,16 +841,6 @@ extension RunnerTests {
 #endif
   }
 
-  func runSeries(count: Int, pauseMs: Double, operation: (Int) -> Void) {
-    let total = max(count, 1)
-    let pause = max(pauseMs, 0)
-    for idx in 0..<total {
-      operation(idx)
-      if idx < total - 1 && pause > 0 {
-        sleepFor(pause / 1000.0)
-      }
-    }
-  }
 
   func swipe(app: XCUIApplication, direction: String) -> DragVisualizationFrame? {
     if performTvRemoteSwipeIfAvailable(direction: direction) {

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -3,14 +3,12 @@
 enum CommandType: String, Codable {
   case tap
   case mouseClick
-  case tapSeries
   case longPress
-  case interactionFrame
   case drag
-  case dragSeries
   case remotePress
   case type
   case swipe
+  case scroll
   case findText
   case querySelector
   case readText
@@ -26,6 +24,7 @@ enum CommandType: String, Codable {
   case keyboardReturn
   case alert
   case pinch
+  case sequence
   case rotateGesture
   case transformGesture
   case recordStart
@@ -68,16 +67,16 @@ extension CommandType {
   var traits: CommandTraits {
     switch self {
     // Interaction commands: require the foreground-guard + stabilization preflight.
-    // tapSeries/dragSeries are the series forms of tap/drag; keyboardReturn is the sibling
-    // of keyboardDismiss — all three were missing from the historical switch (drift the
-    // table now prevents) and are classified as interactions here.
-    case .tap, .tapSeries, .longPress, .drag, .dragSeries, .remotePress, .type, .swipe,
+    // keyboardReturn is the sibling of keyboardDismiss (missing from the historical switch —
+    // drift the table now prevents). .scroll is the fused frame-resolve + drag scroll; same
+    // classification as .drag. .sequence is the fused multi-step gesture batch.
+    case .tap, .longPress, .drag, .remotePress, .type, .swipe, .scroll,
          .back, .backInApp, .backSystem, .rotate, .appSwitcher,
-         .keyboardDismiss, .keyboardReturn, .pinch, .rotateGesture, .transformGesture:
+         .keyboardDismiss, .keyboardReturn, .pinch, .sequence, .rotateGesture, .transformGesture:
       return CommandTraits(isInteraction: true, readOnly: .never, isLifecycle: false)
 
     // Read-only reads: eligible for the session-invalidating retry.
-    case .interactionFrame, .findText, .readText, .snapshot:
+    case .findText, .readText, .snapshot:
       return CommandTraits(isInteraction: false, readOnly: .always, isLifecycle: false)
 
     // Screenshot is both a read and a runner-lifecycle command (skips app-activation preflight).
@@ -123,17 +122,14 @@ struct Command: Codable {
   let y: Double?
   let button: String?
   let remoteButton: String?
-  let count: Double?
-  let intervalMs: Double?
-  let doubleTap: Bool?
-  let pauseMs: Double?
-  let pattern: String?
   let x2: Double?
   let y2: Double?
   let dx: Double?
   let dy: Double?
   let durationMs: Double?
   let direction: String?
+  let amount: Double?
+  let pixels: Double?
   let orientation: String?
   let scale: Double?
   let degrees: Double?
@@ -142,12 +138,39 @@ struct Command: Codable {
   let fps: Int?
   let quality: Int?
   let interactiveOnly: Bool?
-  let compact: Bool?
   let depth: Int?
   let scope: String?
   let raw: Bool?
   let fullscreen: Bool?
   let synthesized: Bool?
+  let steps: [SequenceStep]?
+}
+
+/// One allowlisted coordinate gesture step inside a fused `sequence` command.
+/// `kind` is decoded as a raw String (not an enum) so the runner can return a clear
+/// INVALID_ARGS for an unknown kind instead of a generic decode failure.
+struct SequenceStep: Codable {
+  let kind: String
+  let x: Double?
+  let y: Double?
+  let x2: Double?
+  let y2: Double?
+  let durationMs: Double?
+  let pauseMs: Double?
+  /// For `tap` steps on iOS non-tv: use the synthesized HID tap fast path (synthesizedTapAt)
+  /// instead of the drag-based XCUICoordinate tapAt, matching the individual `tap` command.
+  let synthesized: Bool?
+}
+
+/// Per-step result for a `sequence` response. `ok:false` carries the failing step's
+/// errorCode/errorMessage; execution stops at the first failed step.
+struct SequenceStepResult: Codable {
+  let ok: Bool
+  let kind: String
+  let errorCode: String?
+  let errorMessage: String?
+  let gestureStartUptimeMs: Double?
+  let gestureEndUptimeMs: Double?
 }
 
 struct Response: Codable {
@@ -162,6 +185,18 @@ struct Response: Codable {
   }
 }
 
+extension Response {
+  // The daemon pairs this gesture-clock anchor with its own receipt time to map
+  // gesture uptimes onto wall-clock for the recording touch overlay. Error responses
+  // carry no anchor so the daemon falls back instead of pairing a stale value.
+  func stampingCurrentUptimeMs(_ value: Double) -> Response {
+    guard ok else { return self }
+    var payload = data ?? DataPayload()
+    payload.currentUptimeMs = value
+    return Response(ok: ok, data: payload, error: error)
+  }
+}
+
 struct DataPayload: Codable {
   let message: String?
   let text: String?
@@ -169,6 +204,7 @@ struct DataPayload: Codable {
   let items: [String]?
   let nodes: [SnapshotNode]?
   let truncated: Bool?
+  let snapshotQuality: SnapshotQuality?
   let gestureStartUptimeMs: Double?
   let gestureEndUptimeMs: Double?
   let x: Double?
@@ -177,7 +213,7 @@ struct DataPayload: Codable {
   let y2: Double?
   let referenceWidth: Double?
   let referenceHeight: Double?
-  let currentUptimeMs: Double?
+  var currentUptimeMs: Double?
   let commandId: String?
   let lifecycleState: String?
   let lifecycleCommand: String?
@@ -195,6 +231,9 @@ struct DataPayload: Codable {
   let gestureFallbackHint: String?
   let runnerFatal: Bool?
   let runnerFatalReason: String?
+  let completedSteps: Int?
+  let failedStepIndex: Int?
+  let sequenceResults: [SequenceStepResult]?
 
   init(
     message: String? = nil,
@@ -203,6 +242,7 @@ struct DataPayload: Codable {
     items: [String]? = nil,
     nodes: [SnapshotNode]? = nil,
     truncated: Bool? = nil,
+    snapshotQuality: SnapshotQuality? = nil,
     gestureStartUptimeMs: Double? = nil,
     gestureEndUptimeMs: Double? = nil,
     x: Double? = nil,
@@ -228,7 +268,10 @@ struct DataPayload: Codable {
     gestureFallbackMessage: String? = nil,
     gestureFallbackHint: String? = nil,
     runnerFatal: Bool? = nil,
-    runnerFatalReason: String? = nil
+    runnerFatalReason: String? = nil,
+    completedSteps: Int? = nil,
+    failedStepIndex: Int? = nil,
+    sequenceResults: [SequenceStepResult]? = nil
   ) {
     self.message = message
     self.text = text
@@ -236,6 +279,7 @@ struct DataPayload: Codable {
     self.items = items
     self.nodes = nodes
     self.truncated = truncated
+    self.snapshotQuality = snapshotQuality
     self.gestureStartUptimeMs = gestureStartUptimeMs
     self.gestureEndUptimeMs = gestureEndUptimeMs
     self.x = x
@@ -262,6 +306,9 @@ struct DataPayload: Codable {
     self.gestureFallbackHint = gestureFallbackHint
     self.runnerFatal = runnerFatal
     self.runnerFatalReason = runnerFatalReason
+    self.completedSteps = completedSteps
+    self.failedStepIndex = failedStepIndex
+    self.sequenceResults = sequenceResults
   }
 }
 
@@ -303,7 +350,6 @@ struct SnapshotNode: Codable {
 
 struct SnapshotOptions {
   let interactiveOnly: Bool
-  let compact: Bool
   let depth: Int?
   let scope: String?
   let raw: Bool