agent-device 0.10.0 → 0.10.2

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -1,6 +1,27 @@
 import XCTest
 
 extension RunnerTests {
+  struct TouchVisualizationFrame {
+    let x: Double
+    let y: Double
+    let referenceWidth: Double
+    let referenceHeight: Double
+  }
+
+  struct DragVisualizationFrame {
+    let x: Double
+    let y: Double
+    let x2: Double
+    let y2: Double
+    let referenceWidth: Double
+    let referenceHeight: Double
+  }
+
+  struct GestureReferenceFrame {
+    let referenceWidth: Double
+    let referenceHeight: Double
+  }
+
   // MARK: - Navigation Gestures
 
   func tapNavigationBack(app: XCUIApplication) -> Bool {
@@ -99,6 +120,40 @@ extension RunnerTests {
     return element.exists ? element : nil
   }
 
+  func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? {
+    let point = CGPoint(x: x, y: y)
+    let candidates = app.descendants(matching: .any).allElementsBoundByIndex
+      .filter { element in
+        element.exists && !element.frame.isEmpty && element.frame.contains(point)
+      }
+      .sorted { left, right in
+        let leftArea = max(1, left.frame.width * left.frame.height)
+        let rightArea = max(1, right.frame.width * right.frame.height)
+        if leftArea != rightArea {
+          return leftArea < rightArea
+        }
+        if left.frame.minY != right.frame.minY {
+          return left.frame.minY < right.frame.minY
+        }
+        if left.frame.minX != right.frame.minX {
+          return left.frame.minX < right.frame.minX
+        }
+        return left.elementType.rawValue < right.elementType.rawValue
+      }
+
+    for element in candidates where prefersExpandedTextRead(element) {
+      if let text = readableText(for: element) {
+        return text
+      }
+    }
+    for element in candidates {
+      if let text = readableText(for: element) {
+        return text
+      }
+    }
+    return nil
+  }
+
   func clearTextInput(_ element: XCUIElement) {
     moveCaretToEnd(element: element)
     let count = estimatedDeleteCount(for: element)
@@ -141,6 +196,32 @@ extension RunnerTests {
     return max(24, min(120, base))
   }
 
+  private func readableText(for element: XCUIElement) -> String? {
+    let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
+    let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
+    let valueText = String(describing: element.value ?? "")
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+    switch element.elementType {
+    case .textField, .secureTextField, .searchField, .textView:
+      if !valueText.isEmpty { return valueText }
+      if !label.isEmpty { return label }
+      return identifier.isEmpty ? nil : identifier
+    default:
+      if !label.isEmpty { return label }
+      if !valueText.isEmpty { return valueText }
+      return identifier.isEmpty ? nil : identifier
+    }
+  }
+
+  private func prefersExpandedTextRead(_ element: XCUIElement) -> Bool {
+    switch element.elementType {
+    case .textField, .secureTextField, .searchField, .textView:
+      return true
+    default:
+      return false
+    }
+  }
+
   func findScopeElement(app: XCUIApplication, scope: String) -> XCUIElement? {
     let predicate = NSPredicate(
       format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@",
@@ -209,6 +290,58 @@ extension RunnerTests {
     start.press(forDuration: holdDuration, thenDragTo: end)
   }
 
+  func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) -> TouchVisualizationFrame {
+    let appFrame = app.frame
+    let referenceFrame = resolvedTouchReferenceFrame(app: app, appFrame: appFrame)
+    let originX = appFrame.isEmpty ? referenceFrame.minX : appFrame.minX
+    let originY = appFrame.isEmpty ? referenceFrame.minY : appFrame.minY
+    return TouchVisualizationFrame(
+      x: originX + x,
+      y: originY + y,
+      referenceWidth: referenceFrame.width,
+      referenceHeight: referenceFrame.height
+    )
+  }
+
+  func resolvedDragVisualizationFrame(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double
+  ) -> DragVisualizationFrame {
+    let start = resolvedTouchVisualizationFrame(app: app, x: x, y: y)
+    let end = resolvedTouchVisualizationFrame(app: app, x: x2, y: y2)
+    return DragVisualizationFrame(
+      x: start.x,
+      y: start.y,
+      x2: end.x,
+      y2: end.y,
+      referenceWidth: start.referenceWidth,
+      referenceHeight: start.referenceHeight
+    )
+  }
+
+  private func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
+    let window = app.windows.firstMatch
+    let windowFrame = window.frame
+    if window.exists && !windowFrame.isEmpty {
+      return windowFrame
+    }
+    if !appFrame.isEmpty {
+      return appFrame
+    }
+    return CGRect(x: 0, y: 0, width: 0, height: 0)
+  }
+
+  func resolvedGestureReferenceFrame(app: XCUIApplication) -> GestureReferenceFrame {
+    let frame = resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
+    return GestureReferenceFrame(
+      referenceWidth: frame.width,
+      referenceHeight: frame.height
+    )
+  }
+
   func runSeries(count: Int, pauseMs: Double, operation: (Int) -> Void) {
     let total = max(count, 1)
     let pause = max(pauseMs, 0)

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -152,7 +152,7 @@ extension RunnerTests {
 
   func isReadOnlyCommand(_ command: Command) -> Bool {
     switch command.command {
-    case .findText, .snapshot, .screenshot:
+    case .findText, .readText, .snapshot, .screenshot:
       return true
     case .alert:
       let action = (command.action ?? "get").lowercased()

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -10,6 +10,7 @@ enum CommandType: String, Codable {
   case type
   case swipe
   case findText
+  case readText
   case snapshot
   case screenshot
   case back
@@ -19,6 +20,7 @@ enum CommandType: String, Codable {
   case pinch
   case recordStart
   case recordStop
+  case uptime
   case shutdown
 }
 
@@ -71,23 +73,53 @@ struct Response: Codable {
 
 struct DataPayload: Codable {
   let message: String?
+  let text: String?
   let found: Bool?
   let items: [String]?
   let nodes: [SnapshotNode]?
   let truncated: Bool?
+  let gestureStartUptimeMs: Double?
+  let gestureEndUptimeMs: Double?
+  let x: Double?
+  let y: Double?
+  let x2: Double?
+  let y2: Double?
+  let referenceWidth: Double?
+  let referenceHeight: Double?
+  let currentUptimeMs: Double?
 
   init(
     message: String? = nil,
+    text: String? = nil,
     found: Bool? = nil,
     items: [String]? = nil,
     nodes: [SnapshotNode]? = nil,
-    truncated: Bool? = nil
+    truncated: Bool? = nil,
+    gestureStartUptimeMs: Double? = nil,
+    gestureEndUptimeMs: Double? = nil,
+    x: Double? = nil,
+    y: Double? = nil,
+    x2: Double? = nil,
+    y2: Double? = nil,
+    referenceWidth: Double? = nil,
+    referenceHeight: Double? = nil,
+    currentUptimeMs: Double? = nil
   ) {
     self.message = message
+    self.text = text
     self.found = found
     self.items = items
     self.nodes = nodes
     self.truncated = truncated
+    self.gestureStartUptimeMs = gestureStartUptimeMs
+    self.gestureEndUptimeMs = gestureEndUptimeMs
+    self.x = x
+    self.y = y
+    self.x2 = x2
+    self.y2 = y2
+    self.referenceWidth = referenceWidth
+    self.referenceHeight = referenceHeight
+    self.currentUptimeMs = currentUptimeMs
   }
 }
 

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift

@@ -7,13 +7,11 @@ extension RunnerTests {
   final class ScreenRecorder {
     private let outputPath: String
     private let fps: Int32?
-    private let uncappedFrameInterval: TimeInterval = 0.001
-    private var uncappedTimestampTimescale: Int32 {
-      Int32(max(1, Int((1.0 / uncappedFrameInterval).rounded())))
+    private var effectiveFps: Int32 {
+      max(1, fps ?? RunnerTests.defaultRecordingFps)
     }
     private var frameInterval: TimeInterval {
-      guard let fps else { return uncappedFrameInterval }
-      return 1.0 / Double(fps)
+      1.0 / Double(effectiveFps)
     }
     private let queue = DispatchQueue(label: "agent-device.runner.recorder")
     private let lock = NSLock()
@@ -206,7 +204,7 @@ extension RunnerTests {
         recordingStartUptime = nowUptime
       }
       let elapsed = max(0, nowUptime - (recordingStartUptime ?? nowUptime))
-      let timescale = fps ?? uncappedTimestampTimescale
+      let timescale = effectiveFps
       var timestampValue = Int64((elapsed * Double(timescale)).rounded(.down))
       if timestampValue <= lastTimestampValue {
         timestampValue = lastTimestampValue + 1

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -29,6 +29,7 @@ final class RunnerTests: XCTestCase {
   }
 
   static let springboardBundleId = "com.apple.springboard"
+  static let defaultRecordingFps: Int32 = 15
   var listener: NWListener?
   var doneExpectation: XCTestExpectation?
   let app = XCUIApplication()