agent-device 0.7.4 → 0.7.5

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift

@@ -0,0 +1,174 @@
+import XCTest
+import UIKit
+
+extension RunnerTests {
+  // MARK: - Recording
+
+  func captureRunnerFrame() -> UIImage? {
+    var image: UIImage?
+    let capture = {
+      let screenshot = XCUIScreen.main.screenshot()
+      image = screenshot.image
+    }
+    if Thread.isMainThread {
+      capture()
+    } else {
+      DispatchQueue.main.sync(execute: capture)
+    }
+    return image
+  }
+
+  func stopRecordingIfNeeded() {
+    guard let recorder = activeRecording else { return }
+    do {
+      try recorder.stop()
+    } catch {
+      NSLog("AGENT_DEVICE_RUNNER_RECORD_STOP_FAILED=%@", String(describing: error))
+    }
+    activeRecording = nil
+  }
+
+  func resolveRecordingOutPath(_ requestedOutPath: String) -> String {
+    let fileName = URL(fileURLWithPath: requestedOutPath).lastPathComponent
+    let fallbackName = "agent-device-recording-\(Int(Date().timeIntervalSince1970 * 1000)).mp4"
+    let safeFileName = fileName.isEmpty ? fallbackName : fileName
+    return (NSTemporaryDirectory() as NSString).appendingPathComponent(safeFileName)
+  }
+
+  // MARK: - Target Activation
+
+  func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
+    switch target.state {
+    case .unknown, .notRunning, .runningBackground, .runningBackgroundSuspended:
+      return true
+    default:
+      return false
+    }
+  }
+
+  func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
+    let target = XCUIApplication(bundleIdentifier: bundleId)
+    NSLog(
+      "AGENT_DEVICE_RUNNER_ACTIVATE bundle=%@ state=%d reason=%@",
+      bundleId,
+      target.state.rawValue,
+      reason
+    )
+    // activate avoids terminating and relaunching the target app
+    target.activate()
+    currentApp = target
+    currentBundleId = bundleId
+    needsFirstInteractionDelay = true
+    return target
+  }
+
+  func withTemporaryScrollIdleTimeoutIfSupported(
+    _ target: XCUIApplication,
+    operation: () -> Void
+  ) {
+    let setter = NSSelectorFromString("setWaitForIdleTimeout:")
+    guard target.responds(to: setter) else {
+      operation()
+      return
+    }
+    let previous = target.value(forKey: "waitForIdleTimeout") as? NSNumber
+    target.setValue(resolveScrollInteractionIdleTimeout(), forKey: "waitForIdleTimeout")
+    defer {
+      if let previous {
+        target.setValue(previous.doubleValue, forKey: "waitForIdleTimeout")
+      }
+    }
+    operation()
+  }
+
+  private func resolveScrollInteractionIdleTimeout() -> TimeInterval {
+    guard
+      let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_IOS_INTERACTION_IDLE_TIMEOUT"],
+      !raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
+    else {
+      return scrollInteractionIdleTimeoutDefault
+    }
+    guard let parsed = Double(raw), parsed >= 0 else {
+      return scrollInteractionIdleTimeoutDefault
+    }
+    return min(parsed, 30)
+  }
+
+  func shouldRetryCommand(_ command: Command) -> Bool {
+    if RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_READONLY_RETRY") {
+      return false
+    }
+    return isReadOnlyCommand(command)
+  }
+
+  func shouldRetryException(_ command: Command, message: String) -> Bool {
+    guard shouldRetryCommand(command) else { return false }
+    let normalized = message.lowercased()
+    if normalized.contains("kaxerrorservernotfound") {
+      return true
+    }
+    if normalized.contains("main thread execution timed out") {
+      return true
+    }
+    if normalized.contains("timed out") && command.command == .snapshot {
+      return true
+    }
+    return false
+  }
+
+  // MARK: - Command Classification
+
+  func isReadOnlyCommand(_ command: Command) -> Bool {
+    switch command.command {
+    case .findText, .snapshot, .screenshot:
+      return true
+    case .alert:
+      let action = (command.action ?? "get").lowercased()
+      return action == "get"
+    default:
+      return false
+    }
+  }
+
+  func shouldRetryResponse(_ response: Response) -> Bool {
+    guard response.ok == false else { return false }
+    guard let message = response.error?.message.lowercased() else { return false }
+    return message.contains("is not available")
+  }
+
+  func isInteractionCommand(_ command: CommandType) -> Bool {
+    switch command {
+    case .tap, .longPress, .drag, .type, .swipe, .back, .appSwitcher, .pinch:
+      return true
+    default:
+      return false
+    }
+  }
+
+  func isRunnerLifecycleCommand(_ command: CommandType) -> Bool {
+    switch command {
+    case .shutdown, .recordStop, .screenshot:
+      return true
+    default:
+      return false
+    }
+  }
+
+  // MARK: - Interaction Stabilization
+
+  func applyInteractionStabilizationIfNeeded() {
+    if needsPostSnapshotInteractionDelay {
+      sleepFor(postSnapshotInteractionDelay)
+      needsPostSnapshotInteractionDelay = false
+    }
+    if needsFirstInteractionDelay {
+      sleepFor(firstInteractionAfterActivateDelay)
+      needsFirstInteractionDelay = false
+    }
+  }
+
+  func sleepFor(_ delay: TimeInterval) {
+    guard delay > 0 else { return }
+    usleep(useconds_t(delay * 1_000_000))
+  }
+}

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -0,0 +1,121 @@
+// MARK: - Wire Models
+
+enum CommandType: String, Codable {
+  case tap
+  case tapSeries
+  case longPress
+  case drag
+  case dragSeries
+  case type
+  case swipe
+  case findText
+  case snapshot
+  case screenshot
+  case back
+  case home
+  case appSwitcher
+  case alert
+  case pinch
+  case recordStart
+  case recordStop
+  case shutdown
+}
+
+enum SwipeDirection: String, Codable {
+  case up
+  case down
+  case left
+  case right
+}
+
+struct Command: Codable {
+  let command: CommandType
+  let appBundleId: String?
+  let text: String?
+  let clearFirst: Bool?
+  let action: String?
+  let x: Double?
+  let y: Double?
+  let count: Double?
+  let intervalMs: Double?
+  let doubleTap: Bool?
+  let pauseMs: Double?
+  let pattern: String?
+  let x2: Double?
+  let y2: Double?
+  let durationMs: Double?
+  let direction: SwipeDirection?
+  let scale: Double?
+  let outPath: String?
+  let fps: Int?
+  let interactiveOnly: Bool?
+  let compact: Bool?
+  let depth: Int?
+  let scope: String?
+  let raw: Bool?
+}
+
+struct Response: Codable {
+  let ok: Bool
+  let data: DataPayload?
+  let error: ErrorPayload?
+
+  init(ok: Bool, data: DataPayload? = nil, error: ErrorPayload? = nil) {
+    self.ok = ok
+    self.data = data
+    self.error = error
+  }
+}
+
+struct DataPayload: Codable {
+  let message: String?
+  let found: Bool?
+  let items: [String]?
+  let nodes: [SnapshotNode]?
+  let truncated: Bool?
+
+  init(
+    message: String? = nil,
+    found: Bool? = nil,
+    items: [String]? = nil,
+    nodes: [SnapshotNode]? = nil,
+    truncated: Bool? = nil
+  ) {
+    self.message = message
+    self.found = found
+    self.items = items
+    self.nodes = nodes
+    self.truncated = truncated
+  }
+}
+
+struct ErrorPayload: Codable {
+  let message: String
+}
+
+struct SnapshotRect: Codable {
+  let x: Double
+  let y: Double
+  let width: Double
+  let height: Double
+}
+
+struct SnapshotNode: Codable {
+  let index: Int
+  let type: String
+  let label: String?
+  let identifier: String?
+  let value: String?
+  let rect: SnapshotRect
+  let enabled: Bool
+  let hittable: Bool
+  let depth: Int
+}
+
+struct SnapshotOptions {
+  let interactiveOnly: Bool
+  let compact: Bool
+  let depth: Int?
+  let scope: String?
+  let raw: Bool
+}

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift

@@ -0,0 +1,263 @@
+import AVFoundation
+import CoreVideo
+import UIKit
+
+extension RunnerTests {
+  // MARK: - Screen Recorder
+
+  final class ScreenRecorder {
+    private let outputPath: String
+    private let fps: Int32?
+    private let uncappedFrameInterval: TimeInterval = 0.001
+    private var uncappedTimestampTimescale: Int32 {
+      Int32(max(1, Int((1.0 / uncappedFrameInterval).rounded())))
+    }
+    private var frameInterval: TimeInterval {
+      guard let fps else { return uncappedFrameInterval }
+      return 1.0 / Double(fps)
+    }
+    private let queue = DispatchQueue(label: "agent-device.runner.recorder")
+    private let lock = NSLock()
+    private var assetWriter: AVAssetWriter?
+    private var writerInput: AVAssetWriterInput?
+    private var pixelBufferAdaptor: AVAssetWriterInputPixelBufferAdaptor?
+    private var timer: DispatchSourceTimer?
+    private var recordingStartUptime: TimeInterval?
+    private var lastTimestampValue: Int64 = -1
+    private var isStopping = false
+    private var startedSession = false
+    private var startError: Error?
+
+    init(outputPath: String, fps: Int32?) {
+      self.outputPath = outputPath
+      self.fps = fps
+    }
+
+    func start(captureFrame: @escaping () -> UIImage?) throws {
+      let url = URL(fileURLWithPath: outputPath)
+      let directory = url.deletingLastPathComponent()
+      try FileManager.default.createDirectory(
+        at: directory,
+        withIntermediateDirectories: true,
+        attributes: nil
+      )
+      if FileManager.default.fileExists(atPath: outputPath) {
+        try FileManager.default.removeItem(atPath: outputPath)
+      }
+
+      var dimensions: CGSize = .zero
+      var bootstrapImage: UIImage?
+      let bootstrapDeadline = Date().addingTimeInterval(2.0)
+      while Date() < bootstrapDeadline {
+        if let image = captureFrame(), let cgImage = image.cgImage {
+          bootstrapImage = image
+          dimensions = CGSize(width: cgImage.width, height: cgImage.height)
+          break
+        }
+        Thread.sleep(forTimeInterval: 0.05)
+      }
+      guard dimensions.width > 0, dimensions.height > 0 else {
+        throw NSError(
+          domain: "AgentDeviceRunner.Record",
+          code: 1,
+          userInfo: [NSLocalizedDescriptionKey: "failed to capture initial frame"]
+        )
+      }
+
+      let writer = try AVAssetWriter(outputURL: url, fileType: .mp4)
+      let outputSettings: [String: Any] = [
+        AVVideoCodecKey: AVVideoCodecType.h264,
+        AVVideoWidthKey: Int(dimensions.width),
+        AVVideoHeightKey: Int(dimensions.height),
+      ]
+      let input = AVAssetWriterInput(mediaType: .video, outputSettings: outputSettings)
+      input.expectsMediaDataInRealTime = true
+      let attributes: [String: Any] = [
+        kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32ARGB,
+        kCVPixelBufferWidthKey as String: Int(dimensions.width),
+        kCVPixelBufferHeightKey as String: Int(dimensions.height),
+      ]
+      let adaptor = AVAssetWriterInputPixelBufferAdaptor(
+        assetWriterInput: input,
+        sourcePixelBufferAttributes: attributes
+      )
+      guard writer.canAdd(input) else {
+        throw NSError(
+          domain: "AgentDeviceRunner.Record",
+          code: 2,
+          userInfo: [NSLocalizedDescriptionKey: "failed to add video input"]
+        )
+      }
+      writer.add(input)
+      guard writer.startWriting() else {
+        throw writer.error ?? NSError(
+          domain: "AgentDeviceRunner.Record",
+          code: 3,
+          userInfo: [NSLocalizedDescriptionKey: "failed to start writing"]
+        )
+      }
+
+      lock.lock()
+      assetWriter = writer
+      writerInput = input
+      pixelBufferAdaptor = adaptor
+      recordingStartUptime = nil
+      lastTimestampValue = -1
+      isStopping = false
+      startedSession = false
+      startError = nil
+      lock.unlock()
+
+      if let firstImage = bootstrapImage {
+        append(image: firstImage)
+      }
+
+      let timer = DispatchSource.makeTimerSource(queue: queue)
+      timer.schedule(deadline: .now() + frameInterval, repeating: frameInterval)
+      timer.setEventHandler { [weak self] in
+        guard let self else { return }
+        if self.shouldStop() { return }
+        guard let image = captureFrame() else { return }
+        self.append(image: image)
+      }
+      self.timer = timer
+      timer.resume()
+    }
+
+    func stop() throws {
+      var writer: AVAssetWriter?
+      var input: AVAssetWriterInput?
+      var appendError: Error?
+      lock.lock()
+      if isStopping {
+        lock.unlock()
+        return
+      }
+      isStopping = true
+      let activeTimer = timer
+      timer = nil
+      writer = assetWriter
+      input = writerInput
+      appendError = startError
+      lock.unlock()
+
+      activeTimer?.cancel()
+      input?.markAsFinished()
+      guard let writer else { return }
+
+      let semaphore = DispatchSemaphore(value: 0)
+      writer.finishWriting {
+        semaphore.signal()
+      }
+      var stopFailure: Error?
+      let waitResult = semaphore.wait(timeout: .now() + 10)
+      if waitResult == .timedOut {
+        writer.cancelWriting()
+        stopFailure = NSError(
+          domain: "AgentDeviceRunner.Record",
+          code: 6,
+          userInfo: [NSLocalizedDescriptionKey: "recording finalization timed out"]
+        )
+      } else if let appendError {
+        stopFailure = appendError
+      } else if writer.status == .failed {
+        stopFailure = writer.error ?? NSError(
+          domain: "AgentDeviceRunner.Record",
+          code: 4,
+          userInfo: [NSLocalizedDescriptionKey: "failed to finalize recording"]
+        )
+      }
+
+      lock.lock()
+      assetWriter = nil
+      writerInput = nil
+      pixelBufferAdaptor = nil
+      recordingStartUptime = nil
+      lastTimestampValue = -1
+      startedSession = false
+      startError = nil
+      lock.unlock()
+
+      if let stopFailure {
+        throw stopFailure
+      }
+    }
+
+    private func append(image: UIImage) {
+      guard let cgImage = image.cgImage else { return }
+      lock.lock()
+      defer { lock.unlock() }
+      if isStopping { return }
+      if startError != nil { return }
+      guard
+        let writer = assetWriter,
+        let input = writerInput,
+        let adaptor = pixelBufferAdaptor
+      else {
+        return
+      }
+      if !startedSession {
+        writer.startSession(atSourceTime: .zero)
+        startedSession = true
+      }
+      guard input.isReadyForMoreMediaData else { return }
+      guard let pixelBuffer = makePixelBuffer(from: cgImage) else { return }
+      let nowUptime = ProcessInfo.processInfo.systemUptime
+      if recordingStartUptime == nil {
+        recordingStartUptime = nowUptime
+      }
+      let elapsed = max(0, nowUptime - (recordingStartUptime ?? nowUptime))
+      let timescale = fps ?? uncappedTimestampTimescale
+      var timestampValue = Int64((elapsed * Double(timescale)).rounded(.down))
+      if timestampValue <= lastTimestampValue {
+        timestampValue = lastTimestampValue + 1
+      }
+      let timestamp = CMTime(value: timestampValue, timescale: timescale)
+      if !adaptor.append(pixelBuffer, withPresentationTime: timestamp) {
+        startError = writer.error ?? NSError(
+          domain: "AgentDeviceRunner.Record",
+          code: 5,
+          userInfo: [NSLocalizedDescriptionKey: "failed to append frame"]
+        )
+        return
+      }
+      lastTimestampValue = timestampValue
+    }
+
+    private func shouldStop() -> Bool {
+      lock.lock()
+      defer { lock.unlock() }
+      return isStopping
+    }
+
+    private func makePixelBuffer(from image: CGImage) -> CVPixelBuffer? {
+      guard let adaptor = pixelBufferAdaptor else { return nil }
+      var pixelBuffer: CVPixelBuffer?
+      guard let pool = adaptor.pixelBufferPool else { return nil }
+      let status = CVPixelBufferPoolCreatePixelBuffer(
+        nil,
+        pool,
+        &pixelBuffer
+      )
+      guard status == kCVReturnSuccess, let pixelBuffer else { return nil }
+
+      CVPixelBufferLockBaseAddress(pixelBuffer, [])
+      defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) }
+      guard
+        let context = CGContext(
+          data: CVPixelBufferGetBaseAddress(pixelBuffer),
+          width: image.width,
+          height: image.height,
+          bitsPerComponent: 8,
+          bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer),
+          space: CGColorSpaceCreateDeviceRGB(),
+          bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue
+        )
+      else {
+        return nil
+      }
+      context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))
+      return pixelBuffer
+    }
+  }
+}