@devicelab/maestro-runner-darwin-x64 1.1.25

package/drivers/ios/DevicelabIOSRunner/DevicelabIOSRunnerUITests/RunnerTests+Lifecycle.swift

@@ -0,0 +1,351 @@
+import XCTest
+#if canImport(AppKit)
+import AppKit
+#endif
+
+func runnerPngData(for image: RunnerImage) -> Data? {
+#if canImport(UIKit)
+  return image.pngData()
+#elseif canImport(AppKit)
+  guard let cgImage = runnerCGImage(from: image) else { return nil }
+  let bitmap = NSBitmapImageRep(cgImage: cgImage)
+  return bitmap.representation(using: .png, properties: [:])
+#endif
+}
+
+func runnerCGImage(from image: RunnerImage) -> CGImage? {
+#if canImport(UIKit)
+  return image.cgImage
+#elseif canImport(AppKit)
+  return image.cgImage(forProposedRect: nil, context: nil, hints: nil)
+#endif
+}
+
+extension RunnerTests {
+  // MARK: - Recording
+
+  func captureRunnerFrame() -> RunnerImage? {
+    var image: RunnerImage?
+    let capture = {
+      let screenshot = XCUIScreen.main.screenshot()
+      image = screenshot.image
+    }
+    if Thread.isMainThread {
+      capture()
+    } else {
+      DispatchQueue.main.sync(execute: capture)
+    }
+    return image
+  }
+
+  func screenshotRoot(app: XCUIApplication) -> XCUIElement {
+#if os(macOS)
+    let windows = app.windows.allElementsBoundByIndex
+    if let window = windows.first(where: { $0.exists && !$0.frame.isNull && !$0.frame.isEmpty }) {
+      return window
+    }
+#endif
+    return app
+  }
+
+  func stopRecordingIfNeeded() {
+    guard let recorder = activeRecording else { return }
+    do {
+      try recorder.stop()
+    } catch {
+      NSLog("AGENT_DEVICE_RUNNER_RECORD_STOP_FAILED=%@", String(describing: error))
+    }
+    activeRecording = nil
+  }
+
+  func resolveRecordingOutPath(_ requestedOutPath: String) -> String {
+#if os(macOS)
+    if requestedOutPath.hasPrefix("/") {
+      return requestedOutPath
+    }
+#endif
+    let fileName = URL(fileURLWithPath: requestedOutPath).lastPathComponent
+    let fallbackName = "agent-device-recording-\(Int(Date().timeIntervalSince1970 * 1000)).mp4"
+    let safeFileName = fileName.isEmpty ? fallbackName : fileName
+    return (NSTemporaryDirectory() as NSString).appendingPathComponent(safeFileName)
+  }
+
+  // MARK: - Target Activation
+
+  /// The application that is actually on screen, resolved through the
+  /// same private accessibility surface WebDriverAgent uses: active
+  /// applications carry PIDs, and XCUIApplication can be built from a
+  /// PID. Returns nil when the private interface is unavailable so
+  /// callers fall back to the placeholder host app.
+  func frontmostApplication() -> XCUIApplication? {
+    guard let interface = XCUIDevice.shared.value(forKey: "accessibilityInterface") as? NSObject else {
+      NSLog("AGENT_DEVICE_RUNNER_FRONTMOST_MISS step=interface")
+      return nil
+    }
+    guard interface.responds(to: NSSelectorFromString("activeApplications")) else {
+      NSLog("AGENT_DEVICE_RUNNER_FRONTMOST_MISS step=responds class=%@", NSStringFromClass(type(of: interface)))
+      return nil
+    }
+    guard let raw = interface.perform(NSSelectorFromString("activeApplications"))?.takeUnretainedValue() else {
+      NSLog("AGENT_DEVICE_RUNNER_FRONTMOST_MISS step=perform")
+      return nil
+    }
+    guard let elements = raw as? [XCAccessibilityElement], let frontmost = elements.first else {
+      NSLog("AGENT_DEVICE_RUNNER_FRONTMOST_MISS step=cast raw=%@", String(describing: raw))
+      return nil
+    }
+    let pid = frontmost.processIdentifier
+    guard pid > 0 else {
+      NSLog("AGENT_DEVICE_RUNNER_FRONTMOST_MISS step=pid")
+      return nil
+    }
+    guard let bundleId = Self.bundleID(forPID: pid) else {
+      NSLog("AGENT_DEVICE_RUNNER_FRONTMOST_MISS step=bundleID pid=%d", pid)
+      return nil
+    }
+    NSLog("AGENT_DEVICE_RUNNER_FRONTMOST_TARGET pid=%d bundle=%@", pid, bundleId)
+    return XCUIApplication(bundleIdentifier: bundleId)
+  }
+
+  /// Resolves a simulator process to its app bundle identifier: the
+  /// process's executable lives inside its .app bundle on the host
+  /// filesystem, and Info.plist names the identifier.
+  static func bundleID(forPID pid: pid_t) -> String? {
+    var buf = [CChar](repeating: 0, count: 4096)
+    guard proc_pidpath(pid, &buf, UInt32(buf.count)) > 0 else { return nil }
+    let bundleURL = URL(fileURLWithPath: String(cString: buf)).deletingLastPathComponent()
+    guard bundleURL.pathExtension == "app",
+      let plist = NSDictionary(contentsOf: bundleURL.appendingPathComponent("Info.plist")),
+      let bundleId = plist["CFBundleIdentifier"] as? String
+    else { return nil }
+    return bundleId
+  }
+
+  func ensureRunnerHostAppActive(reason: String) {
+    NSLog(
+      "AGENT_DEVICE_RUNNER_HOST_ACTIVATE state=%d reason=%@",
+      app.state.rawValue,
+      reason
+    )
+    if app.state == .unknown || app.state == .notRunning {
+      app.launch()
+    } else if app.state != .runningForeground {
+      app.activate()
+    }
+    currentApp = app
+    currentBundleId = nil
+  }
+
+  func targetNeedsActivation(_ target: XCUIApplication) -> Bool {
+    let state = target.state
+#if os(macOS)
+    if state == .unknown || state == .notRunning || state == .runningBackground {
+      return true
+    }
+#else
+    if state == .unknown || state == .notRunning || state == .runningBackground
+      || state == .runningBackgroundSuspended
+    {
+      return true
+    }
+#endif
+    return false
+  }
+
+  func canUseFastForegroundAppGuard(
+    activeApp: XCUIApplication,
+    requestedBundleId: String?,
+    command: CommandType
+  ) -> Bool {
+    guard let requestedBundleId, currentBundleId == requestedBundleId, currentApp != nil else {
+      return false
+    }
+    guard activeApp.state == .runningForeground else { return false }
+    NSLog(
+      "AGENT_DEVICE_RUNNER_FAST_APP_GUARD command=%@ bundle=%@ state=%d",
+      String(describing: command),
+      requestedBundleId,
+      activeApp.state.rawValue
+    )
+    return true
+  }
+
+  func activateTarget(bundleId: String, reason: String) -> XCUIApplication {
+    let target = XCUIApplication(bundleIdentifier: bundleId)
+    NSLog(
+      "AGENT_DEVICE_RUNNER_ACTIVATE bundle=%@ state=%d reason=%@",
+      bundleId,
+      target.state.rawValue,
+      reason
+    )
+    // activate avoids terminating and relaunching the target app
+    target.activate()
+    currentApp = target
+    currentBundleId = bundleId
+    needsFirstInteractionDelay = true
+    return target
+  }
+
+  func withTemporaryScrollIdleTimeoutIfSupported(
+    _ target: XCUIApplication,
+    operation: () -> Void
+  ) {
+    let setter = NSSelectorFromString("setWaitForIdleTimeout:")
+    let supportsWaitForIdleTimeout = target.responds(to: setter)
+    let previous = supportsWaitForIdleTimeout
+      ? (target.value(forKey: "waitForIdleTimeout") as? NSNumber)
+      : nil
+    if supportsWaitForIdleTimeout {
+      target.setValue(scrollInteractionIdleTimeoutDefault, forKey: "waitForIdleTimeout")
+    }
+    defer {
+      if let previous {
+        target.setValue(previous.doubleValue, forKey: "waitForIdleTimeout")
+      }
+    }
+    performWithQuiescenceSkippedIfSupported(target, operation: operation)
+  }
+
+  // Some apps never report post-gesture quiescence, even after XCTest has synthesized the event.
+  private func performWithQuiescenceSkippedIfSupported(
+    _ target: XCUIApplication,
+    operation: () -> Void
+  ) {
+    let selector = NSSelectorFromString("_performWithInteractionOptions:block:")
+    guard target.responds(to: selector) else {
+      operation()
+      return
+    }
+    typealias PerformWithInteractionOptions = @convention(c) (
+      NSObject,
+      Selector,
+      UInt,
+      @convention(block) () -> Void
+    ) -> Void
+    let implementation = target.method(for: selector)
+    let performWithOptions = unsafeBitCast(
+      implementation,
+      to: PerformWithInteractionOptions.self
+    )
+    let skipPreEventQuiescence = UInt(1)
+    let skipPostEventQuiescence = UInt(2)
+    withoutActuallyEscaping(operation) { escapableOperation in
+      let block: @convention(block) () -> Void = escapableOperation
+      performWithOptions(
+        target,
+        selector,
+        skipPreEventQuiescence | skipPostEventQuiescence,
+        block
+      )
+    }
+  }
+
+  func shouldRetryCommand(_ command: Command) -> Bool {
+    if RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_READONLY_RETRY") {
+      return false
+    }
+    return isReadOnlyCommand(command)
+  }
+
+  func shouldRetryException(_ command: Command, message: String) -> Bool {
+    guard shouldRetryCommand(command) else { return false }
+    let normalized = message.lowercased()
+    if normalized.contains("kaxerrorservernotfound") {
+      return true
+    }
+    if normalized.contains("main thread execution timed out") {
+      return true
+    }
+    if normalized.contains("timed out") && command.command == .snapshot {
+      return true
+    }
+    return false
+  }
+
+  // MARK: - Command Classification
+
+  func isReadOnlyCommand(_ command: Command) -> Bool {
+    switch command.command {
+    case .interactionFrame, .findText, .readText, .snapshot, .screenshot:
+      return true
+    case .alert:
+      let action = (command.action ?? "get").lowercased()
+      return action == "get"
+    default:
+      return false
+    }
+  }
+
+  func shouldRetryResponse(_ response: Response) -> Bool {
+    guard response.ok == false else { return false }
+    guard let message = response.error?.message.lowercased() else { return false }
+    return message.contains("is not available")
+  }
+
+  func isInteractionCommand(_ command: CommandType) -> Bool {
+    switch command {
+    case
+      .tap,
+      .longPress,
+      .drag,
+      .remotePress,
+      .type,
+      .swipe,
+      .back,
+      .backInApp,
+      .backSystem,
+      .rotate,
+      .appSwitcher,
+      .keyboardDismiss,
+      .pinch:
+      return true
+    default:
+      return false
+    }
+  }
+
+  func isRunnerLifecycleCommand(_ command: CommandType) -> Bool {
+    switch command {
+    case .shutdown, .recordStop, .screenshot, .uptime, .idleCheck, .awaitIdle:
+      return true
+    default:
+      return false
+    }
+  }
+
+  // MARK: - Interaction Stabilization
+
+  func applyInteractionStabilizationIfNeeded() {
+    if needsPostSnapshotInteractionDelay {
+      sleepFor(postSnapshotInteractionDelay)
+      needsPostSnapshotInteractionDelay = false
+    }
+    if needsFirstInteractionDelay {
+      sleepFor(firstInteractionAfterActivateDelay)
+      needsFirstInteractionDelay = false
+    }
+  }
+
+  func sleepFor(_ delay: TimeInterval) {
+    guard delay > 0 else { return }
+    // Keep XCTest/UI sources moving during command-local pauses such as delayed typing.
+    if Thread.isMainThread {
+      let deadline = Date().addingTimeInterval(delay)
+      while Date() < deadline {
+        let slice = min(max(deadline.timeIntervalSinceNow, 0), 0.02)
+        if slice <= 0 {
+          break
+        }
+        let handledSource = RunLoop.current.run(
+          mode: .default,
+          before: Date().addingTimeInterval(slice)
+        )
+        if !handledSource {
+          usleep(useconds_t(slice * 1_000_000))
+        }
+      }
+      return
+    }
+    usleep(useconds_t(delay * 1_000_000))
+  }
+}

package/drivers/ios/DevicelabIOSRunner/DevicelabIOSRunnerUITests/RunnerTests+Media.swift

@@ -0,0 +1,81 @@
+import Photos
+import XCTest
+
+extension RunnerTests {
+  /// Adds a photo/video to the device Photos library via PhotoKit.
+  ///
+  /// This runs on-device because writing the Photos database is only possible
+  /// from a signed, add-permitted process — there is no host-side path on real
+  /// devices. On the simulator the host uses `simctl addmedia` instead and
+  /// never reaches this; this path is what gives real iOS devices addMedia
+  /// support (which upstream Maestro lacks).
+  func executeAddMedia(command: Command) throws -> Response {
+    guard let b64 = command.mediaData, let name = command.mediaName else {
+      return Response(ok: false, error: ErrorPayload(message: "addMedia: mediaData and mediaName are required"))
+    }
+    guard let data = Data(base64Encoded: b64) else {
+      return Response(ok: false, error: ErrorPayload(message: "addMedia: invalid base64 data"))
+    }
+
+    let isVideo = (command.mimeType ?? "").hasPrefix("video/")
+    let tmpURL = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent(name)
+    do {
+      try data.write(to: tmpURL)
+    } catch {
+      return Response(ok: false, error: ErrorPayload(message: "addMedia: failed to write temp file: \(error.localizedDescription)"))
+    }
+    defer { try? FileManager.default.removeItem(at: tmpURL) }
+
+    // Ensure add-only authorization. On a real device the first request shows a
+    // SpringBoard alert, which we auto-accept on a background queue while the
+    // request blocks. On the simulator the permission is pre-granted host-side.
+    if #available(iOS 14, *) {
+      if PHPhotoLibrary.authorizationStatus(for: .addOnly) == .notDetermined {
+        acceptPhotoPermissionAlert()
+        let authSem = DispatchSemaphore(value: 0)
+        PHPhotoLibrary.requestAuthorization(for: .addOnly) { _ in authSem.signal() }
+        _ = authSem.wait(timeout: .now() + 15)
+      }
+    }
+
+    var creationError: Error?
+    let sem = DispatchSemaphore(value: 0)
+    PHPhotoLibrary.shared().performChanges({
+      if isVideo {
+        _ = PHAssetCreationRequest.creationRequestForAssetFromVideo(atFileURL: tmpURL)
+      } else {
+        _ = PHAssetCreationRequest.creationRequestForAssetFromImage(atFileURL: tmpURL)
+      }
+    }, completionHandler: { success, error in
+      if !success { creationError = error }
+      sem.signal()
+    })
+    if sem.wait(timeout: .now() + 30) == .timedOut {
+      return Response(ok: false, error: ErrorPayload(message: "addMedia: PhotoKit timed out"))
+    }
+    if let e = creationError {
+      return Response(ok: false, error: ErrorPayload(message: "addMedia: PhotoKit failed: \(e.localizedDescription)"))
+    }
+    return Response(ok: true, data: DataPayload(message: "added media: \(name)"))
+  }
+
+  /// Best-effort auto-accept of the one-time add-only Photos permission alert on
+  /// a real device (no-op on the simulator where permission is pre-granted).
+  private func acceptPhotoPermissionAlert() {
+    DispatchQueue.global(qos: .userInitiated).async {
+      let springboard = XCUIApplication(bundleIdentifier: "com.apple.springboard")
+      let labels = ["Allow Access to All Photos", "Allow", "Add", "OK"]
+      let deadline = Date().addingTimeInterval(12)
+      while Date() < deadline {
+        for label in labels {
+          let button = springboard.buttons[label]
+          if button.exists && button.isHittable {
+            button.tap()
+            return
+          }
+        }
+        Thread.sleep(forTimeInterval: 0.3)
+      }
+    }
+  }
+}

package/drivers/ios/DevicelabIOSRunner/DevicelabIOSRunnerUITests/RunnerTests+Models.swift

@@ -0,0 +1,258 @@
+// MARK: - Wire Models
+
+enum CommandType: String, Codable {
+  case tap
+  // Local extension: walk the XCUI tree, find best liberal match (substring
+  // + case-insensitive + prefer-editable-inputs ranking), tap at its center.
+  // Returns small {ok:true} on hit, or {ok:false, snapshot:[walked nodes]}
+  // on miss so the Go side can fall back to its own snapshot+filter+tap.
+  case tapBySelector
+  case mouseClick
+  case tapSeries
+  case longPress
+  case interactionFrame
+  case drag
+  case dragSeries
+  case remotePress
+  case type
+  case swipe
+  case findText
+  case querySelector
+  case readText
+  case snapshot
+  case screenshot
+  case back
+  case backInApp
+  case backSystem
+  case home
+  case rotate
+  case appSwitcher
+  case keyboardDismiss
+  case alert
+  case pinch
+  case recordStart
+  case recordStop
+  case uptime
+  case shutdown
+  // Local extension (not in upstream agent-device): take two screenshots
+  // in-process and return the fraction of differing pixels. Lets the Go
+  // side's waitForAnimationToEnd avoid two full PNG roundtrips +
+  // re-decode + pixel-walk per iteration.
+  case idleCheck
+  // Local extension: runner-side wait-for-idle loop. Takes timeoutMs +
+  // diff threshold; loops idleCheck internally until settled or timeout.
+  // Avoids HTTP roundtrips per poll iteration.
+  case awaitIdle
+  // Local extension (not in upstream agent-device): add a photo/video to the
+  // device Photos library via PhotoKit. This is the only way to inject media
+  // on a real iOS device (no host-side path exists), so it runs on-device.
+  case addMedia
+  // Local extension: read the device's light/dark appearance.
+  //
+  // The host used to answer this with `simctl ui appearance`, which exists
+  // only for simulators, so dark mode was unavailable on physical devices.
+  // XCUIDevice exposes the same state to a UI test on both (iOS 15+), so
+  // routing it through the runner covers real hardware too.
+  case appearance
+  // Local extension: set the device's light/dark appearance. Takes
+  // `appearance` as "dark" or "light".
+  case setAppearance
+}
+
+struct Command: Codable {
+  let command: CommandType
+  let appBundleId: String?
+  let text: String?
+  let selectorKey: String?
+  let selectorValue: String?
+  let delayMs: Int?
+  let textEntryMode: String?
+  let clearFirst: Bool?
+  let action: String?
+  let x: Double?
+  let y: Double?
+  let button: String?
+  let remoteButton: String?
+  let count: Double?
+  let intervalMs: Double?
+  let doubleTap: Bool?
+  let pauseMs: Double?
+  let pattern: String?
+  let x2: Double?
+  let y2: Double?
+  let durationMs: Double?
+  // moveDurationMs — local extension for drag: how long the movement itself
+  // takes (durationMs is the press-before-move hold). Absent = XCUITest's
+  // default drag velocity, which keeps the pre-existing swipe/scroll wire
+  // shape byte-for-byte compatible.
+  let moveDurationMs: Double?
+  let direction: String?
+  let orientation: String?
+  let scale: Double?
+  let outPath: String?
+  let fps: Int?
+  let quality: Int?
+  let interactiveOnly: Bool?
+  let compact: Bool?
+  let depth: Int?
+  let scope: String?
+  let raw: Bool?
+  let fullscreen: Bool?
+  let mediaName: String?
+  let mimeType: String?
+  let mediaData: String?  // base64-encoded file bytes (addMedia)
+  let appearance: String?  // "dark" or "light" (setAppearance)
+}
+
+struct Response: Codable {
+  let ok: Bool
+  let data: DataPayload?
+  let error: ErrorPayload?
+
+  init(ok: Bool, data: DataPayload? = nil, error: ErrorPayload? = nil) {
+    self.ok = ok
+    self.data = data
+    self.error = error
+  }
+}
+
+struct DataPayload: Codable {
+  let message: String?
+  let text: String?
+  let found: Bool?
+  let items: [String]?
+  let nodes: [SnapshotNode]?
+  let truncated: Bool?
+  let gestureStartUptimeMs: Double?
+  let gestureEndUptimeMs: Double?
+  let x: Double?
+  let y: Double?
+  let x2: Double?
+  let y2: Double?
+  let referenceWidth: Double?
+  let referenceHeight: Double?
+  let currentUptimeMs: Double?
+  let visible: Bool?
+  let wasVisible: Bool?
+  let dismissed: Bool?
+  let orientation: String?
+  // Local extension: identifier of the matched element when applicable
+  // (tapBySelector uses this so the Go side can populate
+  // lastTappedIdentifier for the next inputText to use as a typing hint).
+  let identifier: String?
+  // Local extension (not in upstream agent-device): inline PNG bytes
+  // (base64-encoded) for screenshot responses. Upstream writes a file
+  // and returns the path in `message`; for maestro-runner we need the
+  // bytes back on the host without an extra simctl roundtrip.
+  let pngBase64: String?
+  // Local extension: fraction of differing pixels from idleCheck.
+  let diffFraction: Double?
+  // Local extension: "dark" or "light" from the appearance command.
+  let appearance: String?
+  // Local extension (DeviceDeck): the target app's XCUIApplication.state
+  // at snapshot time — "runningForeground", "runningBackground",
+  // "runningBackgroundSuspended", "notRunning" or "unknown". A snapshot
+  // is not an interaction command, so it is never auto-activated first;
+  // this is the app's true state, which no signal in the node tree
+  // reports reliably (hittable oscillates while a screen backgrounds).
+  let appState: String?
+
+  init(
+    message: String? = nil,
+    text: String? = nil,
+    found: Bool? = nil,
+    items: [String]? = nil,
+    nodes: [SnapshotNode]? = nil,
+    truncated: Bool? = nil,
+    gestureStartUptimeMs: Double? = nil,
+    gestureEndUptimeMs: Double? = nil,
+    x: Double? = nil,
+    y: Double? = nil,
+    x2: Double? = nil,
+    y2: Double? = nil,
+    referenceWidth: Double? = nil,
+    referenceHeight: Double? = nil,
+    currentUptimeMs: Double? = nil,
+    visible: Bool? = nil,
+    wasVisible: Bool? = nil,
+    dismissed: Bool? = nil,
+    orientation: String? = nil,
+    pngBase64: String? = nil,
+    diffFraction: Double? = nil,
+    identifier: String? = nil,
+    appearance: String? = nil,
+    appState: String? = nil
+  ) {
+    self.message = message
+    self.text = text
+    self.found = found
+    self.items = items
+    self.nodes = nodes
+    self.truncated = truncated
+    self.gestureStartUptimeMs = gestureStartUptimeMs
+    self.gestureEndUptimeMs = gestureEndUptimeMs
+    self.x = x
+    self.y = y
+    self.x2 = x2
+    self.y2 = y2
+    self.referenceWidth = referenceWidth
+    self.referenceHeight = referenceHeight
+    self.currentUptimeMs = currentUptimeMs
+    self.visible = visible
+    self.wasVisible = wasVisible
+    self.dismissed = dismissed
+    self.orientation = orientation
+    self.pngBase64 = pngBase64
+    self.diffFraction = diffFraction
+    self.identifier = identifier
+    self.appearance = appearance
+    self.appState = appState
+  }
+}
+
+struct ErrorPayload: Codable {
+  let code: String?
+  let message: String
+
+  init(code: String? = nil, message: String) {
+    self.code = code
+    self.message = message
+  }
+}
+
+struct SnapshotRect: Codable {
+  let x: Double
+  let y: Double
+  let width: Double
+  let height: Double
+}
+
+struct SnapshotNode: Codable {
+  let index: Int
+  let type: String
+  let label: String?
+  let identifier: String?
+  let value: String?
+  // Local extension (not in upstream agent-device): placeholder text for
+  // editable fields. maestro-runner YAML flows commonly match on
+  // placeholder text (e.g. RN TextInput placeholder="Username"); without
+  // this field, that text isn't visible in the snapshot.
+  let placeholderValue: String?
+  let rect: SnapshotRect
+  let enabled: Bool
+  let focused: Bool?
+  let selected: Bool?
+  let hittable: Bool
+  let depth: Int
+  let parentIndex: Int?
+  let hiddenContentAbove: Bool?
+  let hiddenContentBelow: Bool?
+}
+
+struct SnapshotOptions {
+  let interactiveOnly: Bool
+  let compact: Bool
+  let depth: Int?
+  let scope: String?
+  let raw: Bool
+}