agent-device 0.20.10 → 0.21.0

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift

@@ -0,0 +1,158 @@
+import XCTest
+
+// Text-entry readiness: whether the target is actually able to receive text yet. The waits, the
+// keyboard signals they read, and the focus corroboration that ends the hardware-keyboard window.
+// RunnerTests+TextEntryFocus.swift is the only caller of the two non-private entries here; nothing
+// in this file reaches back into it.
+extension RunnerTests {
+  func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
+#if os(iOS)
+    // iOS focus predicates can return stale or misleading text-input matches
+    // under XCUITest, so text entry readiness is driven by tap/keyboard state.
+    return nil
+#else
+    return safely("FOCUSED_INPUT_QUERY") {
+      let candidates = app
+        .descendants(matching: .any)
+        .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
+        .allElementsBoundByIndex
+      for candidate in candidates where candidate.exists {
+        switch candidate.elementType {
+        case .textField, .secureTextField, .searchField, .textView:
+          return candidate
+        default:
+          continue
+        }
+      }
+      return nil
+    }
+#endif
+  }
+
+  func waitForTextEntryReadiness(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    timeout: TimeInterval = TextEntryTiming.readinessTimeout
+  ) -> XCUIElement? {
+#if os(iOS)
+    var latest = resolveTextEntryElement(app: app, target: target)
+    let keyboardVisibleAtEntry = isKeyboardVisible(app: app)
+    let deadline = Date().addingTimeInterval(timeout)
+    var hardwareKeyboardFallback = Date().addingTimeInterval(
+      min(TextEntryTiming.hardwareKeyboardFallbackTimeout, timeout)
+    )
+    var sawSoftwareKeyboard = false
+    while Date() < deadline {
+      if let focused = focusedTextInput(app: app) {
+        latest = focused
+        if isKeyboardVisible(app: app) {
+          return focused
+        }
+      }
+      // Fast-path on a keyboard hidden->visible transition: our tapped field gained focus, so
+      // return immediately instead of burning the full readinessTimeout (warmup-first-char echo
+      // + post-type verify/repair remain as drop safety nets). When the keyboard was ALREADY up
+      // (back-to-back fills), this isn't a focus signal — fall through to the settle/timeout so
+      // text isn't sent to the previously-focused field.
+      if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) {
+        return latest
+      }
+      sawSoftwareKeyboard = sawSoftwareKeyboard || keyboardElementExists(app: app)
+      // A responder that takes no software keyboard (hardware keyboard connected, or a custom
+      // `inputView`) would otherwise burn the whole readinessTimeout waiting for one that is never
+      // coming. Leaving that window a bare wall-clock guess made readiness a function of ambient
+      // simulator state (#1874): on a loaded host the keyboard is merely late, and returning here
+      // handed the caller an element that had not taken focus yet. Ask the target itself instead,
+      // and re-arm rather than re-asking every poll — the query is cheap, not free.
+      if !sawSoftwareKeyboard, Date() >= hardwareKeyboardFallback, let candidate = latest {
+        if keyboardFocusConfirmed(app: app, element: candidate) {
+          return candidate
+        }
+        hardwareKeyboardFallback = Date().addingTimeInterval(
+          TextEntryTiming.hardwareKeyboardFallbackTimeout
+        )
+      }
+      sleepFor(TextEntryTiming.pollInterval)
+    }
+    return focusedTextInput(app: app) ?? latest
+#else
+    return resolveTextEntryElement(app: app, target: target)
+#endif
+  }
+
+  func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) {
+#if os(iOS)
+    switch element.elementType {
+    case .textField, .secureTextField, .searchField, .textView:
+      if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil {
+        return
+      }
+      let frame = element.frame
+      if !frame.isEmpty {
+        _ = tapAt(app: app, x: frame.midX, y: frame.midY)
+        _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout)
+      }
+    default:
+      return
+    }
+#endif
+  }
+
+  private func waitForFocusedTextInput(app: XCUIApplication, timeout: TimeInterval) -> XCUIElement? {
+    let deadline = Date().addingTimeInterval(timeout)
+    while Date() < deadline {
+      if let focused = focusedTextInput(app: app) {
+        return focused
+      }
+      sleepFor(TextEntryTiming.pollInterval)
+    }
+    return focusedTextInput(app: app)
+  }
+
+  func keyboardBecameVisible(app: XCUIApplication, wasVisibleAtEntry: Bool) -> Bool {
+    return !wasVisibleAtEntry && isKeyboardVisible(app: app)
+  }
+
+  /// Positive evidence that this element — the one readiness is about to hand its caller — holds
+  /// keyboard focus. Readable even with no software keyboard on screen, which is what lets the
+  /// hardware-keyboard fallback stop guessing from a wall clock.
+  ///
+  /// This is the same app-wide predicate `focusedTextInput` refuses to trust on iOS, used the
+  /// other way round. There, the query PICKS the target, so a stale or unrelated match becomes
+  /// the field that gets typed into. Here the target is already chosen and the query only
+  /// corroborates it: at most one element holds keyboard focus, so an answer that is not this
+  /// element is a refusal, not a substitution. Every way of being wrong therefore ends as `false`
+  /// and costs the remaining readiness timeout — exactly what the wait would spend with no
+  /// fallback at all.
+  func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool {
+#if os(iOS)
+    return safely("TEXT_ENTRY_FOCUS_CONFIRMED", false) {
+      // An element that no longer resolves reads as identifier "" and frame `.zero`, which would
+      // match any focused element that also reports an empty frame. Require a real frame first,
+      // so a dead handle cannot corroborate anything.
+      let frame = element.frame
+      guard !frame.isEmpty else {
+        return false
+      }
+      let focused = app
+        .descendants(matching: .any)
+        .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
+        .firstMatch
+      guard focused.exists else {
+        return false
+      }
+      return focused.identifier == element.identifier && focused.frame == frame
+    }
+#else
+    return false
+#endif
+  }
+
+  private func keyboardElementExists(app: XCUIApplication) -> Bool {
+#if os(iOS)
+    return safely("KEYBOARD_EXISTS", false) { app.keyboards.firstMatch.exists }
+#else
+    return false
+#endif
+  }
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift

@@ -15,6 +15,36 @@ extension RunnerTests {
   ) -> TextEntryResult {
     let totalStartedAt = Date()
     guard !text.isEmpty else {
+      // Replacing a field WITH the empty string is the clear-field primitive (#2063): the typing
+      // is vacuous but the clear is not. Returning here unconditionally left the old value in
+      // place while the response reported "typed". Append (`type`) and unrepaired entry stay
+      // no-ops, which is what an empty payload means for them.
+      if repairMode == .replacement {
+        guard let clearTarget = resolveTextEntryElement(app: app, target: target) else {
+          // No resolvable input means no clear happened — including the synthesized
+          // first-responder route, whose target carries no element. Success here would claim a
+          // clear the runner never performed.
+          logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode)
+          return TextEntryResult(
+            verified: nil,
+            repaired: false,
+            expectedText: "",
+            observedText: nil,
+            failure: .notFocused
+          )
+        }
+        clearTextInput(clearTarget)
+        // nil means the value is unreadable (secure fields), not "not empty": leave `verified`
+        // nil there rather than reporting a mismatch the runner cannot actually observe.
+        let observed = editableTextValue(for: clearTarget, treatingPlaceholderAsEmpty: true)
+        logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode)
+        return TextEntryResult(
+          verified: observed.map { $0.isEmpty },
+          repaired: false,
+          expectedText: "",
+          observedText: observed
+        )
+      }
       logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode)
       return TextEntryResult(verified: true, repaired: false, expectedText: "", observedText: "")
     }
@@ -104,11 +134,11 @@ extension RunnerTests {
       }
     }
 
-    func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, dispatched: Bool, failure: TextEntryFailure?) {
+    func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, failure: TextEntryFailure?) {
 #if os(iOS)
       if shouldUseSynthesizedFirstResponderType {
         guard let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) else {
-          return (nil, false, .notFocused)
+          return (nil, .notFocused)
         }
         textEntryRoute = "synthesized-first-responder"
         NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
@@ -117,7 +147,7 @@ extension RunnerTests {
         case .continueTyping:
           // No refresh point: like the tap-witness target itself, the commit wait must observe
           // only the element the tap selected, never rediscover a different field.
-          awaitSynthesizedFirstResponderCommit(
+          let commit = awaitSynthesizedFirstResponderCommit(
             app: app,
             target: TextEntryTarget(
               element: currentTarget,
@@ -128,9 +158,12 @@ extension RunnerTests {
             textBefore: textBefore,
             typedText: value
           )
-          return (currentTarget, true, nil)
+          // The characters were posted, so the element stays on the tuple for the caller's
+          // bookkeeping; the failure is what makes the command refuse. Reporting ok here is the
+          // defect this route had — the deadline was indistinguishable from a commit.
+          return (currentTarget, Self.textEntryFailure(forCommitOutcome: commit))
         case .fallback:
-          return (nil, false, .synthesisUnavailable)
+          return (nil, .synthesisUnavailable)
         case .raise(let message):
           NSException(
             name: NSExceptionName.internalInconsistencyException,
@@ -142,7 +175,7 @@ extension RunnerTests {
       if let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) {
         textEntryRoute = "xctest-element"
         currentTarget.typeText(value)
-        return (currentTarget, true, nil)
+        return (currentTarget, nil)
       } else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) {
 #if os(iOS)
         textEntryRoute = "synthesized-first-responder"
@@ -167,9 +200,9 @@ extension RunnerTests {
 #else
         app.typeText(value)
 #endif
-        return (resolveTextEntryElement(app: app, target: activeTarget), true, nil)
+        return (resolveTextEntryElement(app: app, target: activeTarget), nil)
       }
-      return (nil, false, .notFocused)
+      return (nil, .notFocused)
     }
 
     func dispatchFailureResult(_ failure: TextEntryFailure) -> TextEntryResult {
@@ -203,8 +236,8 @@ extension RunnerTests {
       let delayedTypeStartedAt = Date()
       for (index, character) in characters.enumerated() {
         let dispatch = typeIntoCurrentTarget(String(character))
-        guard dispatch.dispatched else {
-          return dispatchFailureResult(dispatch.failure ?? .notFocused)
+        if let failure = dispatch.failure {
+          return dispatchFailureResult(failure)
         }
         typedTarget = dispatch.element ?? typedTarget
         if index + 1 < characters.count {
@@ -252,8 +285,8 @@ extension RunnerTests {
       let firstCharacter = String(characters[0])
       let firstStartedAt = Date()
       let firstDispatch = typeIntoCurrentTarget(firstCharacter)
-      guard firstDispatch.dispatched else {
-        return dispatchFailureResult(firstDispatch.failure ?? .notFocused)
+      if let failure = firstDispatch.failure {
+        return dispatchFailureResult(failure)
       }
       var firstTypedTarget = firstDispatch.element
       logTextEntryPhase(
@@ -281,8 +314,8 @@ extension RunnerTests {
       let remainingText = String(characters.dropFirst())
       let remainingStartedAt = Date()
       let remainingDispatch = typeIntoCurrentTarget(remainingText)
-      guard remainingDispatch.dispatched else {
-        return dispatchFailureResult(remainingDispatch.failure ?? .notFocused)
+      if let failure = remainingDispatch.failure {
+        return dispatchFailureResult(failure)
       }
       firstTypedTarget = remainingDispatch.element ?? firstTypedTarget
       logTextEntryPhase(
@@ -296,8 +329,8 @@ extension RunnerTests {
     } else {
       let typeStartedAt = Date()
       let dispatch = typeIntoCurrentTarget(text)
-      guard dispatch.dispatched else {
-        return dispatchFailureResult(dispatch.failure ?? .notFocused)
+      if let failure = dispatch.failure {
+        return dispatchFailureResult(failure)
       }
       typedTarget = dispatch.element
       logTextEntryPhase(

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift

@@ -190,7 +190,7 @@ extension RunnerTests {
     command: Command,
     completion: @escaping ((data: Data, shouldFinish: Bool)) -> Void
   ) -> Bool {
-    guard let commandId = normalizedInFlightCommandId(command.commandId) else { return false }
+    guard let commandId = command.commandId?.trimmedNonEmpty else { return false }
     inFlightCommandLock.lock()
     if inFlightCommandIds.contains(commandId) {
       inFlightCommandWaiters[commandId, default: []].append(completion)
@@ -213,7 +213,7 @@ extension RunnerTests {
     completion: ((data: Data, shouldFinish: Bool)) -> Void
   ) {
     var waiters: [((data: Data, shouldFinish: Bool)) -> Void] = []
-    if let commandId = normalizedInFlightCommandId(command.commandId) {
+    if let commandId = command.commandId?.trimmedNonEmpty {
       inFlightCommandLock.lock()
       inFlightCommandIds.remove(commandId)
       waiters = inFlightCommandWaiters.removeValue(forKey: commandId) ?? []
@@ -225,15 +225,6 @@ extension RunnerTests {
     }
   }
 
-  private func normalizedInFlightCommandId(_ commandId: String?) -> String? {
-    guard let trimmed = commandId?.trimmingCharacters(in: .whitespacesAndNewlines),
-      !trimmed.isEmpty
-    else {
-      return nil
-    }
-    return trimmed
-  }
-
 
   // MARK: - Response Encoding
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift

@@ -7,7 +7,7 @@ enum RunnerInteractionOutcome {
   case unsupported(message: String, hint: String?)
 }
 
-enum TvRemoteButton {
+enum TvRemoteButton: String {
   case select
   case menu
   case home
@@ -18,6 +18,7 @@ enum TvRemoteButton {
 }
 
 extension RunnerTests {
+
   func resolveTvRemoteDoublePressDelay() -> TimeInterval {
     guard
       let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_TV_REMOTE_DOUBLE_PRESS_DELAY_MS"],
@@ -47,24 +48,8 @@ extension RunnerTests {
   }
 
   func tvRemoteButton(from raw: String?) -> TvRemoteButton? {
-    switch raw?.lowercased() {
-    case "select":
-      return .select
-    case "menu":
-      return .menu
-    case "home":
-      return .home
-    case "up":
-      return .up
-    case "down":
-      return .down
-    case "left":
-      return .left
-    case "right":
-      return .right
-    default:
-      return nil
-    }
+    guard let raw else { return nil }
+    return TvRemoteButton(rawValue: raw.lowercased())
   }
 
   func elementHasFocus(_ element: XCUIElement) -> Bool {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -66,7 +66,11 @@ final class RunnerTests: XCTestCase {
   let minRecordingFps = 1
   let maxRecordingFps = 120
   var needsPostSnapshotInteractionDelay = false
-  var needsFirstInteractionDelay = false
+  /// When the first interaction after an activation may run, on the monotonic uptime clock.
+  /// The guarantee is a minimum gap *since the activation*, not a pause at the interaction:
+  /// a caller that already spent that gap elsewhere (an agent's round trip is 190-260 ms)
+  /// has satisfied it and waits for nothing. `nil` = no activation is pending stabilization.
+  var firstInteractionReadyUptime: TimeInterval?
   var runnerAccessibilityHealth: RunnerAccessibilityHealth = .unknown
   var activeRecording: ScreenRecorder?
   let commandJournal = RunnerCommandJournal()
@@ -119,29 +123,21 @@ final class RunnerTests: XCTestCase {
   // Bounds the pre-plan SpringBoard system-modal probe, which can otherwise grind for tens of
   // seconds on remote-hosted consent dialogs and bypass the plan budget (#1244).
   let systemModalProbeBudget: TimeInterval = 4
+  // In-bundle unit tests (every `func test…` except `testCommand` below) compile only under
+  // `-D AGENT_DEVICE_RUNNER_UNIT_TESTS` and are classified by their `#if` guard (#1781 A7):
+  //   - `#if AGENT_DEVICE_RUNNER_UNIT_TESTS` alone: a pure runner decision (rule table,
+  //     geometry, parser, policy, journal, dispatch bookkeeping) that needs no launched app.
+  //     Runs on the macOS host lane on every PR (ci.yml, no simulator) and on the iOS lanes.
+  //   - `… && os(iOS)` (or a nested `#if os(iOS)`): runner/XCTest semantics — launches the
+  //     host app, routes through SpringBoard, swizzles XCUIApplication, or asserts an
+  //     iOS-only branch. Simulator lanes only (ios.yml PR list, xctest-nightly.yml).
+  // `pnpm check:xctest-selection` derives each lane's reachable set from these guards and
+  // fails when a declared test is reachable by no lane, so a test gated to a platform nothing
+  // runs (the old tvOS-only pair) cannot go dark silently.
   // Observability for the record(_:) suppression below: how many AX-broken-screen snapshot
   // issues this session muted, so wedge investigations see the volume without grepping logs.
   let suppressedIssueLock = NSLock()
   var suppressedAxSnapshotIssueCount = 0
-  let interactiveTypes: Set<XCUIElement.ElementType> = [
-    .button,
-    .cell,
-    .checkBox,
-    .collectionView,
-    .link,
-    .menuItem,
-    .picker,
-    .searchField,
-    .segmentedControl,
-    .slider,
-    .stepper,
-    .switch,
-    .tabBar,
-    .textField,
-    .secureTextField,
-    .textView,
-    .webView
-  ]
   // Keep blocker actions narrow to avoid false positives from generic hittable containers.
   let actionableTypes: Set<XCUIElement.ElementType> = [
     .button,

package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/RecordingExportSupport.swift

@@ -0,0 +1,111 @@
+import AVFoundation
+import Foundation
+
+/// Shared mechanics for the recording post-processing scripts (overlay burn-in, start trim).
+/// Each script keeps its own argument grammar and export-quality policy; this file owns the
+/// error vocabulary, flag-value reading, composition assembly, and the bounded export wait.
+
+enum RecordingScriptError: Error, CustomStringConvertible {
+  case invalidArgs(String)
+  case invalidTrimRange
+  case missingVideoTrack
+  case exportFailed(String)
+
+  var description: String {
+    switch self {
+    case .invalidArgs(let message):
+      return message
+    case .invalidTrimRange:
+      return "Trim start must be before the end of the recording."
+    case .missingVideoTrack:
+      return "Input video does not contain a video track."
+    case .exportFailed(let message):
+      return message
+    }
+  }
+}
+
+func recordingOptionValue(_ arguments: [String], _ nextIndex: Int, _ flag: String) throws -> String {
+  guard nextIndex < arguments.count else {
+    throw RecordingScriptError.invalidArgs("\(flag) requires a value")
+  }
+  return arguments[nextIndex]
+}
+
+func removeStaleOutput(_ outputURL: URL) throws {
+  if FileManager.default.fileExists(atPath: outputURL.path) {
+    try FileManager.default.removeItem(at: outputURL)
+  }
+}
+
+func sourceVideoTrack(of asset: AVURLAsset) throws -> AVAssetTrack {
+  guard let track = asset.tracks(withMediaType: .video).first else {
+    throw RecordingScriptError.missingVideoTrack
+  }
+  return track
+}
+
+/// Copies `videoTrack` for `timeRange` into a fresh composition, carrying the optional audio track
+/// along when present. The caller owns any `preferredTransform` policy: trim propagates the source
+/// transform directly, while overlay re-applies it through a video-composition layer instruction
+/// instead.
+func makeRecordingComposition(
+  asset: AVURLAsset,
+  videoTrack: AVAssetTrack,
+  timeRange: CMTimeRange
+) throws -> AVMutableComposition {
+  let composition = AVMutableComposition()
+  guard let compositionVideoTrack = composition.addMutableTrack(
+    withMediaType: .video,
+    preferredTrackID: kCMPersistentTrackID_Invalid
+  ) else {
+    throw RecordingScriptError.exportFailed("Failed to create composition video track.")
+  }
+  try compositionVideoTrack.insertTimeRange(timeRange, of: videoTrack, at: .zero)
+
+  if let sourceAudioTrack = asset.tracks(withMediaType: .audio).first,
+     let compositionAudioTrack = composition.addMutableTrack(
+       withMediaType: .audio,
+       preferredTrackID: kCMPersistentTrackID_Invalid
+     ) {
+    try? compositionAudioTrack.insertTimeRange(timeRange, of: sourceAudioTrack, at: .zero)
+  }
+  return composition
+}
+
+func makeRecordingExporter(
+  _ composition: AVAsset,
+  presetName: String,
+  outputURL: URL,
+  videoComposition: AVMutableVideoComposition? = nil
+) throws -> AVAssetExportSession {
+  guard let exporter = AVAssetExportSession(asset: composition, presetName: presetName) else {
+    throw RecordingScriptError.exportFailed("Failed to create export session.")
+  }
+  exporter.outputURL = outputURL
+  exporter.outputFileType = .mp4
+  exporter.videoComposition = videoComposition
+  exporter.shouldOptimizeForNetworkUse = true
+  return exporter
+}
+
+/// Bounded asynchronous export: signals completion through a semaphore and cancels after 120s so a
+/// wedged encoder cannot hang the recording pipeline past the caller's own timeout budget.
+func runRecordingExport(
+  _ exporter: AVAssetExportSession,
+  timeoutMessage: String,
+  failureMessage: String
+) throws {
+  let semaphore = DispatchSemaphore(value: 0)
+  exporter.exportAsynchronously {
+    semaphore.signal()
+  }
+  if semaphore.wait(timeout: .now() + 120) == .timedOut {
+    exporter.cancelExport()
+    throw RecordingScriptError.exportFailed(timeoutMessage)
+  }
+
+  if exporter.status != .completed {
+    throw RecordingScriptError.exportFailed(exporter.error?.localizedDescription ?? failureMessage)
+  }
+}