agent-device 0.21.11 → 0.21.13

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift

@@ -132,7 +132,6 @@ extension RunnerTests {
 
 
 
-
     let commit = awaitSynthesizedReplacementCommit(
       app: request.app,
       target: request.target,
@@ -168,37 +167,16 @@ extension RunnerTests {
     !hasResolvedElement && hasRefreshPoint && xCTestChannelPenalized
   }
 
+
+
+
   static func shouldUseSynthesizedFirstResponderType(
     repairMode: TextTypingRepairMode,
+    text: String,
     fromTapWitness: Bool,
     softwareKeyboardVisible: Bool
   ) -> Bool {
-    repairMode == .none && fromTapWitness && !softwareKeyboardVisible
-  }
-
-  enum SynthesizedTextCommitProgress: Equatable {
-    case committed
-    case pending
-    case diverged
-  }
-
-
-
-
-
-
-
-  static func synthesizedTextCommitProgress(
-    observedText: String?,
-    expectedText: String
-  ) -> SynthesizedTextCommitProgress {
-    guard let observedText else {
-      return .diverged
-    }
-    if observedText == expectedText {
-      return .committed
-    }
-    return expectedText.hasPrefix(observedText) ? .pending : .diverged
+    repairMode == .none && text == "\n" && fromTapWitness && !softwareKeyboardVisible
   }
 
 
@@ -215,15 +193,8 @@ extension RunnerTests {
 
 
 
-
-
-
   enum SynthesizedTextCommitOutcome: Equatable {
 
-
-
-
-
     case settled
 
     case unobservable
@@ -233,53 +204,6 @@ extension RunnerTests {
 
 
 
-
-
-
-
-
-
-  static func awaitSynthesizedCommitOutcome(
-    expectedText: String,
-    placeholder: String?,
-    stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout,
-    ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling,
-    now: () -> Date = { Date() },
-    observe: () -> String?,
-    waitForNextObservation: () -> Void
-  ) -> SynthesizedTextCommitOutcome {
-
-
-
-    if Self.textMatchesPlaceholder(expectedText, placeholder: placeholder) {
-      return .notObserved
-    }
-    var deadline = SynthesizedCommitDeadline(startedAt: now(), stallBudget: stallBudget, ceiling: ceiling)
-
-
-
-    while true {
-      let observedText = observe()
-      switch synthesizedTextCommitProgress(observedText: observedText, expectedText: expectedText) {
-      case .committed, .diverged:
-        return .settled
-      case .pending:
-
-
-        let sampledAt = now()
-        deadline.record(
-          expectedPrefixLength: Self.commonPrefixLength(observedText ?? "", expectedText),
-          at: sampledAt
-        )
-        if deadline.isExpired(at: sampledAt) { return .notObserved }
-        waitForNextObservation()
-      }
-    }
-  }
-
-
-
-
   static func textEntryFailure(
     forCommitOutcome outcome: SynthesizedTextCommitOutcome
   ) -> TextEntryFailure? {
@@ -301,12 +225,6 @@ extension RunnerTests {
 
 
 
-
-
-
-
-
-
   static func awaitSynthesizedReplacementCommitOutcome(
     expectedText: String,
     placeholder: String?,
@@ -316,10 +234,16 @@ extension RunnerTests {
     observe: () -> String?,
     waitForNextObservation: () -> Void
   ) -> SynthesizedTextCommitOutcome {
+
+
+
     if Self.textMatchesPlaceholder(expectedText, placeholder: placeholder) {
       return .notObserved
     }
     var deadline = SynthesizedCommitDeadline(startedAt: now(), stallBudget: stallBudget, ceiling: ceiling)
+
+
+
     while true {
       let observedText = observe()
       if observedText == expectedText {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift

@@ -69,15 +69,7 @@ extension RunnerTests {
     in springboard: XCUIApplication,
     deadline: Date = .distantFuture
   ) -> XCUIElement? {
-    let disableSafeProbe = RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_DISABLE_SAFE_MODAL_PROBE")
-    let queryElements: (() -> [XCUIElement]) -> [XCUIElement] = { fetch in
-      if disableSafeProbe {
-        return fetch()
-      }
-      return self.safeElementsQuery(fetch)
-    }
-
-    let alerts = queryElements {
+    let alerts = safeElementsQuery {
       springboard.alerts.allElementsBoundByIndex
     }
     for alert in alerts {
@@ -92,7 +84,7 @@ extension RunnerTests {
       return nil
     }
 
-    let sheets = queryElements {
+    let sheets = safeElementsQuery {
       springboard.sheets.allElementsBoundByIndex
     }
     for sheet in sheets {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift

@@ -25,7 +25,7 @@ extension RunnerTests {
       case .notFocused:
         return "Focus a visible text input, then retry type or fill. If the input is not exposed by accessibility, use a coordinate focus command before typing."
       case .synthesisUnavailable:
-        return "Show the software keyboard, then retry type or fill."
+        return "Show the software keyboard, then retry type."
       case .commitNotObserved:
         return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms 80. Do not use type, which appends to whatever committed."
       }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift

@@ -152,41 +152,6 @@ extension RunnerTests {
     )
   }
 
-  func focusTextInputForTextEntry(app: XCUIApplication, element: XCUIElement) -> TextEntryTarget {
-    let point = textEntryRefreshPoint(for: element)
-    let keyboardVisibleBeforeTap = isKeyboardVisible(app: app)
-    if let point {
-      _ = tapAt(app: app, x: point.x, y: point.y)
-    }
-
-
-    if keyboardVisibleBeforeTap {
-      return TextEntryTarget(
-        element: element,
-        refreshPoint: textEntryRefreshPoint(for: element) ?? point,
-        prefersFocusedElement: false
-      )
-    }
-    let stabilized = stabilizeTextInputBeforeTyping(
-      app: app,
-      target: element,
-      keyboardVisibleBeforeTap: keyboardVisibleBeforeTap
-    )
-    let readyTarget = TextEntryTarget(
-      element: stabilized.element ?? element,
-      refreshPoint: point,
-      prefersFocusedElement: false
-    )
-    let resolved = stabilized.focusConfirmed
-      ? (stabilized.element ?? element)
-      : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? element)
-    return TextEntryTarget(
-      element: resolved,
-      refreshPoint: textEntryRefreshPoint(for: resolved) ?? point,
-      prefersFocusedElement: false
-    )
-  }
-
   private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? {
     guard let element else {
       return nil

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift

@@ -13,35 +13,4 @@ func isCoordinateTextInputCandidate(
     && point.x <= frame.maxX + tolerance
     && point.y >= frame.minY - tolerance
     && point.y <= frame.maxY + tolerance
-}
-
-extension RunnerTests {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift

@@ -77,6 +77,7 @@ extension RunnerTests {
     }
     let shouldUseSynthesizedFirstResponderType = Self.shouldUseSynthesizedFirstResponderType(
       repairMode: repairMode,
+      text: text,
       fromTapWitness: activeTarget.fromTapWitness,
       softwareKeyboardVisible: isKeyboardVisible(app: app)
     )
@@ -142,26 +143,9 @@ extension RunnerTests {
         }
         textEntryRoute = "synthesized-first-responder"
         NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
-        let textBefore = editableTextValue(for: currentTarget, treatingPlaceholderAsEmpty: true)
         switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) {
         case .continueTyping:
-
-
-          let commit = awaitSynthesizedFirstResponderCommit(
-            app: app,
-            target: TextEntryTarget(
-              element: currentTarget,
-              refreshPoint: nil,
-              prefersFocusedElement: false,
-              fromTapWitness: true
-            ),
-            textBefore: textBefore,
-            typedText: value
-          )
-
-
-
-          return (currentTarget, Self.textEntryFailure(forCommitOutcome: commit))
+          return (currentTarget, nil)
         case .fallback:
           return (nil, .synthesisUnavailable)
         case .raise(let message):

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift

@@ -108,12 +108,8 @@ extension RunnerTests {
 
     do {
       let command = try JSONDecoder().decode(Command.self, from: body)
-      if command.command == .status {
-        completion((jsonResponse(status: 200, response: executeStatus(command: command)), false))
-        return
-      }
-      if command.command == .uptime {
-        completion((jsonResponse(status: 200, response: executeUptime()), false))
+      if let response = inlineResponse(for: command) {
+        completion((jsonResponse(status: 200, response: response), false))
         return
       }
 
@@ -127,10 +123,9 @@ extension RunnerTests {
         command.command.rawValue,
         command.commandId ?? ""
       )
-      commandJournal.accept(command: command)
-      commandExecutionQueue.async {
-        do {
-          let response = try self.executeAccepted(command: command)
+      enqueueAccepted(command: command) { result in
+        switch result {
+        case .success(let response):
           NSLog(
             "AGENT_DEVICE_RUNNER_COMMAND_COMPLETED command=%@ commandId=%@ ok=%d",
             command.command.rawValue,
@@ -142,7 +137,7 @@ extension RunnerTests {
             result: (self.jsonResponse(status: 200, response: response), command.command == .shutdown),
             completion: completion
           )
-        } catch {
+        case .failure(let error):
           NSLog(
             "AGENT_DEVICE_RUNNER_COMMAND_FAILED command=%@ commandId=%@ error=%@",
             command.command.rawValue,
@@ -177,9 +172,35 @@ extension RunnerTests {
 
 
 
+  func inlineResponse(for command: Command) -> Response? {
+    switch command.command {
+    case .status:
+      return executeStatus(command: command)
+    case .uptime:
+      return executeUptime()
+    default:
+      return nil
+    }
+  }
+
+
+
+  func enqueueAccepted(
+    command: Command,
+    completion: @escaping (Result<Response, Error>) -> Void
+  ) {
+    commandJournal.accept(command: command)
+    commandExecutionQueue.async {
+      completion(Result { try self.executeAccepted(command: command) })
+    }
+  }
+
+
+
+
 
 
-  private func attachToInFlightCommandIfNeeded(
+  func attachToInFlightCommandIfNeeded(
     command: Command,
     completion: @escaping ((data: Data, shouldFinish: Bool)) -> Void
   ) -> Bool {
@@ -200,7 +221,7 @@ extension RunnerTests {
     return false
   }
 
-  private func deliverCommandResult(
+  func deliverCommandResult(
     command: Command,
     result: (data: Data, shouldFinish: Bool),
     completion: ((data: Data, shouldFinish: Bool)) -> Void
@@ -220,45 +241,6 @@ extension RunnerTests {
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
   private func jsonResponse(status: Int, response: Response) -> Data {
 
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift

@@ -18,28 +18,6 @@ enum TvRemoteButton: String {
 }
 
 extension RunnerTests {
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
   func resolveTvRemoteDoublePressDelay() -> TimeInterval {
     guard
       let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_TV_REMOTE_DOUBLE_PRESS_DELAY_MS"],

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift

@@ -0,0 +1,142 @@
+import XCTest
+import AgentDeviceSnapshotPresentation
+
+extension RunnerTests {
+  func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
+    guard let text = command.text else {
+      return Response(ok: false, error: ErrorPayload(message: "type requires text"))
+    }
+    let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0
+    let textEntryMode = resolveTextEntryMode(command)
+    let target: TextEntryTarget
+    var resolvedCoordinateContext: SynthesizedCoordinateContext?
+
+
+    let maestroNonHittableCoordinateFallbackUsed: Bool? =
+      command.allowNonHittableCoordinateFallback == true && command.x != nil && command.y != nil
+      ? true
+      : nil
+    let focusStartedAt = Date()
+#if os(iOS)
+    let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
+    var resolvedCoordinateTarget: TextEntryTarget?
+    if Self.shouldUseResolvedCoordinateTextEntryRoute(
+      repairMode: textEntryMode,
+      hasX: command.x != nil,
+      hasY: command.y != nil,
+      xCTestChannelPenalized: xCTestChannelPenalized
+    ), let x = command.x, let y = command.y {
+      let policyKind = SynthesizedGesturePolicyKind.coordinateTap
+      let context = synthesizedCoordinateContext(
+        app: activeApp,
+        policy: synthesizedGesturePolicy(policyKind)
+      )
+      let (_, outcome) = performGesture(activeApp, idleTimeout: false) {
+        synthesizedTapAt(app: activeApp, x: x, y: y, context: context)
+      }
+      if Self.shouldFallbackFromSynthesizedTextEntryFocus(outcome) {
+        logSynthesizedGesturePolicyDecision(
+          kind: policyKind,
+          context: context,
+          fallbackAttempted: true
+        )
+      } else {
+        logSynthesizedGesturePolicyDecision(
+          kind: policyKind,
+          context: context,
+          fallbackAttempted: false
+        )
+        resolvedCoordinateContext = context
+        resolvedCoordinateTarget = TextEntryTarget(
+          element: nil,
+          refreshPoint: CGPoint(x: x, y: y),
+          prefersFocusedElement: false
+        )
+      }
+    }
+#else
+    let xCTestChannelPenalized = false
+    let resolvedCoordinateTarget: TextEntryTarget? = nil
+#endif
+    if let resolvedCoordinateTarget {
+      target = resolvedCoordinateTarget
+    } else {
+      target = focusTextInputForTextEntry(app: activeApp, x: command.x, y: command.y)
+    }
+    NSLog(
+      "AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=%@ phase=focus durationMs=%.1f chars=%d mode=%@",
+      command.commandId ?? "",
+      Date().timeIntervalSince(focusStartedAt) * 1000.0,
+      text.count,
+      textEntryModeName(textEntryMode)
+    )
+    if textEntryMode == .replacement {
+#if os(iOS)
+      let canReplaceResolvedFirstResponder = Self.shouldUseSynthesizedFirstResponderReplacement(
+        hasResolvedElement: target.element != nil,
+        hasRefreshPoint: target.refreshPoint != nil,
+        xCTestChannelPenalized: xCTestChannelPenalized
+      )
+#else
+      let canReplaceResolvedFirstResponder = false
+#endif
+      guard target.element != nil || canReplaceResolvedFirstResponder else {
+        let message =
+          (command.x != nil && command.y != nil)
+          ? "no text input found at the provided coordinates to clear"
+          : "no focused text input to clear"
+        return Response(ok: false, error: ErrorPayload(message: message))
+      }
+    }
+    let textResult = typeTextReliably(
+      app: activeApp,
+      target: target,
+      text: text,
+      delaySeconds: delaySeconds,
+      repairMode: textEntryMode,
+      xCTestChannelPenalized: xCTestChannelPenalized,
+      synthesizer: PrivateXCTestTextEntrySynthesizer(),
+      commandId: command.commandId
+    )
+    if let failure = textResult.failure {
+      return Response(
+        ok: false,
+        error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)
+      )
+    }
+    if textResult.verified == false {
+      let expected = textResult.expectedText ?? ""
+      let observed = textResult.observedText ?? ""
+      return Response(
+        ok: false,
+        error: ErrorPayload(
+          code: "TEXT_ENTRY_MISMATCH",
+          message: "text entry verification failed: expected \"\(expected)\", observed \"\(observed)\""
+        )
+      )
+    }
+    let point = target.refreshPoint
+    let frame: CGRect
+    if let resolvedCoordinateContext {
+      frame = resolvedCoordinateContext.referenceFrame
+    } else if point != nil {
+      frame = activeApp.frame
+    } else {
+
+
+      frame = .zero
+    }
+    return Response(
+      ok: true,
+      data: DataPayload(
+        message: textResult.repaired ? "typed after repair" : "typed",
+        x: point.map { Double($0.x) },
+        y: point.map { Double($0.y) },
+        referenceWidth: frame.isEmpty ? nil : Double(frame.width),
+        referenceHeight: frame.isEmpty ? nil : Double(frame.height),
+        maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed,
+        textEntryRoute: textResult.textEntryRoute
+      )
+    )
+  }
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -31,8 +31,10 @@ final class RunnerTests: XCTestCase {
 
 
 
+
   enum RunnerWireErrorCode {
     static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT"
+    static let appNotRunning = "APP_NOT_RUNNING"
   }
 
   static let springboardBundleId = "com.apple.springboard"
@@ -52,6 +54,8 @@ final class RunnerTests: XCTestCase {
   let commandExecutionQueue = DispatchQueue(label: "agent-device.runner.commands")
   let app = XCUIApplication()
   lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId)
+
+
   var currentApp: XCUIApplication?
   var currentBundleId: String?
   var currentAppProcessIdentifier: Int?
@@ -75,11 +79,22 @@ final class RunnerTests: XCTestCase {
   let xctestIdleKeepaliveInterval: TimeInterval = 60.0
   let minRecordingFps = 1
   let maxRecordingFps = 120
+
+
+
+  let recordingFrameCaptureTimeout: TimeInterval = 1
   var needsPostSnapshotInteractionDelay = false
 
 
 
 
+  var lastLoggedFastAppGuardLine: String?
+  var lastLoggedGesturePolicyLines: [SynthesizedGesturePolicyKind: String] = [:]
+  var runnerMarkerWriter: (String) -> Void = { NSLog("%@", $0) }
+
+
+
+
   var firstInteractionReadyUptime: TimeInterval?
   var runnerAccessibilityHealth: RunnerAccessibilityHealth = .unknown
   var activeRecording: ScreenRecorder?
@@ -95,6 +110,9 @@ final class RunnerTests: XCTestCase {
 
   let mainThreadWorkLock = NSLock()
   var abandonedMainThreadWorkCount = 0
+
+
+  var mainThreadWorkInFlightCount = 0
   var abandonedMainThreadWorkSince: Date?
 
 
@@ -107,7 +125,7 @@ final class RunnerTests: XCTestCase {
   var snapshotXCTestChannelPenaltyBundleId: String?
   var snapshotXCTestChannelPenaltyUntil = Date.distantPast
   let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120
-  var snapshotXCTestPenaltyWarmupExemptionPending = false
+  let snapshotXCTestPenaltyWarmupExemption = SnapshotXCTestPenaltyWarmupExemption()
 
 
 
@@ -189,6 +207,9 @@ final class RunnerTests: XCTestCase {
 
 
 
+
+
+
 
 
 
@@ -263,11 +284,6 @@ final class RunnerTests: XCTestCase {
 
   @MainActor
   func testCommand() throws {
-    if RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_NOOP_STARTUP") {
-      NSLog("AGENT_DEVICE_RUNNER_NOOP_STARTUP=1")
-      return
-    }
-
     doneExpectation = expectation(description: "agent-device command handled")
     NSLog("AGENT_DEVICE_RUNNER_HEADLESS_STARTUP=1")
     let desiredPort = RunnerEnv.resolvePort()