agent-device 0.21.19 → 0.21.21

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift

@@ -95,8 +95,9 @@ extension RunnerTests {
     app: XCUIApplication,
     x: Double,
     y: Double,
-    context: SynthesizedCoordinateContext? = nil
-  ) -> RunnerInteractionOutcome {
+    context: SynthesizedCoordinateContext? = nil,
+    deadline: Date? = nil
+  ) -> RunnerInteractionOutcome? {
 #if os(iOS)
     guard x.isFinite, y.isFinite else {
       return .unsupported(
@@ -126,18 +127,31 @@ extension RunnerTests {
       referenceFrame: context.referenceFrame,
       orientation: orientation
     )
-    if let message = RunnerSynthesizedGesture.synthesizeTap(
+    var message: NSString?
+    let status = RunnerSynthesizedGesture.synthesizeTap(
       withApplication: app,
       resolvedWindow: context.resolvedWindow,
       x: Double(point.x),
-      y: Double(point.y)
-    ) {
+      y: Double(point.y),
+      deadline: deadline,
+      errorMessage: &message
+    )
+    switch status {
+    case .succeeded:
+      return .performed
+    case .deadlineExceeded:
+      return nil
+    case .failed:
       return .unsupported(
-        message: message,
+        message: message as String? ?? "private XCTest event synthesis failed",
         hint: "Falling back to XCTest coordinate tap may be slower and can still need a healthy accessibility tree."
       )
+    @unknown default:
+      return .unsupported(
+        message: "private XCTest event synthesis returned an unknown status",
+        hint: "Inspect the current app before deciding whether to act again."
+      )
     }
-    return .performed
 #elseif os(tvOS)
     return .unsupported(
       message: "coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element",

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift

@@ -101,8 +101,27 @@ extension RunnerTests {
       var warmsUpField = false
 
 
+
+
+
+
+      static let maxClearPassCount = 4
+
+      static let unreadableClearPassCount = 2
+
+
       var synthesizeCallCount: Int {
-        replacesExistingText ? 2 : 1
+        replacesExistingText ? 2 * Self.maxClearPassCount + 1 : 1
+      }
+
+
+      var keystrokeCount: Int {
+        characterCount + clearReadCount
+      }
+
+
+      var clearReadCount: Int {
+        replacesExistingText ? Self.maxClearPassCount : 0
       }
     }
 
@@ -129,8 +148,9 @@ extension RunnerTests {
     var seconds: TimeInterval {
       steps.reduce(0) { total, step in
         total
-          + Double(step.characterCount) * TextEntryTiming.synthesizedCharacterInterval
+          + Double(step.keystrokeCount) * TextEntryTiming.synthesizedCharacterInterval
           + Double(step.synthesizeCallCount) * TextEntryTiming.synthesizeCallOverhead
+          + Double(step.clearReadCount) * TextEntryTiming.synthesizedClearReadAllowance
           + step.pauseAfterSeconds
       }
     }
@@ -226,6 +246,48 @@ extension RunnerTests {
     case stop
   }
 
+
+  enum ClearedFieldRead: Equatable {
+    case text(String)
+
+    case unreadable
+
+    case unavailable
+  }
+
+
+  enum SynthesizedClearOutcome: Equatable {
+    case cleared
+
+    case unverified
+
+    case notCleared
+
+    case stopped
+  }
+
+
+
+  static func clearForSynthesizedReplacement(
+    clearOnce: () -> SynthesizedStepDispatch,
+    read: () -> ClearedFieldRead
+  ) -> SynthesizedClearOutcome {
+    for pass in 1...SynthesizedTextPlan.Step.maxClearPassCount {
+      guard clearOnce() == .posted else { return .stopped }
+      switch read() {
+      case .text(let value) where value.isEmpty:
+        return .cleared
+      case .text:
+        continue
+      case .unreadable:
+        if pass >= SynthesizedTextPlan.Step.unreadableClearPassCount { return .unverified }
+      case .unavailable:
+        return .notCleared
+      }
+    }
+    return .notCleared
+  }
+
   struct SynthesizedPlanRun {
     let postedCharacterCount: Int
 
@@ -298,32 +360,68 @@ extension RunnerTests {
         )
       )
     }
+    var clearOutcome: SynthesizedClearOutcome?
 
 
     let synthesisAvailable = runSynthesizedTextPlan(
       plan,
       text: request.text,
       post: { slice, step in
-        switch request.synthesizer.enterText(
-          app: request.app,
-          text: slice,
-          replacingExistingText: step.replacesExistingText
-        ) {
-        case .continueTyping:
-          return .posted
-        case .fallback:
-          return .stop
-        case .raise(let message):
-          NSException(
-            name: NSExceptionName.internalInconsistencyException,
-            reason: message ?? "private XCTest text synthesis failed"
-          ).raise()
-          return .stop
+        func enter(_ text: String, replacingExistingText: Bool) -> SynthesizedStepDispatch {
+          switch request.synthesizer.enterText(
+            app: request.app,
+            text: text,
+            replacingExistingText: replacingExistingText
+          ) {
+          case .continueTyping:
+            return .posted
+          case .fallback:
+            return .stop
+          case .raise(let message):
+            NSException(
+              name: NSExceptionName.internalInconsistencyException,
+              reason: message ?? "private XCTest text synthesis failed"
+            ).raise()
+            return .stop
+          }
+        }
+        if step.replacesExistingText {
+          var passes = 0
+          let outcome = Self.clearForSynthesizedReplacement(
+            clearOnce: {
+              passes += 1
+              return enter(XCUIKeyboardKey.delete.rawValue, replacingExistingText: true)
+            },
+            read: { self.clearedFieldRead(app: request.app, target: request.target) }
+          )
+          clearOutcome = outcome
+          NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_CLEAR passes=%d outcome=%@", passes, String(describing: outcome))
+          guard outcome == .cleared || outcome == .unverified else { return .stop }
         }
+        return enter(slice, replacingExistingText: false)
       },
 
       waitAfterWarmupCharacter: { _ in }
     )
+    if clearOutcome == .notCleared {
+      logTextEntryPhase(
+        commandId: request.commandId,
+        phase: "total",
+        startedAt: request.startedAt,
+        chars: request.text.count,
+        mode: .replacement
+      )
+      return .completed(
+        TextEntryResult(
+          verified: nil,
+          repaired: false,
+          expectedText: request.text,
+          observedText: nil,
+          textEntryRoute: "synthesized-first-responder-replacement",
+          failure: .clearNotObserved
+        )
+      )
+    }
     if synthesisAvailable.stoppedEarly {
       NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=verified-fallback reason=synthesis-unavailable")
       guard let point = request.target.refreshPoint else { return .notApplicable }
@@ -432,6 +530,7 @@ extension RunnerTests {
   static func awaitSynthesizedReplacementCommitOutcome(
     expectedText: String,
     placeholder: String?,
+    fieldIsSecure: Bool = false,
     stallBudget: TimeInterval = TextEntryTiming.synthesizedCommitStallTimeout,
     ceiling: TimeInterval = TextEntryTiming.synthesizedCommitCeiling,
     now: () -> Date = { Date() },
@@ -440,6 +539,11 @@ extension RunnerTests {
   ) -> SynthesizedTextCommitOutcome {
 
 
+    if fieldIsSecure {
+      return .unobservable
+    }
+
+
 
     if Self.textMatchesPlaceholder(expectedText, placeholder: placeholder) {
       return .notObserved

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift

@@ -8,6 +8,7 @@ extension RunnerTests {
     case notFocused = "TEXT_INPUT_NOT_FOCUSED"
     case synthesisUnavailable = "TEXT_INPUT_SYNTHESIS_UNAVAILABLE"
     case commitNotObserved = "TEXT_INPUT_COMMIT_NOT_OBSERVED"
+    case clearNotObserved = "TEXT_INPUT_CLEAR_NOT_OBSERVED"
     case synthesisBudgetExceeded = "TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED"
 
     var message: String {
@@ -18,6 +19,8 @@ extension RunnerTests {
         return "Reliable text synthesis is unavailable while the software keyboard is hidden."
       case .commitNotObserved:
         return "The runner could not confirm the typed text reached the field."
+      case .clearNotObserved:
+        return "The runner could not clear the field before typing, so it typed nothing."
       case .synthesisBudgetExceeded:
         return "The text is longer than one runner command can type at this pace."
       }
@@ -31,6 +34,8 @@ extension RunnerTests {
         return "Show the software keyboard, then retry type."
       case .commitNotObserved:
         return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms \(TextEntryTiming.recoveryDelayMilliseconds). Do not use type, which appends to whatever committed."
+      case .clearNotObserved:
+        return "The field may still hold some or all of its old text. Run snapshot -i and inspect the field, then retry fill. Do not use type, which appends to whatever the field holds."
       case .synthesisBudgetExceeded:
         let recoveryDelay = TextEntryTiming.recoveryDelayMilliseconds
         let recoveryBudget = SynthesizedDeliveryBudget.maxTextLength(
@@ -61,13 +66,24 @@ extension RunnerTests {
 
 
 
+    static let unconfirmedSettleCeiling: TimeInterval = 1.0
+
+
+
     static let synthesizedCommitStallTimeout: TimeInterval = 3.0
 
 
     static let synthesizedCommitCeiling: TimeInterval = 10.0
 
 
-    static let synthesizedReplacementFocusAllowance: TimeInterval = 2.0
+
+
+    static let synthesizedReplacementFocusAllowance: TimeInterval = 4.0
+
+
+
+
+    static let synthesizedClearReadAllowance: TimeInterval = 0.9
 
 
 
@@ -108,6 +124,27 @@ extension RunnerTests {
     let observedText: String?
     var textEntryRoute: String? = nil
     var failure: TextEntryFailure? = nil
+    var unconfirmed: TextEntryUnconfirmedEvidence? = nil
+  }
+
+
+
+
+
+
+  struct TextEntryInputIdentity: Equatable {
+    let elementType: XCUIElement.ElementType
+    let identifier: String
+
+    var isDistinguishable: Bool { !identifier.isEmpty }
+  }
+
+
+
+
+  struct TextInputAtPoint {
+    let element: XCUIElement
+    let identity: TextEntryInputIdentity
   }
 
   struct TextEntryTarget {
@@ -116,33 +153,81 @@ extension RunnerTests {
     let prefersFocusedElement: Bool
     let fromTapWitness: Bool
 
+    let boundIdentity: TextEntryInputIdentity?
+
+
+
+
+
+    let inputAtRefreshPoint: TextInputAtPoint?
+
     init(
       element: XCUIElement?,
       refreshPoint: CGPoint?,
       prefersFocusedElement: Bool,
-      fromTapWitness: Bool = false
+      fromTapWitness: Bool = false,
+      boundIdentity: TextEntryInputIdentity? = nil,
+      inputAtRefreshPoint: TextInputAtPoint? = nil
     ) {
       self.element = element
       self.refreshPoint = refreshPoint
       self.prefersFocusedElement = prefersFocusedElement
       self.fromTapWitness = fromTapWitness
+      self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity
+      self.inputAtRefreshPoint = inputAtRefreshPoint
     }
 
     func withElement(_ nextElement: XCUIElement?) -> TextEntryTarget {
       guard let nextElement else {
         return self
       }
-      let frame = nextElement.frame
+
+
+
+      let snapshot = nextElement.exists ? try? nextElement.snapshot() : nil
+      let frame = snapshot?.frame ?? .zero
       let point = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY)
       return TextEntryTarget(
         element: nextElement,
         refreshPoint: point,
         prefersFocusedElement: prefersFocusedElement,
-        fromTapWitness: fromTapWitness
+        fromTapWitness: fromTapWitness,
+        boundIdentity: boundIdentity ?? snapshot.map {
+          TextEntryInputIdentity(elementType: $0.elementType, identifier: $0.identifier)
+        },
+        inputAtRefreshPoint: inputAtRefreshPoint
       )
     }
   }
 
+
+  enum TextEntryInputProbe: Equatable {
+    case input(TextEntryInputIdentity)
+
+    case noMatch
+
+    case unavailable
+  }
+
+  static let xCTestUITestingErrorDomain = "com.apple.dt.xctest.ui-testing.error"
+
+
+  static let xCTestNoMatchesErrorCode = 10008
+
+
+  static func textEntryInputProbe(snapshotError error: Error) -> TextEntryInputProbe {
+    let error = error as NSError
+    return error.domain == xCTestUITestingErrorDomain && error.code == xCTestNoMatchesErrorCode
+      ? .noMatch
+      : .unavailable
+  }
+
+
+
+  static func textEntryRepairCanTarget(boundIdentity: TextEntryInputIdentity?) -> Bool {
+    boundIdentity?.isDistinguishable ?? true
+  }
+
   struct TextEntryStabilization {
     let element: XCUIElement?
     let focusConfirmed: Bool
@@ -199,7 +284,95 @@ extension RunnerTests {
     }
   }
 
+
+
+
   func resolveTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
+    guard let boundIdentity = target.boundIdentity else {
+      return resolveUnboundTextEntryElement(app: app, target: target)
+    }
+    func isBoundInput(_ candidate: XCUIElement) -> Bool {
+      candidate.exists && probeTextEntryInput(candidate) == .input(boundIdentity)
+    }
+    if target.prefersFocusedElement, let focused = focusedTextInput(app: app), isBoundInput(focused) {
+      return focused
+    }
+    if let element = target.element, isBoundInput(element) {
+      return element
+    }
+    if let input = target.inputAtRefreshPoint, textInputStillResolves(input) {
+      return input.element
+    }
+    if target.inputAtRefreshPoint == nil,
+       let refreshPoint = target.refreshPoint,
+       case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint),
+       let match = candidates.first(where: isBoundInput) {
+      return match
+    }
+    if let focused = focusedTextInput(app: app), isBoundInput(focused) {
+      return focused
+    }
+    if let byIdentifier = boundTextEntryInputQuery(app: app, identity: boundIdentity), isBoundInput(byIdentifier) {
+      return byIdentifier
+    }
+    return nil
+  }
+
+
+
+
+
+  func boundTextEntryInputIsGone(app: XCUIApplication, target: TextEntryTarget) -> Bool {
+    guard let boundIdentity = target.boundIdentity,
+          let query = boundTextEntryInputQuery(app: app, identity: boundIdentity) ?? target.element
+    else {
+      return false
+    }
+    switch probeTextEntryInput(query) {
+    case .input(let identity) where identity != boundIdentity:
+      break
+    case .noMatch:
+      break
+    case .input, .unavailable:
+      return false
+    }
+    guard let refreshPoint = target.refreshPoint else {
+      return true
+    }
+    switch probeTextInputs(app: app, point: refreshPoint) {
+    case .absent:
+      return true
+    case .matches(let candidates):
+      return candidates.allSatisfy { probeTextEntryInput($0) != .input(boundIdentity) }
+    case .unavailable:
+      return false
+    }
+  }
+
+
+  private func boundTextEntryInputQuery(app: XCUIApplication, identity: TextEntryInputIdentity) -> XCUIElement? {
+    guard identity.isDistinguishable else {
+      return nil
+    }
+    return app.descendants(matching: identity.elementType).matching(identifier: identity.identifier).element
+  }
+
+
+  func probeTextEntryInput(_ element: XCUIElement) -> TextEntryInputProbe {
+    var probe = TextEntryInputProbe.unavailable
+    let (_, exception) = catchingObjCException(fallback: ()) {
+      do {
+        let snapshot = try element.snapshot()
+        probe = .input(TextEntryInputIdentity(elementType: snapshot.elementType, identifier: snapshot.identifier))
+      } catch {
+        probe = Self.textEntryInputProbe(snapshotError: error)
+      }
+    }
+    return exception == nil ? probe : .unavailable
+  }
+
+
+  private func resolveUnboundTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
     if target.prefersFocusedElement {
       if let focused = focusedTextInput(app: app) {
         return focused
@@ -251,7 +424,7 @@ extension RunnerTests {
   }
 
   func editableTextValue(
-    for element: XCUIElement?,
+    for element: (any XCUIElementAttributes)?,
     treatingPlaceholderAsEmpty: Bool = false
   ) -> String? {
     guard let element else {
@@ -271,7 +444,7 @@ extension RunnerTests {
     }
   }
 
-  private func isPlaceholderValue(_ value: String, for element: XCUIElement) -> Bool {
+  private func isPlaceholderValue(_ value: String, for element: any XCUIElementAttributes) -> Bool {
     if Self.textMatchesPlaceholder(value, placeholder: element.placeholderValue) {
       return true
     }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift

@@ -0,0 +1,119 @@
+import XCTest
+
+
+
+
+extension RunnerTests {
+
+  struct TextEntryElementIdentity: Equatable {
+    let identifier: String?
+    let elementType: String
+    let frame: CGRect
+
+
+
+    func isSameElement(as other: TextEntryElementIdentity) -> Bool {
+      if identifier != nil || other.identifier != nil {
+        return identifier == other.identifier
+      }
+      return elementType == other.elementType && frame == other.frame
+    }
+  }
+
+
+  struct TextEntryObservation: Equatable {
+    let value: String
+    let identity: TextEntryElementIdentity
+
+
+    func isSettled(with other: TextEntryObservation) -> Bool {
+      value == other.value && identity.isSameElement(as: other.identity)
+    }
+  }
+
+
+  struct TextEntryUnconfirmedEvidence: Equatable {
+    let requested: String
+    let before: String
+    let after: String
+    let target: TextEntryElementIdentity
+  }
+
+
+
+
+
+
+  static func textEntryValueEchoes(observed: String, expected: String, baseline: String) -> Bool {
+    let request = textEntryRequestWithoutSubmitKeys(expected)
+    let residualAndRequest = baseline + request
+    return isOrderedSubsequence(observed, of: residualAndRequest)
+      || (isOrderedSubsequence(request, of: observed) && !isOrderedSubsequence(request, of: baseline))
+      || isOrderedSubsequence(residualAndRequest, of: observed)
+  }
+
+
+
+
+  static func unconfirmedTextEntryEvidence(
+    requested: String,
+    baseline: TextEntryObservation?,
+    observed: TextEntryObservation?
+  ) -> TextEntryUnconfirmedEvidence? {
+    guard !textEntryRequestWithoutSubmitKeys(requested).isEmpty,
+          let baseline,
+          let observed,
+          baseline.identity.isSameElement(as: observed.identity),
+          observed.value != baseline.value,
+          !textEntryValueEchoes(observed: observed.value, expected: requested, baseline: baseline.value)
+    else {
+      return nil
+    }
+    return TextEntryUnconfirmedEvidence(
+      requested: requested,
+      before: baseline.value,
+      after: observed.value,
+      target: observed.identity
+    )
+  }
+
+
+
+  func textEntryObservation(for element: XCUIElement?) -> TextEntryObservation? {
+    guard let element,
+          let snapshot = try? element.snapshot(),
+          let value = editableTextValue(for: snapshot, treatingPlaceholderAsEmpty: true)
+    else {
+      return nil
+    }
+    return TextEntryObservation(
+      value: value,
+      identity: TextEntryElementIdentity(
+        identifier: snapshot.identifier.isEmpty ? nil : snapshot.identifier,
+        elementType: elementTypeName(snapshot.elementType),
+        frame: snapshot.frame
+      )
+    )
+  }
+
+
+  static func textEntryRequestWithoutSubmitKeys(_ text: String) -> String {
+    var request = text
+    while request.hasSuffix("\n") || request.hasSuffix("\r") {
+      request.removeLast()
+    }
+    return request
+  }
+
+
+  static func isOrderedSubsequence(_ candidate: String, of text: String) -> Bool {
+    var remaining = text[...]
+    for character in candidate {
+      guard let match = remaining.firstIndex(of: character) else {
+        return false
+      }
+      remaining = remaining[remaining.index(after: match)...]
+    }
+    return true
+  }
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift

@@ -6,9 +6,16 @@ import XCTest
 
 
 extension RunnerTests {
+
+
+
+
+
+
+
   @MainActor
   func rememberTextEntryTap(_ element: XCUIElement?) {
-    guard let element, isTextEntryElement(element) else {
+    guard let element else {
       clearRememberedTextEntryTap()
       return
     }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift

@@ -80,21 +80,33 @@ extension RunnerTests {
 #endif
   }
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
   func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) {
 #if os(iOS)
-    switch element.elementType {
-    case .textField, .secureTextField, .searchField, .textView:
-      if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil {
-        return
-      }
-      let frame = element.frame
-      if !frame.isEmpty {
-        _ = tapAt(app: app, x: frame.midX, y: frame.midY)
-        _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout)
-      }
-    default:
+    if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil {
       return
     }
+    guard let settled = safely("TEXT_ENTRY_READINESS_FRAME", { try? element.snapshot() }) else {
+      return
+    }
+    let frame = settled.frame
+    if !frame.isEmpty {
+      _ = tapAt(app: app, x: frame.midX, y: frame.midY)
+      _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout)
+    }
 #endif
   }