agent-device 0.21.20 → 0.21.22

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift

@@ -8,6 +8,7 @@ extension RunnerTests {
     case notFocused = "TEXT_INPUT_NOT_FOCUSED"
     case synthesisUnavailable = "TEXT_INPUT_SYNTHESIS_UNAVAILABLE"
     case commitNotObserved = "TEXT_INPUT_COMMIT_NOT_OBSERVED"
+    case clearNotObserved = "TEXT_INPUT_CLEAR_NOT_OBSERVED"
     case synthesisBudgetExceeded = "TEXT_INPUT_SYNTHESIS_BUDGET_EXCEEDED"
 
     var message: String {
@@ -18,6 +19,8 @@ extension RunnerTests {
         return "Reliable text synthesis is unavailable while the software keyboard is hidden."
       case .commitNotObserved:
         return "The runner could not confirm the typed text reached the field."
+      case .clearNotObserved:
+        return "The runner could not clear the field before typing, so it typed nothing."
       case .synthesisBudgetExceeded:
         return "The text is longer than one runner command can type at this pace."
       }
@@ -31,6 +34,8 @@ extension RunnerTests {
         return "Show the software keyboard, then retry type."
       case .commitNotObserved:
         return "The field may hold none, part, or all of the text. Run snapshot -i and inspect the field: if it already matches, continue; otherwise retry fill with the full text quoted and --delay-ms \(TextEntryTiming.recoveryDelayMilliseconds). Do not use type, which appends to whatever committed."
+      case .clearNotObserved:
+        return "The field may still hold some or all of its old text. Run snapshot -i and inspect the field, then retry fill. Do not use type, which appends to whatever the field holds."
       case .synthesisBudgetExceeded:
         let recoveryDelay = TextEntryTiming.recoveryDelayMilliseconds
         let recoveryBudget = SynthesizedDeliveryBudget.maxTextLength(
@@ -61,13 +66,24 @@ extension RunnerTests {
 
 
 
+    static let unconfirmedSettleCeiling: TimeInterval = 1.0
+
+
+
     static let synthesizedCommitStallTimeout: TimeInterval = 3.0
 
 
     static let synthesizedCommitCeiling: TimeInterval = 10.0
 
 
-    static let synthesizedReplacementFocusAllowance: TimeInterval = 2.0
+
+
+    static let synthesizedReplacementFocusAllowance: TimeInterval = 4.0
+
+
+
+
+    static let synthesizedClearReadAllowance: TimeInterval = 0.9
 
 
 
@@ -108,6 +124,27 @@ extension RunnerTests {
     let observedText: String?
     var textEntryRoute: String? = nil
     var failure: TextEntryFailure? = nil
+    var unconfirmed: TextEntryUnconfirmedEvidence? = nil
+  }
+
+
+
+
+
+
+  struct TextEntryInputIdentity: Equatable {
+    let elementType: XCUIElement.ElementType
+    let identifier: String
+
+    var isDistinguishable: Bool { !identifier.isEmpty }
+  }
+
+
+
+
+  struct TextInputAtPoint {
+    let element: XCUIElement
+    let identity: TextEntryInputIdentity
   }
 
   struct TextEntryTarget {
@@ -116,33 +153,81 @@ extension RunnerTests {
     let prefersFocusedElement: Bool
     let fromTapWitness: Bool
 
+    let boundIdentity: TextEntryInputIdentity?
+
+
+
+
+
+    let inputAtRefreshPoint: TextInputAtPoint?
+
     init(
       element: XCUIElement?,
       refreshPoint: CGPoint?,
       prefersFocusedElement: Bool,
-      fromTapWitness: Bool = false
+      fromTapWitness: Bool = false,
+      boundIdentity: TextEntryInputIdentity? = nil,
+      inputAtRefreshPoint: TextInputAtPoint? = nil
     ) {
       self.element = element
       self.refreshPoint = refreshPoint
       self.prefersFocusedElement = prefersFocusedElement
       self.fromTapWitness = fromTapWitness
+      self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity
+      self.inputAtRefreshPoint = inputAtRefreshPoint
     }
 
     func withElement(_ nextElement: XCUIElement?) -> TextEntryTarget {
       guard let nextElement else {
         return self
       }
-      let frame = nextElement.frame
+
+
+
+      let snapshot = nextElement.exists ? try? nextElement.snapshot() : nil
+      let frame = snapshot?.frame ?? .zero
       let point = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY)
       return TextEntryTarget(
         element: nextElement,
         refreshPoint: point,
         prefersFocusedElement: prefersFocusedElement,
-        fromTapWitness: fromTapWitness
+        fromTapWitness: fromTapWitness,
+        boundIdentity: boundIdentity ?? snapshot.map {
+          TextEntryInputIdentity(elementType: $0.elementType, identifier: $0.identifier)
+        },
+        inputAtRefreshPoint: inputAtRefreshPoint
       )
     }
   }
 
+
+  enum TextEntryInputProbe: Equatable {
+    case input(TextEntryInputIdentity)
+
+    case noMatch
+
+    case unavailable
+  }
+
+  static let xCTestUITestingErrorDomain = "com.apple.dt.xctest.ui-testing.error"
+
+
+  static let xCTestNoMatchesErrorCode = 10008
+
+
+  static func textEntryInputProbe(snapshotError error: Error) -> TextEntryInputProbe {
+    let error = error as NSError
+    return error.domain == xCTestUITestingErrorDomain && error.code == xCTestNoMatchesErrorCode
+      ? .noMatch
+      : .unavailable
+  }
+
+
+
+  static func textEntryRepairCanTarget(boundIdentity: TextEntryInputIdentity?) -> Bool {
+    boundIdentity?.isDistinguishable ?? true
+  }
+
   struct TextEntryStabilization {
     let element: XCUIElement?
     let focusConfirmed: Bool
@@ -199,7 +284,95 @@ extension RunnerTests {
     }
   }
 
+
+
+
   func resolveTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
+    guard let boundIdentity = target.boundIdentity else {
+      return resolveUnboundTextEntryElement(app: app, target: target)
+    }
+    func isBoundInput(_ candidate: XCUIElement) -> Bool {
+      candidate.exists && probeTextEntryInput(candidate) == .input(boundIdentity)
+    }
+    if target.prefersFocusedElement, let focused = focusedTextInput(app: app), isBoundInput(focused) {
+      return focused
+    }
+    if let element = target.element, isBoundInput(element) {
+      return element
+    }
+    if let input = target.inputAtRefreshPoint, textInputStillResolves(input) {
+      return input.element
+    }
+    if target.inputAtRefreshPoint == nil,
+       let refreshPoint = target.refreshPoint,
+       case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint),
+       let match = candidates.first(where: isBoundInput) {
+      return match
+    }
+    if let focused = focusedTextInput(app: app), isBoundInput(focused) {
+      return focused
+    }
+    if let byIdentifier = boundTextEntryInputQuery(app: app, identity: boundIdentity), isBoundInput(byIdentifier) {
+      return byIdentifier
+    }
+    return nil
+  }
+
+
+
+
+
+  func boundTextEntryInputIsGone(app: XCUIApplication, target: TextEntryTarget) -> Bool {
+    guard let boundIdentity = target.boundIdentity,
+          let query = boundTextEntryInputQuery(app: app, identity: boundIdentity) ?? target.element
+    else {
+      return false
+    }
+    switch probeTextEntryInput(query) {
+    case .input(let identity) where identity != boundIdentity:
+      break
+    case .noMatch:
+      break
+    case .input, .unavailable:
+      return false
+    }
+    guard let refreshPoint = target.refreshPoint else {
+      return true
+    }
+    switch probeTextInputs(app: app, point: refreshPoint) {
+    case .absent:
+      return true
+    case .matches(let candidates):
+      return candidates.allSatisfy { probeTextEntryInput($0) != .input(boundIdentity) }
+    case .unavailable:
+      return false
+    }
+  }
+
+
+  private func boundTextEntryInputQuery(app: XCUIApplication, identity: TextEntryInputIdentity) -> XCUIElement? {
+    guard identity.isDistinguishable else {
+      return nil
+    }
+    return app.descendants(matching: identity.elementType).matching(identifier: identity.identifier).element
+  }
+
+
+  func probeTextEntryInput(_ element: XCUIElement) -> TextEntryInputProbe {
+    var probe = TextEntryInputProbe.unavailable
+    let (_, exception) = catchingObjCException(fallback: ()) {
+      do {
+        let snapshot = try element.snapshot()
+        probe = .input(TextEntryInputIdentity(elementType: snapshot.elementType, identifier: snapshot.identifier))
+      } catch {
+        probe = Self.textEntryInputProbe(snapshotError: error)
+      }
+    }
+    return exception == nil ? probe : .unavailable
+  }
+
+
+  private func resolveUnboundTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
     if target.prefersFocusedElement {
       if let focused = focusedTextInput(app: app) {
         return focused
@@ -251,7 +424,7 @@ extension RunnerTests {
   }
 
   func editableTextValue(
-    for element: XCUIElement?,
+    for element: (any XCUIElementAttributes)?,
     treatingPlaceholderAsEmpty: Bool = false
   ) -> String? {
     guard let element else {
@@ -271,7 +444,7 @@ extension RunnerTests {
     }
   }
 
-  private func isPlaceholderValue(_ value: String, for element: XCUIElement) -> Bool {
+  private func isPlaceholderValue(_ value: String, for element: any XCUIElementAttributes) -> Bool {
     if Self.textMatchesPlaceholder(value, placeholder: element.placeholderValue) {
       return true
     }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift

@@ -0,0 +1,119 @@
+import XCTest
+
+
+
+
+extension RunnerTests {
+
+  struct TextEntryElementIdentity: Equatable {
+    let identifier: String?
+    let elementType: String
+    let frame: CGRect
+
+
+
+    func isSameElement(as other: TextEntryElementIdentity) -> Bool {
+      if identifier != nil || other.identifier != nil {
+        return identifier == other.identifier
+      }
+      return elementType == other.elementType && frame == other.frame
+    }
+  }
+
+
+  struct TextEntryObservation: Equatable {
+    let value: String
+    let identity: TextEntryElementIdentity
+
+
+    func isSettled(with other: TextEntryObservation) -> Bool {
+      value == other.value && identity.isSameElement(as: other.identity)
+    }
+  }
+
+
+  struct TextEntryUnconfirmedEvidence: Equatable {
+    let requested: String
+    let before: String
+    let after: String
+    let target: TextEntryElementIdentity
+  }
+
+
+
+
+
+
+  static func textEntryValueEchoes(observed: String, expected: String, baseline: String) -> Bool {
+    let request = textEntryRequestWithoutSubmitKeys(expected)
+    let residualAndRequest = baseline + request
+    return isOrderedSubsequence(observed, of: residualAndRequest)
+      || (isOrderedSubsequence(request, of: observed) && !isOrderedSubsequence(request, of: baseline))
+      || isOrderedSubsequence(residualAndRequest, of: observed)
+  }
+
+
+
+
+  static func unconfirmedTextEntryEvidence(
+    requested: String,
+    baseline: TextEntryObservation?,
+    observed: TextEntryObservation?
+  ) -> TextEntryUnconfirmedEvidence? {
+    guard !textEntryRequestWithoutSubmitKeys(requested).isEmpty,
+          let baseline,
+          let observed,
+          baseline.identity.isSameElement(as: observed.identity),
+          observed.value != baseline.value,
+          !textEntryValueEchoes(observed: observed.value, expected: requested, baseline: baseline.value)
+    else {
+      return nil
+    }
+    return TextEntryUnconfirmedEvidence(
+      requested: requested,
+      before: baseline.value,
+      after: observed.value,
+      target: observed.identity
+    )
+  }
+
+
+
+  func textEntryObservation(for element: XCUIElement?) -> TextEntryObservation? {
+    guard let element,
+          let snapshot = try? element.snapshot(),
+          let value = editableTextValue(for: snapshot, treatingPlaceholderAsEmpty: true)
+    else {
+      return nil
+    }
+    return TextEntryObservation(
+      value: value,
+      identity: TextEntryElementIdentity(
+        identifier: snapshot.identifier.isEmpty ? nil : snapshot.identifier,
+        elementType: elementTypeName(snapshot.elementType),
+        frame: snapshot.frame
+      )
+    )
+  }
+
+
+  static func textEntryRequestWithoutSubmitKeys(_ text: String) -> String {
+    var request = text
+    while request.hasSuffix("\n") || request.hasSuffix("\r") {
+      request.removeLast()
+    }
+    return request
+  }
+
+
+  static func isOrderedSubsequence(_ candidate: String, of text: String) -> Bool {
+    var remaining = text[...]
+    for character in candidate {
+      guard let match = remaining.firstIndex(of: character) else {
+        return false
+      }
+      remaining = remaining[remaining.index(after: match)...]
+    }
+    return true
+  }
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift

@@ -6,9 +6,16 @@ import XCTest
 
 
 extension RunnerTests {
+
+
+
+
+
+
+
   @MainActor
   func rememberTextEntryTap(_ element: XCUIElement?) {
-    guard let element, isTextEntryElement(element) else {
+    guard let element else {
       clearRememberedTextEntryTap()
       return
     }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift

@@ -80,21 +80,33 @@ extension RunnerTests {
 #endif
   }
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
   func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) {
 #if os(iOS)
-    switch element.elementType {
-    case .textField, .secureTextField, .searchField, .textView:
-      if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil {
-        return
-      }
-      let frame = element.frame
-      if !frame.isEmpty {
-        _ = tapAt(app: app, x: frame.midX, y: frame.midY)
-        _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout)
-      }
-    default:
+    if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil {
       return
     }
+    guard let settled = safely("TEXT_ENTRY_READINESS_FRAME", { try? element.snapshot() }) else {
+      return
+    }
+    let frame = settled.frame
+    if !frame.isEmpty {
+      _ = tapAt(app: app, x: frame.midX, y: frame.midY)
+      _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout)
+    }
 #endif
   }
 

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift

@@ -5,7 +5,7 @@ final class TextInputProbeIssues {
   var count = 0
 }
 
-enum TextInputProbeFailure: String {
+enum TextInputProbeFailure: String, Error {
   case recordedIssue = "text_input_probe_recorded_issue"
   case exception = "text_input_probe_exception"
 }
@@ -38,7 +38,60 @@ extension RunnerTests {
     }
   }
 
+
+
+  func coordinateTapTextInputIdentityAt(app: XCUIApplication, x: Double, y: Double) -> TextInputAtPoint? {
+    let probed = containingTextInputProbeIssues(fallback: nil) { shouldStop -> TextInputAtPoint? in
+      guard let element = queryTextInputs(app: app, point: CGPoint(x: x, y: y), shouldStop: shouldStop).first,
+            case .input(let identity) = probeTextEntryInput(element)
+      else {
+        return nil
+      }
+      return TextInputAtPoint(element: element, identity: identity)
+    }
+    guard case .success(let input) = probed else { return nil }
+    return input
+  }
+
+
+
+  func textInputStillResolves(_ input: TextInputAtPoint) -> Bool {
+    let probed = containingTextInputProbeIssues(fallback: false) { _ in
+      input.element.exists && probeTextEntryInput(input.element) == .input(input.identity)
+    }
+    guard case .success(let resolves) = probed else { return false }
+    return resolves
+  }
+
+
+
+  func clearedFieldRead(app: XCUIApplication, target: TextEntryTarget) -> ClearedFieldRead {
+    let probed = containingTextInputProbeIssues(fallback: ClearedFieldRead.unavailable) { _ in
+      guard let input = resolveTextEntryElement(app: app, target: target) else { return .unavailable }
+      if input.elementType == .secureTextField { return .unreadable }
+      return editableTextValue(for: input, treatingPlaceholderAsEmpty: true).map { .text($0) } ?? .unavailable
+    }
+    guard case .success(let read) = probed else { return .unavailable }
+    return read
+  }
+
   func probeTextInputs(app: XCUIApplication, point: CGPoint) -> TextInputProbeOutcome {
+    switch containingTextInputProbeIssues(fallback: [], { shouldStop in
+      queryTextInputs(app: app, point: point, shouldStop: shouldStop)
+    }) {
+    case .success(let elements):
+      return elements.isEmpty ? .absent : .matches(elements)
+    case .failure(let failure):
+      return .unavailable(failure)
+    }
+  }
+
+
+
+  private func containingTextInputProbeIssues<T>(
+    fallback: T,
+    _ probe: (_ shouldStop: () -> Bool) -> T
+  ) -> Result<T, TextInputProbeFailure> {
     precondition(Thread.isMainThread)
     let issues = TextInputProbeIssues()
     suppressedIssueLock.lock()
@@ -50,12 +103,12 @@ extension RunnerTests {
       textInputProbeIssues = previous
       suppressedIssueLock.unlock()
     }
-    let (elements, exception) = catchingObjCException(fallback: []) {
-      queryTextInputs(app: app, point: point, shouldStop: { self.hasTextInputProbeIssues(issues) })
+    let (value, exception) = catchingObjCException(fallback: fallback) {
+      probe({ self.hasTextInputProbeIssues(issues) })
     }
-    if hasTextInputProbeIssues(issues) { return .unavailable(.recordedIssue) }
-    if exception != nil { return .unavailable(.exception) }
-    return elements.isEmpty ? .absent : .matches(elements)
+    if hasTextInputProbeIssues(issues) { return .failure(.recordedIssue) }
+    if exception != nil { return .failure(.exception) }
+    return .success(value)
   }
 
   private func hasTextInputProbeIssues(_ scope: TextInputProbeIssues) -> Bool {