agent-device 0.20.5 → 0.20.7

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift

@@ -4,6 +4,29 @@ import XCTest
 // pipeline, readiness polling, and field clearing. Behavior-preserving extraction from
 // RunnerTests+Interaction.swift (no logic changes) to keep that file navigable.
 extension RunnerTests {
+  enum TextEntryFailure: String {
+    case notFocused = "TEXT_INPUT_NOT_FOCUSED"
+    case synthesisUnavailable = "TEXT_INPUT_SYNTHESIS_UNAVAILABLE"
+
+    var message: String {
+      switch self {
+      case .notFocused:
+        return "No focused text input was available for typing."
+      case .synthesisUnavailable:
+        return "Reliable text synthesis is unavailable while the software keyboard is hidden."
+      }
+    }
+
+    var hint: String {
+      switch self {
+      case .notFocused:
+        return "Focus a visible text input, then retry type or fill. If the input is not exposed by accessibility, use a coordinate focus command before typing."
+      case .synthesisUnavailable:
+        return "Show the software keyboard, then retry type or fill."
+      }
+    }
+  }
+
   enum TextTypingRepairMode {
     case none
     case append
@@ -17,6 +40,7 @@ extension RunnerTests {
     static let pollInterval: TimeInterval = 0.02
     static let warmupValueTimeout: TimeInterval = 0.4
     static let verificationStabilityWindow: TimeInterval = 0.2
+    static let synthesizedCommitTimeout: TimeInterval = 3.0
   }
 
   struct TextEntryResult {
@@ -24,12 +48,27 @@ extension RunnerTests {
     let repaired: Bool
     let expectedText: String?
     let observedText: String?
+    var textEntryRoute: String? = nil
+    var failure: TextEntryFailure? = nil
   }
 
   struct TextEntryTarget {
     let element: XCUIElement?
     let refreshPoint: CGPoint?
     let prefersFocusedElement: Bool
+    let fromTapWitness: Bool
+
+    init(
+      element: XCUIElement?,
+      refreshPoint: CGPoint?,
+      prefersFocusedElement: Bool,
+      fromTapWitness: Bool = false
+    ) {
+      self.element = element
+      self.refreshPoint = refreshPoint
+      self.prefersFocusedElement = prefersFocusedElement
+      self.fromTapWitness = fromTapWitness
+    }
 
     func withElement(_ nextElement: XCUIElement?) -> TextEntryTarget {
       guard let nextElement else {
@@ -40,7 +79,8 @@ extension RunnerTests {
       return TextEntryTarget(
         element: nextElement,
         refreshPoint: point,
-        prefersFocusedElement: prefersFocusedElement
+        prefersFocusedElement: prefersFocusedElement,
+        fromTapWitness: fromTapWitness
       )
     }
   }
@@ -50,6 +90,16 @@ extension RunnerTests {
     let focusConfirmed: Bool
   }
 
+  struct TextEntryTapWitness {
+    let element: XCUIElement
+    let bundleId: String?
+    let processIdentifier: Int?
+
+    func matches(bundleId: String?, processIdentifier: Int?) -> Bool {
+      self.bundleId == bundleId && self.processIdentifier == processIdentifier
+    }
+  }
+
   func clearTextInput(_ element: XCUIElement) {
     // Skip the clear (delete burst + moveCaretToEnd edge-tap) ONLY when we can confirm the
     // field is empty. Why skip: the edge-tap computes a point from the element frame, which can
@@ -95,6 +145,54 @@ extension RunnerTests {
 #endif
   }
 
+  func rememberTextEntryTap(_ element: XCUIElement?) {
+    guard let element, isTextEntryElement(element) else {
+      clearRememberedTextEntryTap()
+      return
+    }
+    textEntryTapWitness = TextEntryTapWitness(
+      element: element,
+      bundleId: currentBundleId,
+      processIdentifier: currentAppProcessIdentifier
+    )
+  }
+
+  func clearRememberedTextEntryTap() {
+    textEntryTapWitness = nil
+  }
+
+  private func rememberedTextEntryTarget() -> TextEntryTarget? {
+    guard let witness = textEntryTapWitness else {
+      return nil
+    }
+    // The tap is proof for one immediately-following bare type only. Consume it before checking
+    // the element so a failed or interrupted type cannot reuse stale focus evidence.
+    clearRememberedTextEntryTap()
+    guard witness.matches(
+      bundleId: currentBundleId,
+      processIdentifier: currentAppProcessIdentifier
+    ) else {
+      return nil
+    }
+    let element = witness.element
+    // XCUIElement is query-backed rather than a stable node identity. A same-identifier field
+    // introduced by app-side navigation between tap and this immediate type can therefore
+    // re-resolve here; keep the witness one-shot and fail closed on every observable identity
+    // boundary instead of using frame equality, which would reject legitimate layout changes.
+    guard safely("LAST_TAPPED_TEXT_INPUT_EXISTS", false, { element.exists }) else {
+      return nil
+    }
+    // Keep the target scoped to the element that the preceding tap actually selected. Do not
+    // attach a refresh point: if that element disappeared, bare type must fail closed rather
+    // than rediscovering a different field or dispatching unscoped app.typeText.
+    return TextEntryTarget(
+      element: element,
+      refreshPoint: nil,
+      prefersFocusedElement: false,
+      fromTapWitness: true
+    )
+  }
+
   func stabilizeTextInputBeforeTyping(
     app: XCUIApplication,
     target: XCUIElement?,
@@ -123,11 +221,15 @@ extension RunnerTests {
 
   func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget {
     guard let x, let y else {
+      let softwareKeyboardVisible = isKeyboardVisible(app: app)
+      if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget() {
+        return rememberedTarget
+      }
       // Bare `type` targets the current first responder. On iOS we intentionally do not trust
       // `hasKeyboardFocus`, but an already-visible software keyboard is sufficient evidence that
       // app.typeText has a receiver; waiting the full readiness timeout cannot prove a stronger
       // target because there is no selector/coordinate focus move to validate.
-      if isKeyboardVisible(app: app) {
+      if softwareKeyboardVisible {
         return TextEntryTarget(
           element: focusedTextInput(app: app),
           refreshPoint: nil,
@@ -244,451 +346,7 @@ extension RunnerTests {
     }
   }
 
-  func typeTextReliably(
-    app: XCUIApplication,
-    target: TextEntryTarget,
-    text: String,
-    delaySeconds: Double,
-    repairMode: TextTypingRepairMode = .none,
-    commandId: String? = nil
-  ) -> TextEntryResult {
-    let totalStartedAt = Date()
-    guard !text.isEmpty else {
-      logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode)
-      return TextEntryResult(verified: true, repaired: false, expectedText: "", observedText: "")
-    }
-    var activeTarget = target
-    let initialResolveStartedAt = Date()
-    let initialTarget = resolveTextEntryElement(app: app, target: activeTarget)
-    activeTarget = activeTarget.withElement(initialTarget)
-    let currentText = editableTextValue(for: initialTarget, treatingPlaceholderAsEmpty: true)
-    let initialText = repairMode == .append ? currentText : nil
-    let expectedText = expectedTextEntryValue(typedText: text, mode: repairMode, initialText: initialText)
-    logTextEntryPhase(
-      commandId: commandId,
-      phase: "initial-resolve",
-      startedAt: initialResolveStartedAt,
-      chars: text.count,
-      mode: repairMode
-    )
-
-    if repairMode == .replacement {
-      guard let replacementTarget = initialTarget else {
-        logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
-        return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil)
-      }
-      if currentText == nil || currentText?.isEmpty == false {
-        let clearStartedAt = Date()
-        clearTextInput(replacementTarget)
-        activeTarget = activeTarget.withElement(replacementTarget)
-        logTextEntryPhase(
-          commandId: commandId,
-          phase: "clear",
-          startedAt: clearStartedAt,
-          chars: text.count,
-          mode: repairMode
-        )
-      }
-    }
-
-    func typeIntoCurrentTarget(_ value: String) -> XCUIElement? {
-      if let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) {
-        currentTarget.typeText(value)
-        return currentTarget
-      } else {
-        app.typeText(value)
-        return resolveTextEntryElement(app: app, target: activeTarget)
-      }
-    }
-
-    func waitForWarmupValue(_ expectedValue: String?, target: TextEntryTarget) {
-      guard let expectedValue else {
-        sleepFor(TextEntryTiming.pollInterval)
-        return
-      }
-      let deadline = Date().addingTimeInterval(TextEntryTiming.warmupValueTimeout)
-      while Date() < deadline {
-        if editableTextValue(for: resolveTextEntryElement(app: app, target: target)) == expectedValue {
-          return
-        }
-        sleepFor(TextEntryTiming.pollInterval)
-      }
-    }
-
-    let characters = Array(text)
-    if delaySeconds > 0 && characters.count > 1 {
-      var typedTarget: XCUIElement?
-      let delayedTypeStartedAt = Date()
-      for (index, character) in characters.enumerated() {
-        typedTarget = typeIntoCurrentTarget(String(character)) ?? typedTarget
-        if index + 1 < characters.count {
-          sleepFor(delaySeconds)
-        }
-      }
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "type-delayed",
-        startedAt: delayedTypeStartedAt,
-        chars: characters.count,
-        mode: repairMode
-      )
-      if repairMode == .none {
-        logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
-        return TextEntryResult(verified: nil, repaired: false, expectedText: nil, observedText: nil)
-      }
-      let verifyStartedAt = Date()
-      let result = verifyTextEntryWithRepairIfNeeded(
-        app: app,
-        target: activeTarget.withElement(typedTarget),
-        expectedText: expectedText,
-        repairMode: repairMode
-      )
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "verify",
-        startedAt: verifyStartedAt,
-        chars: characters.count,
-        mode: repairMode
-      )
-      logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
-      return result
-    }
-
-    let typedTarget: XCUIElement?
-    if repairMode != .none && characters.count > 1 {
-      let firstCharacter = String(characters[0])
-      let firstStartedAt = Date()
-      var firstTypedTarget = typeIntoCurrentTarget(firstCharacter)
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "type-first",
-        startedAt: firstStartedAt,
-        chars: 1,
-        mode: repairMode
-      )
-      activeTarget = activeTarget.withElement(firstTypedTarget)
-      let warmupExpectedText = expectedTextEntryValue(
-        typedText: firstCharacter,
-        mode: repairMode,
-        initialText: initialText
-      )
-      let warmupStartedAt = Date()
-      waitForWarmupValue(warmupExpectedText, target: activeTarget)
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "warmup",
-        startedAt: warmupStartedAt,
-        chars: 1,
-        mode: repairMode
-      )
-      let remainingText = String(characters.dropFirst())
-      let remainingStartedAt = Date()
-      firstTypedTarget = typeIntoCurrentTarget(remainingText) ?? firstTypedTarget
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "type-remaining",
-        startedAt: remainingStartedAt,
-        chars: characters.count - 1,
-        mode: repairMode
-      )
-      typedTarget = firstTypedTarget
-    } else {
-      let typeStartedAt = Date()
-      typedTarget = typeIntoCurrentTarget(text)
-      logTextEntryPhase(
-        commandId: commandId,
-        phase: "type-all",
-        startedAt: typeStartedAt,
-        chars: characters.count,
-        mode: repairMode
-      )
-    }
-    if repairMode == .none {
-      logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
-      return TextEntryResult(verified: nil, repaired: false, expectedText: nil, observedText: nil)
-    }
-    let verifyStartedAt = Date()
-    let result = verifyTextEntryWithRepairIfNeeded(
-      app: app,
-      target: activeTarget.withElement(typedTarget),
-      expectedText: expectedText,
-      repairMode: repairMode
-    )
-    logTextEntryPhase(
-      commandId: commandId,
-      phase: "verify",
-      startedAt: verifyStartedAt,
-      chars: characters.count,
-      mode: repairMode
-    )
-    logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: text.count, mode: repairMode)
-    return result
-  }
-
-  private func logTextEntryPhase(
-    commandId: String?,
-    phase: String,
-    startedAt: Date,
-    chars: Int,
-    mode: TextTypingRepairMode
-  ) {
-    NSLog(
-      "AGENT_DEVICE_RUNNER_TEXT_ENTRY_PHASE commandId=%@ phase=%@ durationMs=%.1f chars=%d mode=%@",
-      commandId ?? "",
-      phase,
-      Date().timeIntervalSince(startedAt) * 1000.0,
-      chars,
-      textEntryModeName(mode)
-    )
-  }
-
-  func textEntryModeName(_ mode: TextTypingRepairMode) -> String {
-    switch mode {
-    case .none:
-      return "none"
-    case .append:
-      return "append"
-    case .replacement:
-      return "replacement"
-    }
-  }
-
-  private func verifyTextEntryWithRepairIfNeeded(
-    app: XCUIApplication,
-    target: TextEntryTarget,
-    expectedText: String?,
-    repairMode: TextTypingRepairMode
-  ) -> TextEntryResult {
-    let initialResult = verifyTextEntry(
-      app: app,
-      target: target,
-      expectedText: expectedText,
-      repaired: false
-    )
-#if os(iOS)
-    guard initialResult.verified == false,
-          let expectedText = initialResult.expectedText
-    else {
-      return initialResult
-    }
-    guard shouldRepairTextEntry(
-      app: app,
-      target: target,
-      expectedText: expectedText,
-      repairMode: repairMode
-    ) else {
-      return verifyTextEntry(
-        app: app,
-        target: target,
-        expectedText: expectedText,
-        repaired: false
-      )
-    }
-
-    guard let repairTarget = resolveTextEntryElement(app: app, target: target) else {
-      return initialResult
-    }
-    let observedText = editableTextValue(for: repairTarget) ?? ""
-    NSLog(
-      "AGENT_DEVICE_RUNNER_REPAIR_TEXT_ENTRY expectedLength=%d observedLength=%d",
-      expectedText.count,
-      observedText.count
-    )
-    clearTextInput(repairTarget)
-    repairTarget.typeText(expectedText)
-    return verifyTextEntry(
-      app: app,
-      target: target.withElement(repairTarget),
-      expectedText: expectedText,
-      repaired: true
-    )
-#else
-    return initialResult
-#endif
-  }
-
-  private func verifyTextEntry(
-    app: XCUIApplication,
-    target: TextEntryTarget,
-    expectedText: String?,
-    repaired: Bool
-  ) -> TextEntryResult {
-    let targetElement = resolveTextEntryElement(app: app, target: target)
-    guard let expectedText else {
-      return TextEntryResult(
-        verified: nil,
-        repaired: repaired,
-        expectedText: nil,
-        observedText: editableTextValue(for: targetElement)
-      )
-    }
-    guard let observedText = editableTextValue(for: targetElement) else {
-      return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil)
-    }
-    guard textEntryValueMatchesExpected(targetElement, observedText: observedText, expectedText: expectedText) else {
-      return TextEntryResult(
-        verified: false,
-        repaired: repaired,
-        expectedText: expectedText,
-        observedText: observedText
-      )
-    }
-    let stableDeadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow)
-    var latestObservedText = observedText
-    while Date() < stableDeadline {
-      sleepFor(TextEntryTiming.pollInterval)
-      guard let nextObservedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else {
-        return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil)
-      }
-      latestObservedText = nextObservedText
-      guard textEntryValueMatchesExpected(
-        resolveTextEntryElement(app: app, target: target),
-        observedText: nextObservedText,
-        expectedText: expectedText
-      ) else {
-        return TextEntryResult(
-          verified: false,
-          repaired: repaired,
-          expectedText: expectedText,
-          observedText: nextObservedText
-        )
-      }
-    }
-    return TextEntryResult(
-      verified: true,
-      repaired: repaired,
-      expectedText: expectedText,
-      observedText: latestObservedText
-    )
-  }
-
-  private func textEntryValueMatchesExpected(
-    _ element: XCUIElement?,
-    observedText: String,
-    expectedText: String
-  ) -> Bool {
-    if observedText == expectedText {
-      return true
-    }
-    guard hasTextEntrySubmitSuffix(expectedText), element?.elementType != .textView else {
-      return false
-    }
-    var submittedText = expectedText
-    while hasTextEntrySubmitSuffix(submittedText) {
-      submittedText.removeLast()
-    }
-    return observedText == submittedText
-  }
-
-  private func hasTextEntrySubmitSuffix(_ text: String) -> Bool {
-    text.hasSuffix("\n") || text.hasSuffix("\r")
-  }
-
-  private func expectedTextEntryValue(
-    typedText: String,
-    mode: TextTypingRepairMode,
-    initialText: String?
-  ) -> String? {
-    switch mode {
-    case .none:
-      return nil
-    case .append:
-      guard let initialText else {
-        return nil
-      }
-      return initialText + typedText
-    case .replacement:
-      return typedText
-    }
-  }
-
-  private func shouldRepairTextEntry(
-    app: XCUIApplication,
-    target: TextEntryTarget,
-    expectedText: String,
-    repairMode: TextTypingRepairMode
-  ) -> Bool {
-#if os(iOS)
-    var latestObservedText: String?
-    let deadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow)
-    repeat {
-      guard let observedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else {
-        return false
-      }
-      if textEntryValueMatchesExpected(
-        resolveTextEntryElement(app: app, target: target),
-        observedText: observedText,
-        expectedText: expectedText
-      ) {
-        return false
-      }
-      latestObservedText = observedText
-      if !isRepairableTextEntryMismatch(
-        observedText: observedText,
-        expectedText: expectedText,
-        repairMode: repairMode
-      ) {
-        return false
-      }
-      sleepFor(TextEntryTiming.pollInterval)
-    } while Date() < deadline
-
-    guard let latestObservedText else {
-      return false
-    }
-    guard !textEntryValueMatchesExpected(
-      resolveTextEntryElement(app: app, target: target),
-      observedText: latestObservedText,
-      expectedText: expectedText
-    ) else {
-      return false
-    }
-    return isRepairableTextEntryMismatch(
-      observedText: latestObservedText,
-      expectedText: expectedText,
-      repairMode: repairMode
-    )
-#else
-    return false
-#endif
-  }
-
-  private func isRepairableTextEntryMismatch(
-    observedText: String,
-    expectedText: String,
-    repairMode: TextTypingRepairMode
-  ) -> Bool {
-    guard observedText != expectedText else {
-      return false
-    }
-    if repairMode == .replacement {
-      return true
-    }
-    return observedText.isEmpty || isLikelyDroppedCharacterTextEntryMismatch(
-      observedText: observedText,
-      expectedText: expectedText
-    )
-  }
-
-  private func isLikelyDroppedCharacterTextEntryMismatch(observedText: String, expectedText: String) -> Bool {
-    guard observedText.count < expectedText.count else {
-      return false
-    }
-    let missingCharacterCount = expectedText.count - observedText.count
-    guard missingCharacterCount <= max(2, expectedText.count / 4) else {
-      return false
-    }
-    var expectedIndex = expectedText.startIndex
-    for character in observedText {
-      guard let matchIndex = expectedText[expectedIndex...].firstIndex(of: character) else {
-        return false
-      }
-      expectedIndex = expectedText.index(after: matchIndex)
-    }
-    return true
-  }
-
-  private func resolveTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
+  func resolveTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
     if target.prefersFocusedElement {
       if let focused = focusedTextInput(app: app) {
         return focused
@@ -836,7 +494,7 @@ extension RunnerTests {
       .trimmingCharacters(in: .whitespacesAndNewlines)
   }
 
-  private func editableTextValue(
+  func editableTextValue(
     for element: XCUIElement?,
     treatingPlaceholderAsEmpty: Bool = false
   ) -> String? {

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift

@@ -0,0 +1,4 @@
+import XCTest
+
+extension RunnerTests {
+}