agent-device 0.20.10 → 0.21.1

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift

@@ -0,0 +1,206 @@
+import XCTest
+
+
+
+
+
+
+extension RunnerTests {
+  func rememberTextEntryTap(_ element: XCUIElement?) {
+    guard let element, isTextEntryElement(element) else {
+      clearRememberedTextEntryTap()
+      return
+    }
+    textEntryTapWitness = TextEntryTapWitness(
+      element: element,
+      bundleId: currentBundleId,
+      processIdentifier: currentAppProcessIdentifier
+    )
+  }
+
+  func clearRememberedTextEntryTap() {
+    textEntryTapWitness = nil
+  }
+
+  private func rememberedTextEntryTarget() -> TextEntryTarget? {
+    guard let witness = textEntryTapWitness else {
+      return nil
+    }
+
+
+    clearRememberedTextEntryTap()
+    guard witness.matches(
+      bundleId: currentBundleId,
+      processIdentifier: currentAppProcessIdentifier
+    ) else {
+      return nil
+    }
+    let element = witness.element
+
+
+
+
+    guard safely("LAST_TAPPED_TEXT_INPUT_EXISTS", false, { element.exists }) else {
+      return nil
+    }
+
+
+
+    return TextEntryTarget(
+      element: element,
+      refreshPoint: nil,
+      prefersFocusedElement: false,
+      fromTapWitness: true
+    )
+  }
+
+  func stabilizeTextInputBeforeTyping(
+    app: XCUIApplication,
+    target: XCUIElement?,
+    keyboardVisibleBeforeTap: Bool? = nil
+  ) -> TextEntryStabilization {
+#if os(tvOS)
+    return TextEntryStabilization(element: target, focusConfirmed: true)
+#else
+    let latest = target
+    let keyboardVisibleAtEntry = keyboardVisibleBeforeTap ?? isKeyboardVisible(app: app)
+    let deadline = Date().addingTimeInterval(TextEntryTiming.focusTimeout)
+    while Date() < deadline {
+      if let focused = focusedTextInput(app: app) {
+        return TextEntryStabilization(element: focused, focusConfirmed: true)
+      }
+
+
+      if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) {
+        return TextEntryStabilization(element: latest, focusConfirmed: true)
+      }
+      sleepFor(TextEntryTiming.pollInterval)
+    }
+    return TextEntryStabilization(element: latest, focusConfirmed: false)
+#endif
+  }
+
+  func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget {
+    guard let x, let y else {
+      let softwareKeyboardVisible = isKeyboardVisible(app: app)
+      if !softwareKeyboardVisible, let rememberedTarget = rememberedTextEntryTarget() {
+        return rememberedTarget
+      }
+
+
+
+
+      if softwareKeyboardVisible {
+        return TextEntryTarget(
+          element: focusedTextInput(app: app),
+          refreshPoint: nil,
+          prefersFocusedElement: true
+        )
+      }
+      let focused = waitForTextEntryReadiness(
+        app: app,
+        target: TextEntryTarget(
+          element: focusedTextInput(app: app),
+          refreshPoint: nil,
+          prefersFocusedElement: true
+        )
+      )
+      return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true)
+    }
+
+    let keyboardVisibleBeforeTap = isKeyboardVisible(app: app)
+    let target = textInputAt(app: app, x: x, y: y)
+    let requestedPoint = CGPoint(x: x, y: y)
+    if let target {
+      let frame = target.frame
+      if !frame.isEmpty {
+        _ = tapAt(app: app, x: frame.midX, y: frame.midY)
+      } else {
+        _ = tapAt(app: app, x: x, y: y)
+      }
+    } else {
+      _ = tapAt(app: app, x: x, y: y)
+    }
+
+
+
+    if keyboardVisibleBeforeTap, let target {
+      return TextEntryTarget(
+        element: target,
+        refreshPoint: textEntryRefreshPoint(for: target) ?? requestedPoint,
+        prefersFocusedElement: false
+      )
+    }
+    let stabilized = stabilizeTextInputBeforeTyping(
+      app: app,
+      target: target,
+      keyboardVisibleBeforeTap: keyboardVisibleBeforeTap
+    )
+    let readyTarget = TextEntryTarget(
+      element: stabilized.element ?? target,
+      refreshPoint: requestedPoint,
+      prefersFocusedElement: false
+    )
+    let concreteTargetReady = keyboardVisibleBeforeTap && readyTarget.element != nil
+    let element = stabilized.focusConfirmed || concreteTargetReady
+      ? (stabilized.element ?? target)
+      : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? target)
+    return TextEntryTarget(
+      element: element,
+      refreshPoint: textEntryRefreshPoint(for: element) ?? requestedPoint,
+      prefersFocusedElement: false
+    )
+  }
+
+  func focusTextInputForTextEntry(app: XCUIApplication, element: XCUIElement) -> TextEntryTarget {
+    let point = textEntryRefreshPoint(for: element)
+    let keyboardVisibleBeforeTap = isKeyboardVisible(app: app)
+    if let point {
+      _ = tapAt(app: app, x: point.x, y: point.y)
+    }
+
+
+    if keyboardVisibleBeforeTap {
+      return TextEntryTarget(
+        element: element,
+        refreshPoint: textEntryRefreshPoint(for: element) ?? point,
+        prefersFocusedElement: false
+      )
+    }
+    let stabilized = stabilizeTextInputBeforeTyping(
+      app: app,
+      target: element,
+      keyboardVisibleBeforeTap: keyboardVisibleBeforeTap
+    )
+    let readyTarget = TextEntryTarget(
+      element: stabilized.element ?? element,
+      refreshPoint: point,
+      prefersFocusedElement: false
+    )
+    let resolved = stabilized.focusConfirmed
+      ? (stabilized.element ?? element)
+      : (waitForTextEntryReadiness(app: app, target: readyTarget) ?? stabilized.element ?? element)
+    return TextEntryTarget(
+      element: resolved,
+      refreshPoint: textEntryRefreshPoint(for: resolved) ?? point,
+      prefersFocusedElement: false
+    )
+  }
+
+  private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? {
+    guard let element else {
+      return nil
+    }
+    let frame = element.frame
+    guard !frame.isEmpty else {
+      return nil
+    }
+    return CGPoint(x: frame.midX, y: frame.midY)
+  }
+
+
+
+
+
+
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift

@@ -0,0 +1,158 @@
+import XCTest
+
+
+
+
+
+extension RunnerTests {
+  func focusedTextInput(app: XCUIApplication) -> XCUIElement? {
+#if os(iOS)
+
+
+    return nil
+#else
+    return safely("FOCUSED_INPUT_QUERY") {
+      let candidates = app
+        .descendants(matching: .any)
+        .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
+        .allElementsBoundByIndex
+      for candidate in candidates where candidate.exists {
+        switch candidate.elementType {
+        case .textField, .secureTextField, .searchField, .textView:
+          return candidate
+        default:
+          continue
+        }
+      }
+      return nil
+    }
+#endif
+  }
+
+  func waitForTextEntryReadiness(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    timeout: TimeInterval = TextEntryTiming.readinessTimeout
+  ) -> XCUIElement? {
+#if os(iOS)
+    var latest = resolveTextEntryElement(app: app, target: target)
+    let keyboardVisibleAtEntry = isKeyboardVisible(app: app)
+    let deadline = Date().addingTimeInterval(timeout)
+    var hardwareKeyboardFallback = Date().addingTimeInterval(
+      min(TextEntryTiming.hardwareKeyboardFallbackTimeout, timeout)
+    )
+    var sawSoftwareKeyboard = false
+    while Date() < deadline {
+      if let focused = focusedTextInput(app: app) {
+        latest = focused
+        if isKeyboardVisible(app: app) {
+          return focused
+        }
+      }
+
+
+
+
+
+      if keyboardBecameVisible(app: app, wasVisibleAtEntry: keyboardVisibleAtEntry) {
+        return latest
+      }
+      sawSoftwareKeyboard = sawSoftwareKeyboard || keyboardElementExists(app: app)
+
+
+
+
+
+
+      if !sawSoftwareKeyboard, Date() >= hardwareKeyboardFallback, let candidate = latest {
+        if keyboardFocusConfirmed(app: app, element: candidate) {
+          return candidate
+        }
+        hardwareKeyboardFallback = Date().addingTimeInterval(
+          TextEntryTiming.hardwareKeyboardFallbackTimeout
+        )
+      }
+      sleepFor(TextEntryTiming.pollInterval)
+    }
+    return focusedTextInput(app: app) ?? latest
+#else
+    return resolveTextEntryElement(app: app, target: target)
+#endif
+  }
+
+  func waitForTextEntryReadinessAfterTap(app: XCUIApplication, element: XCUIElement) {
+#if os(iOS)
+    switch element.elementType {
+    case .textField, .secureTextField, .searchField, .textView:
+      if waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout) != nil {
+        return
+      }
+      let frame = element.frame
+      if !frame.isEmpty {
+        _ = tapAt(app: app, x: frame.midX, y: frame.midY)
+        _ = waitForFocusedTextInput(app: app, timeout: TextEntryTiming.readinessTimeout)
+      }
+    default:
+      return
+    }
+#endif
+  }
+
+  private func waitForFocusedTextInput(app: XCUIApplication, timeout: TimeInterval) -> XCUIElement? {
+    let deadline = Date().addingTimeInterval(timeout)
+    while Date() < deadline {
+      if let focused = focusedTextInput(app: app) {
+        return focused
+      }
+      sleepFor(TextEntryTiming.pollInterval)
+    }
+    return focusedTextInput(app: app)
+  }
+
+  func keyboardBecameVisible(app: XCUIApplication, wasVisibleAtEntry: Bool) -> Bool {
+    return !wasVisibleAtEntry && isKeyboardVisible(app: app)
+  }
+
+
+
+
+
+
+
+
+
+
+
+
+  func keyboardFocusConfirmed(app: XCUIApplication, element: XCUIElement) -> Bool {
+#if os(iOS)
+    return safely("TEXT_ENTRY_FOCUS_CONFIRMED", false) {
+
+
+
+      let frame = element.frame
+      guard !frame.isEmpty else {
+        return false
+      }
+      let focused = app
+        .descendants(matching: .any)
+        .matching(NSPredicate(format: "hasKeyboardFocus == 1"))
+        .firstMatch
+      guard focused.exists else {
+        return false
+      }
+      return focused.identifier == element.identifier && focused.frame == frame
+    }
+#else
+    return false
+#endif
+  }
+
+  private func keyboardElementExists(app: XCUIApplication) -> Bool {
+#if os(iOS)
+    return safely("KEYBOARD_EXISTS", false) { app.keyboards.firstMatch.exists }
+#else
+    return false
+#endif
+  }
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift

@@ -16,4 +16,32 @@ func isCoordinateTextInputCandidate(
 }
 
 extension RunnerTests {
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
 }

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift

@@ -0,0 +1,101 @@
+import XCTest
+
+final class TextInputProbeIssues {
+  let thread = Thread.current
+  var count = 0
+}
+
+enum TextInputProbeFailure: String {
+  case recordedIssue = "text_input_probe_recorded_issue"
+  case exception = "text_input_probe_exception"
+}
+
+enum TextInputProbeOutcome {
+  case matches([XCUIElement])
+  case absent
+  case unavailable(TextInputProbeFailure)
+}
+
+extension RunnerTests {
+  func textInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
+    textInputCandidatesAt(app: app, point: CGPoint(x: x, y: y)).first
+  }
+
+  func textInputCandidatesAt(app: XCUIApplication, point: CGPoint) -> [XCUIElement] {
+    safely("TEXT_INPUT_AT_POINT", []) {
+      queryTextInputs(app: app, point: point, shouldStop: { false })
+    }
+  }
+
+  func coordinateTapTextInputAt(app: XCUIApplication, x: Double, y: Double) -> XCUIElement? {
+    switch probeTextInputs(app: app, point: CGPoint(x: x, y: y)) {
+    case .matches(let elements):
+      return elements.first
+    case .absent:
+      return nil
+    case .unavailable:
+      return nil
+    }
+  }
+
+  func probeTextInputs(app: XCUIApplication, point: CGPoint) -> TextInputProbeOutcome {
+    precondition(Thread.isMainThread)
+    let issues = TextInputProbeIssues()
+    suppressedIssueLock.lock()
+    let previous = textInputProbeIssues
+    textInputProbeIssues = issues
+    suppressedIssueLock.unlock()
+    defer {
+      suppressedIssueLock.lock()
+      textInputProbeIssues = previous
+      suppressedIssueLock.unlock()
+    }
+    let (elements, exception) = catchingObjCException(fallback: []) {
+      queryTextInputs(app: app, point: point, shouldStop: { self.hasTextInputProbeIssues(issues) })
+    }
+    if hasTextInputProbeIssues(issues) { return .unavailable(.recordedIssue) }
+    if exception != nil { return .unavailable(.exception) }
+    return elements.isEmpty ? .absent : .matches(elements)
+  }
+
+  private func hasTextInputProbeIssues(_ scope: TextInputProbeIssues) -> Bool {
+    suppressedIssueLock.lock()
+    defer { suppressedIssueLock.unlock() }
+    return scope.count > 0
+  }
+
+  func containTextInputProbeIssue(_ issue: XCTIssue) -> Bool {
+    suppressedIssueLock.lock()
+    guard let scope = textInputProbeIssues, scope.thread === Thread.current else {
+      suppressedIssueLock.unlock()
+      return false
+    }
+    scope.count += 1
+    suppressedIssueLock.unlock()
+    NSLog("AGENT_DEVICE_RUNNER_TEXT_INPUT_PROBE_UNAVAILABLE issue=%@", issue.compactDescription)
+    return true
+  }
+
+  private func queryTextInputs(
+    app: XCUIApplication,
+    point: CGPoint,
+    shouldStop: () -> Bool
+  ) -> [XCUIElement] {
+    var candidates: [XCUIElement] = []
+    for query in [app.textFields, app.secureTextFields, app.searchFields, app.textViews] {
+      if shouldStop() { break }
+      candidates.append(contentsOf: query.allElementsBoundByIndex)
+
+
+
+
+
+
+    }
+    guard !shouldStop() else { return [] }
+    return candidates.filter { element in
+      guard !shouldStop(), element.exists else { return false }
+      return isCoordinateTextInputCandidate(enabled: element.isEnabled, frame: element.frame, point: point)
+    }.sorted(by: smallestElementFirst)
+  }
+}

package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift

@@ -1,7 +1,7 @@
 import XCTest
 
-// Text typing, verification, and repair. Kept separate from focus/readiness so each
-// text-entry policy remains a bounded review surface.
+
+
 extension RunnerTests {
   func typeTextReliably(
     app: XCUIApplication,
@@ -15,6 +15,36 @@ extension RunnerTests {
   ) -> TextEntryResult {
     let totalStartedAt = Date()
     guard !text.isEmpty else {
+
+
+
+
+      if repairMode == .replacement {
+        guard let clearTarget = resolveTextEntryElement(app: app, target: target) else {
+
+
+
+          logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode)
+          return TextEntryResult(
+            verified: nil,
+            repaired: false,
+            expectedText: "",
+            observedText: nil,
+            failure: .notFocused
+          )
+        }
+        clearTextInput(clearTarget)
+
+
+        let observed = editableTextValue(for: clearTarget, treatingPlaceholderAsEmpty: true)
+        logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode)
+        return TextEntryResult(
+          verified: observed.map { $0.isEmpty },
+          repaired: false,
+          expectedText: "",
+          observedText: observed
+        )
+      }
       logTextEntryPhase(commandId: commandId, phase: "total", startedAt: totalStartedAt, chars: 0, mode: repairMode)
       return TextEntryResult(verified: true, repaired: false, expectedText: "", observedText: "")
     }
@@ -65,9 +95,9 @@ extension RunnerTests {
       mode: repairMode
     )
 
-    // Dispatching text through XCTest without a resolved target or evidence of a focused
-    // responder records a test failure. That tears down the long-lived runner and turns a
-    // single invalid request into a restart cascade, so fail before entering that channel.
+
+
+
     guard initialTarget != nil || (activeTarget.prefersFocusedElement && isKeyboardVisible(app: app)) else {
       logTextEntryPhase(
         commandId: commandId,
@@ -104,20 +134,20 @@ extension RunnerTests {
       }
     }
 
-    func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, dispatched: Bool, failure: TextEntryFailure?) {
+    func typeIntoCurrentTarget(_ value: String) -> (element: XCUIElement?, failure: TextEntryFailure?) {
 #if os(iOS)
       if shouldUseSynthesizedFirstResponderType {
         guard let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) else {
-          return (nil, false, .notFocused)
+          return (nil, .notFocused)
         }
         textEntryRoute = "synthesized-first-responder"
         NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_ROUTE route=synthesized-first-responder")
         let textBefore = editableTextValue(for: currentTarget, treatingPlaceholderAsEmpty: true)
         switch synthesizer.enterText(app: app, text: value, replacingExistingText: false) {
         case .continueTyping:
-          // No refresh point: like the tap-witness target itself, the commit wait must observe
-          // only the element the tap selected, never rediscover a different field.
-          awaitSynthesizedFirstResponderCommit(
+
+
+          let commit = awaitSynthesizedFirstResponderCommit(
             app: app,
             target: TextEntryTarget(
               element: currentTarget,
@@ -128,9 +158,12 @@ extension RunnerTests {
             textBefore: textBefore,
             typedText: value
           )
-          return (currentTarget, true, nil)
+
+
+
+          return (currentTarget, Self.textEntryFailure(forCommitOutcome: commit))
         case .fallback:
-          return (nil, false, .synthesisUnavailable)
+          return (nil, .synthesisUnavailable)
         case .raise(let message):
           NSException(
             name: NSExceptionName.internalInconsistencyException,
@@ -142,7 +175,7 @@ extension RunnerTests {
       if let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) {
         textEntryRoute = "xctest-element"
         currentTarget.typeText(value)
-        return (currentTarget, true, nil)
+        return (currentTarget, nil)
       } else if activeTarget.prefersFocusedElement && isKeyboardVisible(app: app) {
 #if os(iOS)
         textEntryRoute = "synthesized-first-responder"
@@ -167,9 +200,9 @@ extension RunnerTests {
 #else
         app.typeText(value)
 #endif
-        return (resolveTextEntryElement(app: app, target: activeTarget), true, nil)
+        return (resolveTextEntryElement(app: app, target: activeTarget), nil)
       }
-      return (nil, false, .notFocused)
+      return (nil, .notFocused)
     }
 
     func dispatchFailureResult(_ failure: TextEntryFailure) -> TextEntryResult {
@@ -203,8 +236,8 @@ extension RunnerTests {
       let delayedTypeStartedAt = Date()
       for (index, character) in characters.enumerated() {
         let dispatch = typeIntoCurrentTarget(String(character))
-        guard dispatch.dispatched else {
-          return dispatchFailureResult(dispatch.failure ?? .notFocused)
+        if let failure = dispatch.failure {
+          return dispatchFailureResult(failure)
         }
         typedTarget = dispatch.element ?? typedTarget
         if index + 1 < characters.count {
@@ -252,8 +285,8 @@ extension RunnerTests {
       let firstCharacter = String(characters[0])
       let firstStartedAt = Date()
       let firstDispatch = typeIntoCurrentTarget(firstCharacter)
-      guard firstDispatch.dispatched else {
-        return dispatchFailureResult(firstDispatch.failure ?? .notFocused)
+      if let failure = firstDispatch.failure {
+        return dispatchFailureResult(failure)
       }
       var firstTypedTarget = firstDispatch.element
       logTextEntryPhase(
@@ -281,8 +314,8 @@ extension RunnerTests {
       let remainingText = String(characters.dropFirst())
       let remainingStartedAt = Date()
       let remainingDispatch = typeIntoCurrentTarget(remainingText)
-      guard remainingDispatch.dispatched else {
-        return dispatchFailureResult(remainingDispatch.failure ?? .notFocused)
+      if let failure = remainingDispatch.failure {
+        return dispatchFailureResult(failure)
       }
       firstTypedTarget = remainingDispatch.element ?? firstTypedTarget
       logTextEntryPhase(
@@ -296,8 +329,8 @@ extension RunnerTests {
     } else {
       let typeStartedAt = Date()
       let dispatch = typeIntoCurrentTarget(text)
-      guard dispatch.dispatched else {
-        return dispatchFailureResult(dispatch.failure ?? .notFocused)
+      if let failure = dispatch.failure {
+        return dispatchFailureResult(failure)
       }
       typedTarget = dispatch.element
       logTextEntryPhase(