agent-device 0.14.8 → 0.15.0

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift

@@ -17,6 +17,60 @@ extension RunnerTests {
     let referenceHeight: Double
   }
 
+  struct DragPoints {
+    let x: Double
+    let y: Double
+    let x2: Double
+    let y2: Double
+  }
+
+  struct SelectorElementMatch {
+    let element: XCUIElement?
+    let isAmbiguous: Bool
+  }
+
+  enum TextTypingRepairMode {
+    case none
+    case append
+    case replacement
+  }
+
+  enum TextEntryTiming {
+    static let focusTimeout: TimeInterval = 0.4
+    static let repairReadinessTimeout: TimeInterval = 1.0
+    static let readinessTimeout: TimeInterval = 2.0
+    static let hardwareKeyboardFallbackTimeout: TimeInterval = 0.35
+    static let pollInterval: TimeInterval = 0.02
+    static let warmupValueTimeout: TimeInterval = 0.4
+    static let verificationStabilityWindow: TimeInterval = 0.2
+  }
+
+  struct TextEntryResult {
+    let verified: Bool?
+    let repaired: Bool
+    let expectedText: String?
+    let observedText: String?
+  }
+
+  struct TextEntryTarget {
+    let element: XCUIElement?
+    let refreshPoint: CGPoint?
+    let prefersFocusedElement: Bool
+
+    func withElement(_ nextElement: XCUIElement?) -> TextEntryTarget {
+      guard let nextElement else {
+        return self
+      }
+      let frame = nextElement.frame
+      let point = frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY)
+      return TextEntryTarget(
+        element: nextElement,
+        refreshPoint: point,
+        prefersFocusedElement: prefersFocusedElement
+      )
+    }
+  }
+
   // MARK: - Navigation Gestures
 
   func tapInAppBackControl(app: XCUIApplication) -> Bool {
@@ -26,6 +80,9 @@ extension RunnerTests {
       return true
     }
     return false
+#elseif os(tvOS)
+    _ = pressTvRemote(.menu)
+    return true
 #else
     let buttons = app.navigationBars.buttons.allElementsBoundByIndex
     if let back = buttons.first(where: { $0.isHittable }) {
@@ -37,20 +94,26 @@ extension RunnerTests {
   }
 
   func performBackGesture(app: XCUIApplication) {
-    if pressTvRemoteMenuIfAvailable() {
+    if pressTvRemote(.menu) {
       return
     }
+    performCoordinateBackGesture(app: app)
+  }
+
+  private func performCoordinateBackGesture(app: XCUIApplication) {
+#if !os(tvOS)
     let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
     let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.05, dy: 0.5))
     let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.8, dy: 0.5))
     start.press(forDuration: 0.05, thenDragTo: end)
+#endif
   }
 
   func performSystemBackAction(app: XCUIApplication) -> Bool {
 #if os(macOS)
     return false
 #else
-    if pressTvRemoteMenuIfAvailable() {
+    if pressTvRemote(.menu) {
       return true
     }
     performBackGesture(app: app)
@@ -59,20 +122,28 @@ extension RunnerTests {
   }
 
   func performAppSwitcherGesture(app: XCUIApplication) {
-    if performTvRemoteAppSwitcherIfAvailable() {
+    if pressTvRemote(.home) {
+      sleepFor(resolveTvRemoteDoublePressDelay())
+      _ = pressTvRemote(.home)
       return
     }
+    performCoordinateAppSwitcherGesture(app: app)
+  }
+
+  private func performCoordinateAppSwitcherGesture(app: XCUIApplication) {
+#if !os(tvOS)
     let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
     let start = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.99))
     let end = target.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.7))
     start.press(forDuration: 0.6, thenDragTo: end)
+#endif
   }
 
   func pressHomeButton() {
 #if os(macOS)
     return
 #else
-    if pressTvRemoteHomeIfAvailable() {
+    if pressTvRemote(.home) {
       return
     }
     XCUIDevice.shared.press(.home)
@@ -80,7 +151,7 @@ extension RunnerTests {
   }
 
   func rotateDevice(to orientationName: String) -> Bool {
-#if os(macOS)
+#if os(macOS) || os(tvOS)
     return false
 #else
     switch orientationName {
@@ -100,52 +171,82 @@ extension RunnerTests {
 #endif
   }
 
-  private func pressTvRemoteMenuIfAvailable() -> Bool {
-#if os(tvOS)
-    XCUIRemote.shared.press(.menu)
-    return true
-#else
-    return false
-#endif
+  func findElement(app: XCUIApplication, text: String) -> XCUIElement? {
+    let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text)
+    let element = app.descendants(matching: .any).matching(predicate).firstMatch
+    return element.exists ? element : nil
   }
 
-  private func pressTvRemoteHomeIfAvailable() -> Bool {
-#if os(tvOS)
-    XCUIRemote.shared.press(.home)
-    return true
-#else
-    return false
-#endif
-  }
+  func findElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> SelectorElementMatch {
+    let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines)
+    guard !value.isEmpty else {
+      return SelectorElementMatch(element: nil, isAmbiguous: false)
+    }
+    let predicate: NSPredicate
+    switch selectorKey {
+    case "id":
+      predicate = NSPredicate(format: "identifier ==[c] %@", value)
+    case "label":
+      predicate = NSPredicate(format: "label ==[c] %@", value)
+    case "value":
+      predicate = NSPredicate(format: "value ==[c] %@", value)
+    case "text":
+      predicate = NSPredicate(format: "label ==[c] %@ OR identifier ==[c] %@ OR value ==[c] %@", value, value, value)
+    default:
+      return SelectorElementMatch(element: nil, isAmbiguous: false)
+    }
 
-  private func performTvRemoteAppSwitcherIfAvailable() -> Bool {
-#if os(tvOS)
-    XCUIRemote.shared.press(.home)
-    sleepFor(resolveTvRemoteDoublePressDelay())
-    XCUIRemote.shared.press(.home)
-    return true
-#else
-    return false
-#endif
+    var matchedElement: XCUIElement?
+    let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex
+    for element in matches where element.exists {
+      guard element.isHittable else {
+        continue
+      }
+      guard matchedElement == nil else {
+        return SelectorElementMatch(element: nil, isAmbiguous: true)
+      }
+      matchedElement = element
+    }
+    return SelectorElementMatch(element: matchedElement, isAmbiguous: false)
   }
 
-  private func resolveTvRemoteDoublePressDelay() -> TimeInterval {
-    guard
-      let raw = ProcessInfo.processInfo.environment["AGENT_DEVICE_TV_REMOTE_DOUBLE_PRESS_DELAY_MS"],
-      !raw.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
-    else {
-      return tvRemoteDoublePressDelayDefault
+  func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response {
+    let match = findElement(app: app, selectorKey: selectorKey, selectorValue: selectorValue)
+    if match.isAmbiguous {
+      return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements"))
     }
-    guard let parsedMs = Double(raw), parsedMs >= 0 else {
-      return tvRemoteDoublePressDelayDefault
+    guard let element = match.element else {
+      return Response(ok: true, data: DataPayload(found: false, nodes: []))
     }
-    return min(parsedMs, 1000) / 1000.0
-  }
 
-  func findElement(app: XCUIApplication, text: String) -> XCUIElement? {
-    let predicate = NSPredicate(format: "label CONTAINS[c] %@ OR identifier CONTAINS[c] %@ OR value CONTAINS[c] %@", text, text, text)
-    let element = app.descendants(matching: .any).matching(predicate).firstMatch
-    return element.exists ? element : nil
+    let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
+    let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
+    let valueText = String(describing: element.value ?? "")
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+    let node = SnapshotNode(
+      index: 0,
+      type: elementTypeName(element.elementType),
+      label: label.isEmpty ? nil : label,
+      identifier: identifier.isEmpty ? nil : identifier,
+      value: valueText.isEmpty ? nil : valueText,
+      rect: snapshotRect(from: element.frame),
+      enabled: element.isEnabled,
+      focused: nil,
+      selected: element.isSelected ? true : nil,
+      hittable: element.isHittable,
+      depth: 0,
+      parentIndex: nil,
+      hiddenContentAbove: nil,
+      hiddenContentBelow: nil
+    )
+    return Response(
+      ok: true,
+      data: DataPayload(
+        text: readableText(for: element),
+        found: true,
+        nodes: [node]
+      )
+    )
   }
 
   func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? {
@@ -183,7 +284,9 @@ extension RunnerTests {
   }
 
   func clearTextInput(_ element: XCUIElement) {
+#if !os(tvOS)
     moveCaretToEnd(element: element)
+#endif
     let count = estimatedDeleteCount(for: element)
     let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count)
     element.typeText(deletes)
@@ -257,9 +360,458 @@ extension RunnerTests {
     return focused
   }
 
+  func stabilizeTextInputBeforeTyping(app: XCUIApplication, target: XCUIElement?) -> XCUIElement? {
+#if os(tvOS)
+    return target
+#else
+    let latest = target
+    let deadline = Date().addingTimeInterval(TextEntryTiming.focusTimeout)
+    while Date() < deadline {
+      if let focused = focusedTextInput(app: app) {
+        return focused
+      }
+      sleepFor(TextEntryTiming.pollInterval)
+    }
+    return latest
+#endif
+  }
+
+  func focusTextInputForTextEntry(app: XCUIApplication, x: Double?, y: Double?) -> TextEntryTarget {
+    guard let x, let y else {
+      let focused = waitForTextEntryReadiness(
+        app: app,
+        target: TextEntryTarget(
+          element: focusedTextInput(app: app),
+          refreshPoint: nil,
+          prefersFocusedElement: true
+        )
+      )
+      return TextEntryTarget(element: focused, refreshPoint: nil, prefersFocusedElement: true)
+    }
+
+    let target = textInputAt(app: app, x: x, y: y)
+    let requestedPoint = CGPoint(x: x, y: y)
+    if let target {
+      let frame = target.frame
+      if !frame.isEmpty {
+        _ = tapAt(app: app, x: frame.midX, y: frame.midY)
+      } else {
+        _ = tapAt(app: app, x: x, y: y)
+      }
+    } else {
+      _ = tapAt(app: app, x: x, y: y)
+    }
+    let stabilized = stabilizeTextInputBeforeTyping(app: app, target: target)
+    let element = waitForTextEntryReadiness(
+      app: app,
+      target: TextEntryTarget(
+        element: stabilized ?? target,
+        refreshPoint: requestedPoint,
+        prefersFocusedElement: false
+      )
+    ) ?? stabilized ?? target
+    return TextEntryTarget(
+      element: element,
+      refreshPoint: textEntryRefreshPoint(for: element) ?? requestedPoint,
+      prefersFocusedElement: false
+    )
+  }
+
+  func resolveTextEntryMode(_ command: Command) -> TextTypingRepairMode {
+    switch command.textEntryMode {
+    case "append":
+      return .append
+    case "replace":
+      return .replacement
+    default:
+      return command.clearFirst == true ? .replacement : .none
+    }
+  }
+
+  func typeTextReliably(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    text: String,
+    delaySeconds: Double,
+    repairMode: TextTypingRepairMode = .none
+  ) -> TextEntryResult {
+    guard !text.isEmpty else {
+      return TextEntryResult(verified: true, repaired: false, expectedText: "", observedText: "")
+    }
+    var activeTarget = target
+    let initialTarget = resolveTextEntryElement(app: app, target: activeTarget)
+    activeTarget = activeTarget.withElement(initialTarget)
+    let currentText = editableTextValue(for: initialTarget, treatingPlaceholderAsEmpty: true)
+    let initialText = repairMode == .append ? currentText : nil
+    let expectedText = expectedTextEntryValue(typedText: text, mode: repairMode, initialText: initialText)
+
+    if repairMode == .replacement {
+      guard let replacementTarget = initialTarget else {
+        return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil)
+      }
+      if currentText == nil || currentText?.isEmpty == false {
+        clearTextInput(replacementTarget)
+        activeTarget = activeTarget.withElement(replacementTarget)
+      }
+    }
+
+    func typeIntoCurrentTarget(_ value: String) -> XCUIElement? {
+      if let currentTarget = resolveTextEntryElement(app: app, target: activeTarget) {
+        app.typeText(value)
+        return currentTarget
+      } else {
+        app.typeText(value)
+        return resolveTextEntryElement(app: app, target: activeTarget)
+      }
+    }
+
+    func waitForWarmupValue(_ expectedValue: String?, target: TextEntryTarget) {
+      guard let expectedValue else {
+        sleepFor(TextEntryTiming.pollInterval)
+        return
+      }
+      let deadline = Date().addingTimeInterval(TextEntryTiming.warmupValueTimeout)
+      while Date() < deadline {
+        if editableTextValue(for: resolveTextEntryElement(app: app, target: target)) == expectedValue {
+          return
+        }
+        sleepFor(TextEntryTiming.pollInterval)
+      }
+    }
+
+    let characters = Array(text)
+    if delaySeconds > 0 && characters.count > 1 {
+      var typedTarget: XCUIElement?
+      for (index, character) in characters.enumerated() {
+        typedTarget = typeIntoCurrentTarget(String(character)) ?? typedTarget
+        if index + 1 < characters.count {
+          sleepFor(delaySeconds)
+        }
+      }
+      if repairMode == .none {
+        return TextEntryResult(verified: nil, repaired: false, expectedText: nil, observedText: nil)
+      }
+      let repairResult = repairTextEntryIfNeeded(
+        app: app,
+        target: activeTarget.withElement(typedTarget),
+        expectedText: expectedText,
+        repairMode: repairMode
+      )
+      return verifyTextEntry(
+        app: app,
+        target: activeTarget.withElement(typedTarget),
+        expectedText: expectedText,
+        repaired: repairResult.repaired
+      )
+    }
+
+    let typedTarget: XCUIElement?
+    if repairMode != .none && characters.count > 1 {
+      let firstCharacter = String(characters[0])
+      var firstTypedTarget = typeIntoCurrentTarget(firstCharacter)
+      activeTarget = activeTarget.withElement(firstTypedTarget)
+      let warmupExpectedText = expectedTextEntryValue(
+        typedText: firstCharacter,
+        mode: repairMode,
+        initialText: initialText
+      )
+      waitForWarmupValue(warmupExpectedText, target: activeTarget)
+      let remainingText = String(characters.dropFirst())
+      firstTypedTarget = typeIntoCurrentTarget(remainingText) ?? firstTypedTarget
+      typedTarget = firstTypedTarget
+    } else {
+      typedTarget = typeIntoCurrentTarget(text)
+    }
+    if repairMode == .none {
+      return TextEntryResult(verified: nil, repaired: false, expectedText: nil, observedText: nil)
+    }
+    let repairResult = repairTextEntryIfNeeded(
+      app: app,
+      target: activeTarget.withElement(typedTarget),
+      expectedText: expectedText,
+      repairMode: repairMode
+    )
+    return verifyTextEntry(
+      app: app,
+      target: activeTarget.withElement(typedTarget),
+      expectedText: expectedText,
+      repaired: repairResult.repaired
+    )
+  }
+
+  private func repairTextEntryIfNeeded(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    expectedText: String?,
+    repairMode: TextTypingRepairMode
+  ) -> TextEntryResult {
+#if os(iOS)
+    guard let targetElement = resolveTextEntryElement(app: app, target: target) else {
+      return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil)
+    }
+    guard let expectedText else {
+      let observedText = editableTextValue(for: targetElement)
+      return TextEntryResult(verified: nil, repaired: false, expectedText: nil, observedText: observedText)
+    }
+    guard shouldRepairTextEntry(
+      app: app,
+      target: target,
+      expectedText: expectedText,
+      repairMode: repairMode
+    ) else {
+      return verifyTextEntry(app: app, target: target, expectedText: expectedText, repaired: false)
+    }
+
+    guard let repairTarget = resolveTextEntryElement(app: app, target: target) else {
+      return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil)
+    }
+    let observedText = editableTextValue(for: repairTarget) ?? ""
+    NSLog(
+      "AGENT_DEVICE_RUNNER_REPAIR_TEXT_ENTRY expectedLength=%d observedLength=%d",
+      expectedText.count,
+      observedText.count
+    )
+    clearTextInput(repairTarget)
+    app.typeText(expectedText)
+    return verifyTextEntry(app: app, target: target, expectedText: expectedText, repaired: true)
+#else
+    return TextEntryResult(verified: nil, repaired: false, expectedText: expectedText, observedText: nil)
+#endif
+  }
+
+  private func verifyTextEntry(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    expectedText: String?,
+    repaired: Bool
+  ) -> TextEntryResult {
+    let targetElement = resolveTextEntryElement(app: app, target: target)
+    guard let expectedText else {
+      return TextEntryResult(
+        verified: nil,
+        repaired: repaired,
+        expectedText: nil,
+        observedText: editableTextValue(for: targetElement)
+      )
+    }
+    guard let observedText = editableTextValue(for: targetElement) else {
+      return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil)
+    }
+    guard observedText == expectedText else {
+      return TextEntryResult(
+        verified: false,
+        repaired: repaired,
+        expectedText: expectedText,
+        observedText: observedText
+      )
+    }
+    let stableDeadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow)
+    var latestObservedText = observedText
+    while Date() < stableDeadline {
+      sleepFor(TextEntryTiming.pollInterval)
+      guard let nextObservedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else {
+        return TextEntryResult(verified: nil, repaired: repaired, expectedText: expectedText, observedText: nil)
+      }
+      latestObservedText = nextObservedText
+      guard nextObservedText == expectedText else {
+        return TextEntryResult(
+          verified: false,
+          repaired: repaired,
+          expectedText: expectedText,
+          observedText: nextObservedText
+        )
+      }
+    }
+    return TextEntryResult(
+      verified: true,
+      repaired: repaired,
+      expectedText: expectedText,
+      observedText: latestObservedText
+    )
+  }
+
+  private func expectedTextEntryValue(
+    typedText: String,
+    mode: TextTypingRepairMode,
+    initialText: String?
+  ) -> String? {
+    switch mode {
+    case .none:
+      return nil
+    case .append:
+      guard let initialText else {
+        return nil
+      }
+      return initialText + typedText
+    case .replacement:
+      return typedText
+    }
+  }
+
+  private func shouldRepairTextEntry(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    expectedText: String,
+    repairMode: TextTypingRepairMode
+  ) -> Bool {
+#if os(iOS)
+    var latestObservedText: String?
+    let deadline = Date().addingTimeInterval(TextEntryTiming.verificationStabilityWindow)
+    repeat {
+      guard let observedText = editableTextValue(for: resolveTextEntryElement(app: app, target: target)) else {
+        return false
+      }
+      if observedText == expectedText {
+        return false
+      }
+      latestObservedText = observedText
+      if !isRepairableTextEntryMismatch(
+        observedText: observedText,
+        expectedText: expectedText,
+        repairMode: repairMode
+      ) {
+        return false
+      }
+      sleepFor(TextEntryTiming.pollInterval)
+    } while Date() < deadline
+
+    guard let latestObservedText else {
+      return false
+    }
+    guard latestObservedText != expectedText else {
+      return false
+    }
+    return isRepairableTextEntryMismatch(
+      observedText: latestObservedText,
+      expectedText: expectedText,
+      repairMode: repairMode
+    )
+#else
+    return false
+#endif
+  }
+
+  private func isRepairableTextEntryMismatch(
+    observedText: String,
+    expectedText: String,
+    repairMode: TextTypingRepairMode
+  ) -> Bool {
+    guard observedText != expectedText else {
+      return false
+    }
+    if repairMode == .replacement {
+      return true
+    }
+    return observedText.isEmpty || isLikelyDroppedCharacterTextEntryMismatch(
+      observedText: observedText,
+      expectedText: expectedText
+    )
+  }
+
+  private func isLikelyDroppedCharacterTextEntryMismatch(observedText: String, expectedText: String) -> Bool {
+    guard observedText.count < expectedText.count else {
+      return false
+    }
+    let missingCharacterCount = expectedText.count - observedText.count
+    guard missingCharacterCount <= max(2, expectedText.count / 4) else {
+      return false
+    }
+    var expectedIndex = expectedText.startIndex
+    for character in observedText {
+      guard let matchIndex = expectedText[expectedIndex...].firstIndex(of: character) else {
+        return false
+      }
+      expectedIndex = expectedText.index(after: matchIndex)
+    }
+    return true
+  }
+
+  private func resolveTextEntryElement(app: XCUIApplication, target: TextEntryTarget) -> XCUIElement? {
+    if target.prefersFocusedElement {
+      if let focused = focusedTextInput(app: app) {
+        return focused
+      }
+      if let element = target.element, element.exists {
+        return element
+      }
+    } else {
+      if let element = target.element, element.exists {
+        return element
+      }
+    }
+    if let refreshPoint = target.refreshPoint,
+       let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y) {
+      return refreshed
+    }
+    if let focused = focusedTextInput(app: app) {
+      return focused
+    }
+    return nil
+  }
+
+  private func waitForTextEntryReadiness(
+    app: XCUIApplication,
+    target: TextEntryTarget,
+    timeout: TimeInterval = TextEntryTiming.readinessTimeout
+  ) -> XCUIElement? {
+#if os(iOS)
+    var latest = resolveTextEntryElement(app: app, target: target)
+    let deadline = Date().addingTimeInterval(timeout)
+    let hardwareKeyboardFallback = Date().addingTimeInterval(
+      min(TextEntryTiming.hardwareKeyboardFallbackTimeout, timeout)
+    )
+    var sawSoftwareKeyboard = false
+    while Date() < deadline {
+      if let focused = focusedTextInput(app: app) {
+        latest = focused
+        if isKeyboardVisible(app: app) {
+          return focused
+        }
+      }
+      sawSoftwareKeyboard = sawSoftwareKeyboard || keyboardElementExists(app: app)
+      if !sawSoftwareKeyboard && Date() >= hardwareKeyboardFallback && latest != nil {
+        return latest
+      }
+      sleepFor(TextEntryTiming.pollInterval)
+    }
+    return focusedTextInput(app: app) ?? latest
+#else
+    return resolveTextEntryElement(app: app, target: target)
+#endif
+  }
+
+  private func textEntryRefreshPoint(for element: XCUIElement?) -> CGPoint? {
+    guard let element else {
+      return nil
+    }
+    let frame = element.frame
+    guard !frame.isEmpty else {
+      return nil
+    }
+    return CGPoint(x: frame.midX, y: frame.midY)
+  }
+
   func isKeyboardVisible(app: XCUIApplication) -> Bool {
-    let keyboard = app.keyboards.firstMatch
-    return keyboard.exists && !keyboard.frame.isEmpty
+    return visibleKeyboardFrame(app: app) != nil
+  }
+
+  private func keyboardElementExists(app: XCUIApplication) -> Bool {
+#if os(iOS)
+    var exists = false
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      exists = app.keyboards.firstMatch.exists
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_KEYBOARD_EXISTS_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return false
+    }
+    return exists
+#else
+    return false
+#endif
   }
 
   func dismissKeyboard(app: XCUIApplication) -> (wasVisible: Bool, dismissed: Bool, visible: Bool) {
@@ -268,6 +820,12 @@ extension RunnerTests {
       return (wasVisible: false, dismissed: false, visible: false)
     }
 
+#if os(tvOS)
+    _ = pressTvRemote(.menu)
+    sleepFor(0.2)
+    let visible = isKeyboardVisible(app: app)
+    return (wasVisible: true, dismissed: !visible, visible: visible)
+#else
     let keyboard = app.keyboards.firstMatch
     keyboard.swipeDown()
     sleepFor(0.2)
@@ -282,10 +840,16 @@ extension RunnerTests {
     }
 
     return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app))
+#endif
   }
 
   private func tapKeyboardDismissControl(app: XCUIApplication) -> Bool {
-    let keyboardFrame = app.keyboards.firstMatch.frame
+#if os(tvOS)
+    return false
+#else
+    guard let keyboardFrame = visibleKeyboardFrame(app: app) else {
+      return false
+    }
     for label in ["Hide keyboard", "Dismiss keyboard", "Done"] {
       let candidates = [
         app.keyboards.buttons[label],
@@ -313,6 +877,7 @@ extension RunnerTests {
       }
     }
     return false
+#endif
   }
 
   private func isKeyboardAccessoryControl(_ element: XCUIElement, keyboardFrame: CGRect) -> Bool {
@@ -324,6 +889,9 @@ extension RunnerTests {
   }
 
   private func moveCaretToEnd(element: XCUIElement) {
+#if os(tvOS)
+    return
+#else
     let frame = element.frame
     guard !frame.isEmpty else {
       element.tap()
@@ -334,15 +902,53 @@ extension RunnerTests {
       CGVector(dx: max(2, frame.width - 4), dy: max(2, frame.height / 2))
     )
     target.tap()
+#endif
   }
 
   private func estimatedDeleteCount(for element: XCUIElement) -> Int {
-    let valueText = String(describing: element.value ?? "")
-      .trimmingCharacters(in: .whitespacesAndNewlines)
+    let valueText = normalizedElementText(element.value)
     let base = valueText.isEmpty ? 24 : (valueText.count + 8)
     return max(24, min(120, base))
   }
 
+  private func normalizedElementText(_ value: Any?) -> String {
+    String(describing: value ?? "")
+      .trimmingCharacters(in: .whitespacesAndNewlines)
+  }
+
+  private func editableTextValue(
+    for element: XCUIElement?,
+    treatingPlaceholderAsEmpty: Bool = false
+  ) -> String? {
+    guard let element else {
+      return nil
+    }
+    switch element.elementType {
+    case .textField, .searchField, .textView:
+      let value = String(describing: element.value ?? "")
+      if treatingPlaceholderAsEmpty && isPlaceholderValue(value, for: element) {
+        return ""
+      }
+      return value
+    case .secureTextField:
+      return nil
+    default:
+      return nil
+    }
+  }
+
+  private func isPlaceholderValue(_ value: String, for element: XCUIElement) -> Bool {
+    let normalizedValue = value.trimmingCharacters(in: .whitespacesAndNewlines)
+    guard !normalizedValue.isEmpty else {
+      return false
+    }
+    guard let placeholder = element.placeholderValue?.trimmingCharacters(in: .whitespacesAndNewlines),
+          !placeholder.isEmpty else {
+      return false
+    }
+    return normalizedValue == placeholder
+  }
+
   private func readableText(for element: XCUIElement) -> String? {
     let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines)
     let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines)
@@ -379,49 +985,64 @@ extension RunnerTests {
     return element.exists ? element : nil
   }
 
-  func tapAt(app: XCUIApplication, x: Double, y: Double) {
-    let coordinate = interactionCoordinate(app: app, x: x, y: y)
-    coordinate.tap()
+  func tapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
+    if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "tap") {
+      return outcome
+    }
+    return performCoordinateTap(app: app, x: x, y: y)
   }
 
   func mouseClickAt(app: XCUIApplication, x: Double, y: Double, button: String) throws {
+#if os(macOS)
     let coordinate = interactionCoordinate(app: app, x: x, y: y)
-    #if os(macOS)
-      switch button {
-      case "primary":
-        coordinate.tap()
-      case "secondary":
-        coordinate.rightClick()
-      case "middle":
-        throw NSError(
-          domain: "AgentDeviceRunner",
-          code: 1,
-          userInfo: [NSLocalizedDescriptionKey: "middle mouse button is not supported"]
-        )
-      default:
-        throw NSError(
-          domain: "AgentDeviceRunner",
-          code: 1,
-          userInfo: [NSLocalizedDescriptionKey: "unsupported mouse button: \(button)"]
-        )
-      }
-    #else
+    switch button {
+    case "primary":
+      coordinate.tap()
+    case "secondary":
+      coordinate.rightClick()
+    case "middle":
       throw NSError(
         domain: "AgentDeviceRunner",
         code: 1,
-        userInfo: [NSLocalizedDescriptionKey: "mouseClick is only supported on macOS"]
+        userInfo: [NSLocalizedDescriptionKey: "middle mouse button is not supported"]
       )
-    #endif
+    default:
+      throw NSError(
+        domain: "AgentDeviceRunner",
+        code: 1,
+        userInfo: [NSLocalizedDescriptionKey: "unsupported mouse button: \(button)"]
+      )
+    }
+#elseif os(tvOS)
+    throw NSError(
+      domain: "AgentDeviceRunner",
+      code: 1,
+      userInfo: [NSLocalizedDescriptionKey: "mouseClick is not supported on tvOS"]
+    )
+#else
+    throw NSError(
+      domain: "AgentDeviceRunner",
+      code: 1,
+      userInfo: [NSLocalizedDescriptionKey: "mouseClick is only supported on macOS"]
+    )
+#endif
   }
 
-  func doubleTapAt(app: XCUIApplication, x: Double, y: Double) {
-    let coordinate = interactionCoordinate(app: app, x: x, y: y)
-    coordinate.doubleTap()
+  func doubleTapAt(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
+    if let outcome = selectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), action: "double tap") {
+      guard case .performed = outcome else { return outcome }
+      sleepFor(0.1)
+      _ = pressTvRemote(.select)
+      return .performed
+    }
+    return performCoordinateDoubleTap(app: app, x: x, y: y)
   }
 
-  func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) {
-    let coordinate = interactionCoordinate(app: app, x: x, y: y)
-    coordinate.press(forDuration: duration)
+  func longPressAt(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome {
+    if let outcome = longSelectFocusedTvElement(app: app, point: CGPoint(x: x, y: y), duration: duration) {
+      return outcome
+    }
+    return performCoordinateLongPress(app: app, x: x, y: y, duration: duration)
   }
 
   func dragAt(
@@ -431,10 +1052,79 @@ extension RunnerTests {
     x2: Double,
     y2: Double,
     holdDuration: TimeInterval
-  ) {
-    let start = interactionCoordinate(app: app, x: x, y: y)
-    let end = interactionCoordinate(app: app, x: x2, y: y2)
-    start.press(forDuration: holdDuration, thenDragTo: end)
+  ) -> RunnerInteractionOutcome {
+    // tvOS has no coordinate drag. Preserve the direction as a focus move.
+    let dx = x2 - x
+    let dy = y2 - y
+    let button: TvRemoteButton = abs(dx) > abs(dy)
+      ? (dx > 0 ? .right : .left)
+      : (dy > 0 ? .down : .up)
+    if pressTvRemote(button) {
+      return .performed
+    }
+    return performCoordinateDrag(app: app, x: x, y: y, x2: x2, y2: y2, holdDuration: holdDuration)
+  }
+
+  func keyboardAvoidingDragPoints(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double
+  ) -> DragPoints {
+    let original = DragPoints(x: x, y: y, x2: x2, y2: y2)
+#if os(iOS)
+    guard let keyboardFrame = visibleKeyboardFrame(app: app) else {
+      return original
+    }
+    let minX = min(x, x2)
+    let minY = min(y, y2)
+    let gestureBounds = CGRect(
+      x: CGFloat(minX),
+      y: CGFloat(minY),
+      width: CGFloat(max(abs(x2 - x), 1)),
+      height: CGFloat(max(abs(y2 - y), 1))
+    )
+    guard gestureBounds.intersects(keyboardFrame) else {
+      return original
+    }
+
+    let window = app.windows.firstMatch
+    let appFrame = window.exists && !window.frame.isEmpty ? window.frame : app.frame
+    guard !appFrame.isEmpty else {
+      return original
+    }
+
+    let padding: Double = 12
+    let targetMaxY = Double(keyboardFrame.minY) - padding
+    let currentMaxY = max(y, y2)
+    let shift = currentMaxY - targetMaxY
+    guard shift > 0 else {
+      return original
+    }
+
+    let adjustedY = y - shift
+    let adjustedY2 = y2 - shift
+    guard min(adjustedY, adjustedY2) >= Double(appFrame.minY) + padding else {
+      return original
+    }
+
+    NSLog(
+      "AGENT_DEVICE_RUNNER_KEYBOARD_AVOIDING_DRAG from=(%.1f,%.1f)->(%.1f,%.1f) adjusted=(%.1f,%.1f)->(%.1f,%.1f) keyboardMinY=%.1f",
+      x,
+      y,
+      x2,
+      y2,
+      x,
+      adjustedY,
+      x2,
+      adjustedY2,
+      Double(keyboardFrame.minY)
+    )
+    return DragPoints(x: x, y: adjustedY, x2: x2, y2: adjustedY2)
+#else
+    return original
+#endif
   }
 
   func resolvedTouchVisualizationFrame(app: XCUIApplication, x: Double, y: Double) -> TouchVisualizationFrame {
@@ -471,23 +1161,71 @@ extension RunnerTests {
 
   func resolvedTouchReferenceFrame(app: XCUIApplication, appFrame: CGRect) -> CGRect {
     let window = app.windows.firstMatch
-    let windowFrame = window.frame
-    if window.exists && !windowFrame.isEmpty {
-      return windowFrame
+    if window.exists {
+      let windowFrame = window.frame
+      if !windowFrame.isEmpty {
+        return frameAvoidingKeyboard(app: app, frame: windowFrame)
+      }
     }
     if !appFrame.isEmpty {
-      return appFrame
+      return frameAvoidingKeyboard(app: app, frame: appFrame)
     }
     return CGRect(x: 0, y: 0, width: 0, height: 0)
   }
 
+  private func frameAvoidingKeyboard(app: XCUIApplication, frame: CGRect) -> CGRect {
+#if os(iOS)
+    guard let keyboardFrame = visibleKeyboardFrame(app: app), !frame.isEmpty else {
+      return frame
+    }
+    let intersection = frame.intersection(keyboardFrame)
+    guard !intersection.isNull && intersection.height > 0 else {
+      return frame
+    }
+    let keyboardCoverage = intersection.width / max(frame.width, 1)
+    guard keyboardCoverage >= 0.5 else {
+      return frame
+    }
+    let safeHeight = keyboardFrame.minY - frame.minY
+    guard safeHeight >= frame.height * 0.25 else {
+      return frame
+    }
+    return CGRect(x: frame.minX, y: frame.minY, width: frame.width, height: safeHeight)
+#else
+    return frame
+#endif
+  }
+
+  private func visibleKeyboardFrame(app: XCUIApplication) -> CGRect? {
+#if os(iOS)
+    var frame: CGRect?
+    let exceptionMessage = RunnerObjCExceptionCatcher.catchException({
+      let keyboard = app.keyboards.firstMatch
+      guard keyboard.exists else { return }
+      let keyboardFrame = keyboard.frame
+      guard !keyboardFrame.isEmpty else { return }
+      frame = keyboardFrame
+    })
+    if let exceptionMessage {
+      NSLog(
+        "AGENT_DEVICE_RUNNER_KEYBOARD_FRAME_IGNORED_EXCEPTION=%@",
+        exceptionMessage
+      )
+      return nil
+    }
+    return frame
+#else
+    return nil
+#endif
+  }
+
   func runSeries(count: Int, pauseMs: Double, operation: (Int) -> Void) {
     let total = max(count, 1)
     let pause = max(pauseMs, 0)
     for idx in 0..<total {
       operation(idx)
       if idx < total - 1 && pause > 0 {
-        Thread.sleep(forTimeInterval: pause / 1000.0)
+        sleepFor(pause / 1000.0)
       }
     }
   }
@@ -510,26 +1248,28 @@ extension RunnerTests {
   }
 
   private func performTvRemoteSwipeIfAvailable(direction: String) -> Bool {
-#if os(tvOS)
     switch direction {
     case "up":
-      XCUIRemote.shared.press(.up)
+      return pressTvRemote(.up)
     case "down":
-      XCUIRemote.shared.press(.down)
+      return pressTvRemote(.down)
     case "left":
-      XCUIRemote.shared.press(.left)
+      return pressTvRemote(.left)
     case "right":
-      XCUIRemote.shared.press(.right)
+      return pressTvRemote(.right)
     default:
       return false
     }
-    return true
-#else
-    return false
-#endif
   }
 
-  func pinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) {
+  func pinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
+    return performCoordinatePinch(app: app, scale: scale, x: x, y: y)
+  }
+
+  private func performCoordinatePinch(app: XCUIApplication, scale: Double, x: Double?, y: Double?) -> RunnerInteractionOutcome {
+#if os(tvOS)
+    return .unsupported("pinch is not supported on tvOS")
+#else
     let target = app.windows.firstMatch.exists ? app.windows.firstMatch : app
 
     // Use double-tap + drag gesture for reliable map zoom
@@ -560,6 +1300,8 @@ extension RunnerTests {
 
     // Immediately press and drag (second tap + drag)
     center.press(forDuration: 0.05, thenDragTo: endPoint)
+    return .performed
+#endif
   }
 
   private func interactionRoot(app: XCUIApplication) -> XCUIElement {
@@ -570,6 +1312,52 @@ extension RunnerTests {
     return app
   }
 
+  private func performCoordinateTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
+#if os(tvOS)
+    return .unsupported("coordinate tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
+#else
+    interactionCoordinate(app: app, x: x, y: y).tap()
+    return .performed
+#endif
+  }
+
+  private func performCoordinateDoubleTap(app: XCUIApplication, x: Double, y: Double) -> RunnerInteractionOutcome {
+#if os(tvOS)
+    return .unsupported("coordinate double tap is not supported on tvOS; move focus with swipe or scroll, then select the focused element")
+#else
+    interactionCoordinate(app: app, x: x, y: y).doubleTap()
+    return .performed
+#endif
+  }
+
+  private func performCoordinateLongPress(app: XCUIApplication, x: Double, y: Double, duration: TimeInterval) -> RunnerInteractionOutcome {
+#if os(tvOS)
+    return .unsupported("coordinate long press is not supported on tvOS; move focus with swipe or scroll, then long-select the focused element")
+#else
+    interactionCoordinate(app: app, x: x, y: y).press(forDuration: duration)
+    return .performed
+#endif
+  }
+
+  private func performCoordinateDrag(
+    app: XCUIApplication,
+    x: Double,
+    y: Double,
+    x2: Double,
+    y2: Double,
+    holdDuration: TimeInterval
+  ) -> RunnerInteractionOutcome {
+#if os(tvOS)
+    return .unsupported("coordinate drag is not supported on tvOS")
+#else
+    let start = interactionCoordinate(app: app, x: x, y: y)
+    let end = interactionCoordinate(app: app, x: x2, y: y2)
+    start.press(forDuration: holdDuration, thenDragTo: end)
+    return .performed
+#endif
+  }
+
+#if !os(tvOS)
   private func interactionCoordinate(app: XCUIApplication, x: Double, y: Double) -> XCUICoordinate {
     let root = interactionRoot(app: app)
     let origin = root.coordinate(withNormalizedOffset: CGVector(dx: 0, dy: 0))
@@ -578,14 +1366,17 @@ extension RunnerTests {
     let offsetY = y - Double(rootFrame.origin.y)
     return origin.withOffset(CGVector(dx: offsetX, dy: offsetY))
   }
+#endif
 
   private func tapElementCenter(app: XCUIApplication, element: XCUIElement) {
     let frame = element.frame
     if !frame.isEmpty {
-      tapAt(app: app, x: frame.midX, y: frame.midY)
+      _ = tapAt(app: app, x: frame.midX, y: frame.midY)
       return
     }
+#if !os(tvOS)
     element.tap()
+#endif
   }
 
   private func macOSNavigationBackElement(app: XCUIApplication) -> XCUIElement? {