agent-device 0.20.5 → 0.20.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +2 -0
  2. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.apk → agent-device-android-ime-helper-0.20.6.apk} +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.6.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.manifest.json → agent-device-android-ime-helper-0.20.6.manifest.json} +4 -4
  5. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.apk → agent-device-android-snapshot-helper-0.20.6.apk} +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.6.apk.sha256 +1 -0
  7. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.manifest.json → agent-device-android-snapshot-helper-0.20.6.manifest.json} +6 -6
  8. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +28 -0
  9. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +41 -1
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +298 -92
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +126 -187
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +32 -0
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +212 -0
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +118 -8
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +134 -27
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +23 -5
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +10 -0
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +0 -67
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +11 -1
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +2 -22
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +48 -11
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +0 -20
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +177 -0
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +105 -448
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +4 -0
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +587 -0
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +18 -0
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +72 -0
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +89 -0
  31. package/dist/src/agent-device-client.js +2 -2
  32. package/dist/src/android.js +1 -1
  33. package/dist/src/app-log-request-scope.js +1 -1
  34. package/dist/src/app-resolution.js +1 -1
  35. package/dist/src/apps.js +5 -5
  36. package/dist/src/cli-config.js +1 -1
  37. package/dist/src/cli-help.js +34 -24
  38. package/dist/src/cli-runner.js +1 -1
  39. package/dist/src/cli.js +15 -15
  40. package/dist/src/command-metadata.js +1 -1
  41. package/dist/src/command-schema.js +3 -3
  42. package/dist/src/context.js +1 -1
  43. package/dist/src/debug-symbols.js +1 -1
  44. package/dist/src/find.js +1 -1
  45. package/dist/src/finders.js +1 -1
  46. package/dist/src/generic-settle.js +1 -0
  47. package/dist/src/generic.js +1 -1
  48. package/dist/src/ime-lifecycle.js +3 -3
  49. package/dist/src/index.d.ts +27 -7
  50. package/dist/src/input-actions.js +4 -4
  51. package/dist/src/interaction-runtime.js +1 -0
  52. package/dist/src/interaction-snapshot.js +1 -1
  53. package/dist/src/interaction-touch-reference-frame.js +1 -0
  54. package/dist/src/interaction.js +1 -1
  55. package/dist/src/interactor.js +2 -2
  56. package/dist/src/interactor2.js +1 -1
  57. package/dist/src/internal/bin.js +2 -2
  58. package/dist/src/internal/daemon.js +15 -15
  59. package/dist/src/lease-context.js +1 -1
  60. package/dist/src/lease.js +1 -1
  61. package/dist/src/limrun.d.ts +23 -2
  62. package/dist/src/linux.js +1 -1
  63. package/dist/src/process-exit.js +1 -0
  64. package/dist/src/react-native.js +1 -1
  65. package/dist/src/record-trace.js +1 -1
  66. package/dist/src/register-builtins.js +1 -1
  67. package/dist/src/registry.js +25 -24
  68. package/dist/src/remote-config.js +1 -1
  69. package/dist/src/remote-config2.js +1 -1
  70. package/dist/src/reporting.js +5 -5
  71. package/dist/src/runner-client.js +1 -1
  72. package/dist/src/runner-disposal.js +1 -1
  73. package/dist/src/runner-lease.js +1 -1
  74. package/dist/src/runtime.js +3 -3
  75. package/dist/src/screenshot-geometry.js +1 -1
  76. package/dist/src/screenshot-result.js +4 -4
  77. package/dist/src/sdk-android-adb.d.ts +20 -2
  78. package/dist/src/sdk-android-adb.js +1 -1
  79. package/dist/src/sdk-batch-runner.js +3 -3
  80. package/dist/src/sdk-contracts.d.ts +17 -2
  81. package/dist/src/sdk-finders.d.ts +4 -20
  82. package/dist/src/sdk-finders.js +1 -1
  83. package/dist/src/{interaction.d.ts → sdk-selectors.d.ts} +98 -3
  84. package/dist/src/sdk-selectors.js +1 -1
  85. package/dist/src/selector-runtime.js +1 -1
  86. package/dist/src/selector-vocabulary.js +1 -1
  87. package/dist/src/selectors.d.ts +1 -46
  88. package/dist/src/selectors.js +1 -1
  89. package/dist/src/server.js +3 -3
  90. package/dist/src/session-target-evidence.js +1 -1
  91. package/dist/src/session.js +9 -9
  92. package/dist/src/snapshot2.js +1 -1
  93. package/dist/src/text-surface.js +1 -0
  94. package/dist/src/viewport-dimension.js +13 -12
  95. package/dist/src/web.js +1 -1
  96. package/package.json +41 -35
  97. package/server.json +3 -3
  98. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.5.apk.sha256 +0 -1
  99. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.5.apk.sha256 +0 -1
  100. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -225
@@ -152,6 +152,17 @@ extension RunnerTests {
152
152
  )
153
153
  }
154
154
 
155
+ private func plannedGestureResponse(
156
+ plan: RunnerGesturePlan,
157
+ timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
158
+ outcome: RunnerInteractionOutcome
159
+ ) -> Response {
160
+ if let response = unsupportedResponse(for: outcome) {
161
+ return response
162
+ }
163
+ return gestureResponse(message: plan.intent, timing: timing)
164
+ }
165
+
155
166
 
156
167
  func execute(command: Command) throws -> Response {
157
168
  if command.command == .status {
@@ -572,13 +583,20 @@ extension RunnerTests {
572
583
  }
573
584
  }
574
585
 
575
- private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
576
- let options = SnapshotOptions(
586
+ /// Pure command→options projection, extracted so the runner unit bundle can
587
+ /// prove the decoded wire field actually reaches capture options (#1634 P2).
588
+ static func snapshotOptions(from command: Command) -> SnapshotOptions {
589
+ SnapshotOptions(
577
590
  interactiveOnly: command.interactiveOnly ?? false,
578
591
  depth: command.depth,
579
592
  scope: command.scope,
580
- raw: command.raw ?? false
593
+ raw: command.raw ?? false,
594
+ preferredBackend: command.preferredBackend
581
595
  )
596
+ }
597
+
598
+ private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
599
+ let options = Self.snapshotOptions(from: command)
582
600
  do {
583
601
  let payload: DataPayload
584
602
  if options.raw {
@@ -713,24 +731,18 @@ extension RunnerTests {
713
731
  if let requestedFps = command.fps, (requestedFps < minRecordingFps || requestedFps > maxRecordingFps) {
714
732
  return Response(ok: false, error: ErrorPayload(message: "recordStart fps must be between \(minRecordingFps) and \(maxRecordingFps)"))
715
733
  }
716
- if let requestedMaxSize = command.maxSize, requestedMaxSize < 1 {
717
- return Response(ok: false, error: ErrorPayload(message: "recordStart maxSize must be a positive integer"))
718
- }
719
734
  do {
720
735
  let resolvedOutPath = resolveRecordingOutPath(requestedOutPath)
721
736
  let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps)
722
- let maxSizeLabel = command.maxSize.map(String.init) ?? "native"
723
737
  NSLog(
724
- "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@ maxSize=%@",
738
+ "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@",
725
739
  requestedOutPath,
726
740
  resolvedOutPath,
727
- fpsLabel,
728
- maxSizeLabel
741
+ fpsLabel
729
742
  )
730
743
  let recorder = ScreenRecorder(
731
744
  outputPath: resolvedOutPath,
732
- fps: command.fps.map { Int32($0) },
733
- maxSize: command.maxSize
745
+ fps: command.fps.map { Int32($0) }
734
746
  )
735
747
  try recorder.start { [weak self] in
736
748
  return self?.captureRunnerFrame()
@@ -865,6 +877,9 @@ extension RunnerTests {
865
877
  alertDeadline: Date? = nil
866
878
  ) throws -> Response {
867
879
  var activeApp = activeApp
880
+ if command.command != .tap && command.command != .type && !isReadOnlyCommand(command) {
881
+ clearRememberedTextEntryTap()
882
+ }
868
883
  switch command.command {
869
884
  case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime:
870
885
  return Response(
@@ -893,6 +908,7 @@ extension RunnerTests {
893
908
  expectedPoint: expectedPoint
894
909
  )
895
910
  if match.isAmbiguous {
911
+ clearRememberedTextEntryTap()
896
912
  return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements"))
897
913
  }
898
914
  if let element = match.element {
@@ -911,6 +927,7 @@ extension RunnerTests {
911
927
  elementFrame: frame,
912
928
  windowFrame: onScreenWindowFrame(app: activeApp)
913
929
  ) {
930
+ clearRememberedTextEntryTap()
914
931
  return Response(ok: false, error: ErrorPayload(
915
932
  code: "ELEMENT_OFFSCREEN",
916
933
  message: "element resolved off-screen at (\(Int(frame.midX)), \(Int(frame.midY)))"))
@@ -946,6 +963,7 @@ extension RunnerTests {
946
963
  if isTextEntry {
947
964
  waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
948
965
  }
966
+ rememberTextEntryTap(isTextEntry ? element : nil)
949
967
  return gestureResponse(
950
968
  message: match.usedNonHittableFallback
951
969
  ? "tapped via non-hittable coordinate fallback"
@@ -976,11 +994,13 @@ extension RunnerTests {
976
994
  return activateElement(app: activeApp, element: element, action: "tap by selector")
977
995
  }
978
996
  if let response = unsupportedResponse(for: outcome) {
997
+ clearRememberedTextEntryTap()
979
998
  return response
980
999
  }
981
1000
  if isTextEntry {
982
1001
  waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
983
1002
  }
1003
+ rememberTextEntryTap(isTextEntry ? element : nil)
984
1004
  return gestureResponse(
985
1005
  message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" : "tapped",
986
1006
  timing: timing,
@@ -992,9 +1012,11 @@ extension RunnerTests {
992
1012
  : nil
993
1013
  )
994
1014
  }
1015
+ clearRememberedTextEntryTap()
995
1016
  return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found"))
996
1017
  }
997
1018
  if let x = command.x, let y = command.y {
1019
+ let textInput = textInputAt(app: activeApp, x: x, y: y)
998
1020
  var fallback: GestureFallback?
999
1021
  if command.synthesized == true {
1000
1022
  let policyKind = SynthesizedGesturePolicyKind.coordinateTap
@@ -1007,6 +1029,7 @@ extension RunnerTests {
1007
1029
  }
1008
1030
  if case .performed = outcome {
1009
1031
  logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false)
1032
+ rememberTextEntryTap(textInput)
1010
1033
  return gestureResponse(message: "tapped", timing: timing)
1011
1034
  }
1012
1035
  logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true)
@@ -1015,8 +1038,10 @@ extension RunnerTests {
1015
1038
  let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
1016
1039
  let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) }
1017
1040
  if let response = unsupportedResponse(for: outcome) {
1041
+ clearRememberedTextEntryTap()
1018
1042
  return response
1019
1043
  }
1044
+ rememberTextEntryTap(textInput)
1020
1045
  return gestureResponse(
1021
1046
  message: "tapped",
1022
1047
  timing: timing,
@@ -1024,6 +1049,7 @@ extension RunnerTests {
1024
1049
  fallback: fallback
1025
1050
  )
1026
1051
  }
1052
+ clearRememberedTextEntryTap()
1027
1053
  return Response(ok: false, error: ErrorPayload(message: "tap requires a selector or x/y"))
1028
1054
  case .mouseClick:
1029
1055
  guard let x = command.x, let y = command.y else {
@@ -1348,7 +1374,7 @@ extension RunnerTests {
1348
1374
  ok: false,
1349
1375
  error: ErrorPayload(
1350
1376
  code: "UNSUPPORTED_OPERATION",
1351
- message: "Unable to dismiss the iOS keyboard without a safe native dismiss control",
1377
+ message: "Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)",
1352
1378
  hint:
1353
1379
  "The on-screen keyboard usually does not block agent-device interactions: press the next target directly instead of retrying dismiss. If that press fails or reports no visible effect, scroll the target into view, or use keyboard enter to press the return key when submission is wanted."
1354
1380
  )
@@ -1360,7 +1386,8 @@ extension RunnerTests {
1360
1386
  message: "keyboardDismiss",
1361
1387
  visible: result.visible,
1362
1388
  wasVisible: result.wasVisible,
1363
- dismissed: result.dismissed
1389
+ dismissed: result.dismissed,
1390
+ keyboardDismissMechanism: result.mechanism?.rawValue
1364
1391
  )
1365
1392
  )
1366
1393
  case .keyboardReturn:
@@ -1404,7 +1431,8 @@ extension RunnerTests {
1404
1431
  error: ErrorPayload(code: "INVALID_ARGS", message: validationError)
1405
1432
  )
1406
1433
  }
1407
- if plannedGestureExecution(for: plan) == .fastSwipe {
1434
+ switch plannedGestureExecution(for: plan) {
1435
+ case .fastSwipe:
1408
1436
  // Validation above guarantees a non-empty, single-pointer path for this execution kind.
1409
1437
  let first = plan.pointers[0].samples.first!.point
1410
1438
  let last = plan.pointers[0].samples.last!.point
@@ -1422,14 +1450,12 @@ extension RunnerTests {
1422
1450
  synthesizedProfile: .fastSwipe
1423
1451
  )
1424
1452
  )
1453
+ case .sampled:
1454
+ let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
1455
+ sampledPlannedGesture(app: activeApp, plan: plan)
1456
+ }
1457
+ return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome)
1425
1458
  }
1426
- let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
1427
- sampledPlannedGesture(app: activeApp, plan: plan)
1428
- }
1429
- if let response = unsupportedResponse(for: outcome) {
1430
- return response
1431
- }
1432
- return gestureResponse(message: plan.intent, timing: timing)
1433
1459
  case .gestureViewport:
1434
1460
  let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
1435
1461
  guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else {
@@ -1705,7 +1731,7 @@ extension RunnerTests {
1705
1731
  error: ErrorPayload(
1706
1732
  code: "XCTEST_RECORDED_FAILURE",
1707
1733
  message: "XCTest recorded a failure while executing \(command.command.rawValue); the action may not have been performed.",
1708
- hint: "The iOS runner session will be restarted. Retry after a fresh snapshot, or use screenshot plus coordinate commands when the accessibility tree is unavailable."
1734
+ hint: "The iOS runner session was invalidated. Re-observe with a fresh snapshot before retrying; if the accessibility tree is unavailable, use screenshot plus coordinate commands instead of retrying the tap blindly."
1709
1735
  )
1710
1736
  )
1711
1737
  }
@@ -1796,9 +1822,63 @@ extension RunnerTests {
1796
1822
  let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0
1797
1823
  let textEntryMode = resolveTextEntryMode(command)
1798
1824
  let target: TextEntryTarget
1825
+ var resolvedCoordinateContext: SynthesizedCoordinateContext?
1799
1826
  var maestroNonHittableCoordinateFallbackUsed: Bool?
1827
+ if command.allowNonHittableCoordinateFallback == true,
1828
+ command.x != nil,
1829
+ command.y != nil
1830
+ {
1831
+ // The shared runtime has already resolved this node as non-hittable and
1832
+ // deliberately selected Maestro's coordinate compatibility route.
1833
+ maestroNonHittableCoordinateFallbackUsed = true
1834
+ }
1800
1835
  let focusStartedAt = Date()
1801
- if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
1836
+ #if os(iOS)
1837
+ let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
1838
+ var resolvedCoordinateTarget: TextEntryTarget?
1839
+ if Self.shouldUseResolvedCoordinateTextEntryRoute(
1840
+ repairMode: textEntryMode,
1841
+ hasX: command.x != nil,
1842
+ hasY: command.y != nil,
1843
+ xCTestChannelPenalized: xCTestChannelPenalized
1844
+ ), let x = command.x, let y = command.y {
1845
+ let policyKind = SynthesizedGesturePolicyKind.coordinateTap
1846
+ let context = synthesizedCoordinateContext(
1847
+ app: activeApp,
1848
+ policy: synthesizedGesturePolicy(policyKind)
1849
+ )
1850
+ let (_, outcome) = performGesture(activeApp, idleTimeout: false) {
1851
+ synthesizedTapAt(app: activeApp, x: x, y: y, context: context)
1852
+ }
1853
+ if Self.shouldFallbackFromSynthesizedTextEntryFocus(outcome) {
1854
+ logSynthesizedGesturePolicyDecision(
1855
+ kind: policyKind,
1856
+ context: context,
1857
+ fallbackAttempted: true
1858
+ )
1859
+ } else {
1860
+ logSynthesizedGesturePolicyDecision(
1861
+ kind: policyKind,
1862
+ context: context,
1863
+ fallbackAttempted: false
1864
+ )
1865
+ resolvedCoordinateContext = context
1866
+ resolvedCoordinateTarget = TextEntryTarget(
1867
+ element: nil,
1868
+ refreshPoint: CGPoint(x: x, y: y),
1869
+ prefersFocusedElement: false
1870
+ )
1871
+ }
1872
+ }
1873
+ #else
1874
+ let xCTestChannelPenalized = false
1875
+ let resolvedCoordinateTarget: TextEntryTarget? = nil
1876
+ #endif
1877
+ if let resolvedCoordinateTarget {
1878
+ target = resolvedCoordinateTarget
1879
+ } else if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
1880
+ // Released daemons may still send selector-keyed type commands even though current
1881
+ // daemons resolve fill selectors through the runtime tree before reaching the runner.
1802
1882
  let match = findElement(
1803
1883
  app: activeApp,
1804
1884
  selectorKey: selectorKey,
@@ -1829,7 +1909,16 @@ extension RunnerTests {
1829
1909
  textEntryModeName(textEntryMode)
1830
1910
  )
1831
1911
  if textEntryMode == .replacement {
1832
- guard target.element != nil else {
1912
+ #if os(iOS)
1913
+ let canReplaceResolvedFirstResponder = Self.shouldUseSynthesizedFirstResponderReplacement(
1914
+ hasResolvedElement: target.element != nil,
1915
+ hasRefreshPoint: target.refreshPoint != nil,
1916
+ xCTestChannelPenalized: xCTestChannelPenalized
1917
+ )
1918
+ #else
1919
+ let canReplaceResolvedFirstResponder = false
1920
+ #endif
1921
+ guard target.element != nil || canReplaceResolvedFirstResponder else {
1833
1922
  let message =
1834
1923
  (command.x != nil && command.y != nil)
1835
1924
  ? "no text input found at the provided coordinates to clear"
@@ -1843,8 +1932,16 @@ extension RunnerTests {
1843
1932
  text: text,
1844
1933
  delaySeconds: delaySeconds,
1845
1934
  repairMode: textEntryMode,
1935
+ xCTestChannelPenalized: xCTestChannelPenalized,
1936
+ synthesizer: PrivateXCTestTextEntrySynthesizer(),
1846
1937
  commandId: command.commandId
1847
1938
  )
1939
+ if let failure = textResult.failure {
1940
+ return Response(
1941
+ ok: false,
1942
+ error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)
1943
+ )
1944
+ }
1848
1945
  if textResult.verified == false {
1849
1946
  let expected = textResult.expectedText ?? ""
1850
1947
  let observed = textResult.observedText ?? ""
@@ -1857,7 +1954,16 @@ extension RunnerTests {
1857
1954
  )
1858
1955
  }
1859
1956
  let point = target.refreshPoint
1860
- let frame = activeApp.frame
1957
+ let frame: CGRect
1958
+ if let resolvedCoordinateContext {
1959
+ frame = resolvedCoordinateContext.referenceFrame
1960
+ } else if point != nil {
1961
+ frame = activeApp.frame
1962
+ } else {
1963
+ // Bare `type` has no coordinate response to normalize. Avoid serializing the
1964
+ // application AX tree only to emit unused reference dimensions.
1965
+ frame = .zero
1966
+ }
1861
1967
  return Response(
1862
1968
  ok: true,
1863
1969
  data: DataPayload(
@@ -1866,7 +1972,8 @@ extension RunnerTests {
1866
1972
  y: point.map { Double($0.y) },
1867
1973
  referenceWidth: frame.isEmpty ? nil : Double(frame.width),
1868
1974
  referenceHeight: frame.isEmpty ? nil : Double(frame.height),
1869
- maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
1975
+ maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed,
1976
+ textEntryRoute: textResult.textEntryRoute
1870
1977
  )
1871
1978
  )
1872
1979
  }
@@ -21,23 +21,34 @@ private enum KeyboardDismissObservationTiming {
21
21
  static let settleRequiredConsecutiveMatches: Int = 3
22
22
  }
23
23
 
24
+ // The mechanism that actually resigned the keyboard, disclosed to the caller
25
+ // (#1598) so a response never claims "dismissed" without saying how — a
26
+ // safe-area tap has a very different reliability/side-effect profile than
27
+ // tapping the keyboard's own Done key, and callers need to know which one
28
+ // fired.
29
+ enum RunnerKeyboardDismissMechanism: String {
30
+ case dismissKey
31
+ }
32
+
24
33
  extension RunnerTests {
25
34
  func isKeyboardVisible(app: XCUIApplication) -> Bool {
26
35
  return visibleKeyboardFrame(app: app) != nil
27
36
  }
28
37
 
29
- func dismissKeyboard(app: XCUIApplication) -> (wasVisible: Bool, dismissed: Bool, visible: Bool) {
38
+ func dismissKeyboard(
39
+ app: XCUIApplication
40
+ ) -> (wasVisible: Bool, dismissed: Bool, visible: Bool, mechanism: RunnerKeyboardDismissMechanism?) {
30
41
  let keyboard = app.keyboards.firstMatch
31
42
  let wasVisible = isKeyboardVisible(app: app)
32
43
  guard wasVisible else {
33
- return (wasVisible: false, dismissed: false, visible: false)
44
+ return (wasVisible: false, dismissed: false, visible: false, mechanism: nil)
34
45
  }
35
46
 
36
47
  #if os(tvOS)
37
48
  _ = pressTvRemote(.menu)
38
49
  sleepFor(0.2)
39
50
  let visible = isKeyboardVisible(app: app)
40
- return (wasVisible: true, dismissed: !visible, visible: visible)
51
+ return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey)
41
52
  #else
42
53
  if tapKeyboardDismissControl(app: app) {
43
54
  _ = keyboard.waitForNonExistence(timeout: KeyboardDismissObservationTiming.timeout)
@@ -47,13 +58,20 @@ extension RunnerTests {
47
58
  requiredConsecutiveMatches: KeyboardDismissObservationTiming.settleRequiredConsecutiveMatches
48
59
  )
49
60
  let visible = isKeyboardVisible(app: app)
50
- return (wasVisible: true, dismissed: !visible, visible: visible)
61
+ return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey)
51
62
  }
52
63
 
53
- return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app))
64
+ // #1606 review P1 (twice): generic background-tap dismissal is
65
+ // deliberately UNSUPPORTED. No geometry or role query can prove a
66
+ // coordinate is side-effect-free — a full-screen unlabeled Pressable is
67
+ // indistinguishable from an inert backdrop, so a "safe-area" tap can
68
+ // navigate or submit while reporting a successful dismiss. The dismiss
69
+ // key is the only mechanism the runner can vouch for.
70
+ return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app), mechanism: nil)
54
71
  #endif
55
72
  }
56
73
 
74
+
57
75
  // AX-free on purpose (screenshot bytes, not the accessibility tree) so it holds
58
76
  // under the same AX degradation the synthesized gesture lane is built to survive.
59
77
  // Bounded and self-terminating: returns as soon as `requiredConsecutiveMatches`
@@ -86,6 +86,7 @@ extension RunnerTests {
86
86
  currentApp = app
87
87
  currentBundleId = nil
88
88
  currentAppProcessIdentifier = nil
89
+ clearRememberedTextEntryTap()
89
90
  snapshotXCTestPenaltyWarmupExemptionPending = false
90
91
  }
91
92
 
@@ -96,11 +97,17 @@ extension RunnerTests {
96
97
  currentApp = nil
97
98
  currentBundleId = nil
98
99
  currentAppProcessIdentifier = nil
100
+ clearRememberedTextEntryTap()
99
101
  snapshotXCTestPenaltyWarmupExemptionPending = false
100
102
  }
101
103
 
102
104
  func resetTargetAfterExternalRelaunch() -> Response {
103
105
  invalidateCachedTarget(reason: "external_app_relaunch")
106
+ // The app process is replaced, but the retained runner survives. Clear
107
+ // process-bound capture state explicitly because invalidation drops the
108
+ // old PID before refreshCachedTargetIfProcessChanged can observe it.
109
+ clearSnapshotXCTestChannelPenalty(reason: "external_app_relaunch")
110
+ clearPrivateAXAcceptedDepth(reason: "external_app_relaunch")
104
111
  needsFirstInteractionDelay = true
105
112
  return Response(ok: true, data: DataPayload(message: "target reset"))
106
113
  }
@@ -121,7 +128,9 @@ extension RunnerTests {
121
128
  )
122
129
  currentApp = candidate
123
130
  currentAppProcessIdentifier = observedProcessIdentifier
131
+ clearRememberedTextEntryTap()
124
132
  clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
133
+ clearPrivateAXAcceptedDepth(reason: "target_process_changed")
125
134
  snapshotXCTestPenaltyWarmupExemptionPending = true
126
135
  needsFirstInteractionDelay = true
127
136
  }
@@ -194,6 +203,7 @@ extension RunnerTests {
194
203
  currentApp = target
195
204
  currentBundleId = bundleId
196
205
  currentAppProcessIdentifier = Self.processIdentifier(of: target)
206
+ clearRememberedTextEntryTap()
197
207
  snapshotXCTestPenaltyWarmupExemptionPending = false
198
208
  needsFirstInteractionDelay = true
199
209
  return target
@@ -3,71 +3,4 @@ import XCTest
3
3
 
4
4
  extension RunnerTests {
5
5
 
6
- func testCachedTargetRefreshRequiresChangedPositiveProcessIdentity() {
7
- XCTAssertFalse(
8
- Self.shouldRefreshCachedTarget(
9
- cachedProcessIdentifier: nil,
10
- observedProcessIdentifier: 42
11
- )
12
- )
13
- XCTAssertFalse(
14
- Self.shouldRefreshCachedTarget(
15
- cachedProcessIdentifier: 42,
16
- observedProcessIdentifier: 42
17
- )
18
- )
19
- XCTAssertTrue(
20
- Self.shouldRefreshCachedTarget(
21
- cachedProcessIdentifier: 41,
22
- observedProcessIdentifier: 42
23
- )
24
- )
25
- }
26
-
27
- func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() {
28
- snapshotXCTestPenaltyWarmupExemptionPending = true
29
-
30
- XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption())
31
- XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption())
32
- }
33
-
34
- func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() {
35
- penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test")
36
- XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
37
-
38
- clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
39
-
40
- XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
41
- }
42
-
43
- func testCachedTargetInvalidationClearsProcessBoundState() {
44
- currentApp = app
45
- currentBundleId = "com.example.app"
46
- currentAppProcessIdentifier = 42
47
- snapshotXCTestPenaltyWarmupExemptionPending = true
48
-
49
- invalidateCachedTarget(reason: "unit_test")
50
-
51
- XCTAssertNil(currentApp)
52
- XCTAssertNil(currentBundleId)
53
- XCTAssertNil(currentAppProcessIdentifier)
54
- XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
55
- }
56
-
57
- func testTargetResetInvalidatesProcessBoundStateWithoutRestartingRunner() {
58
- currentApp = app
59
- currentBundleId = "com.example.app"
60
- currentAppProcessIdentifier = 42
61
- snapshotXCTestPenaltyWarmupExemptionPending = true
62
- needsFirstInteractionDelay = false
63
-
64
- let response = resetTargetAfterExternalRelaunch()
65
-
66
- XCTAssertTrue(response.ok)
67
- XCTAssertNil(currentApp)
68
- XCTAssertNil(currentBundleId)
69
- XCTAssertNil(currentAppProcessIdentifier)
70
- XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
71
- XCTAssertTrue(needsFirstInteractionDelay)
72
- }
73
6
  }
@@ -136,8 +136,8 @@ struct Command: Codable {
136
136
  let gesturePlan: RunnerGesturePlan?
137
137
  let outPath: String?
138
138
  let fps: Int?
139
- let maxSize: Int?
140
139
  let interactiveOnly: Bool?
140
+ let preferredBackend: String?
141
141
  let depth: Int?
142
142
  let scope: String?
143
143
  let raw: Bool?
@@ -257,11 +257,13 @@ struct DataPayload: Codable {
257
257
  let visible: Bool?
258
258
  let wasVisible: Bool?
259
259
  let dismissed: Bool?
260
+ let keyboardDismissMechanism: String?
260
261
  let orientation: String?
261
262
  let gestureFallback: String?
262
263
  let gestureFallbackMessage: String?
263
264
  let gestureFallbackHint: String?
264
265
  let maestroNonHittableCoordinateFallbackUsed: Bool?
266
+ let textEntryRoute: String?
265
267
  let runnerFatal: Bool?
266
268
  let runnerFatalReason: String?
267
269
  let completedSteps: Int?
@@ -297,11 +299,13 @@ struct DataPayload: Codable {
297
299
  visible: Bool? = nil,
298
300
  wasVisible: Bool? = nil,
299
301
  dismissed: Bool? = nil,
302
+ keyboardDismissMechanism: String? = nil,
300
303
  orientation: String? = nil,
301
304
  gestureFallback: String? = nil,
302
305
  gestureFallbackMessage: String? = nil,
303
306
  gestureFallbackHint: String? = nil,
304
307
  maestroNonHittableCoordinateFallbackUsed: Bool? = nil,
308
+ textEntryRoute: String? = nil,
305
309
  runnerFatal: Bool? = nil,
306
310
  runnerFatalReason: String? = nil,
307
311
  completedSteps: Int? = nil,
@@ -336,11 +340,13 @@ struct DataPayload: Codable {
336
340
  self.visible = visible
337
341
  self.wasVisible = wasVisible
338
342
  self.dismissed = dismissed
343
+ self.keyboardDismissMechanism = keyboardDismissMechanism
339
344
  self.orientation = orientation
340
345
  self.gestureFallback = gestureFallback
341
346
  self.gestureFallbackMessage = gestureFallbackMessage
342
347
  self.gestureFallbackHint = gestureFallbackHint
343
348
  self.maestroNonHittableCoordinateFallbackUsed = maestroNonHittableCoordinateFallbackUsed
349
+ self.textEntryRoute = textEntryRoute
344
350
  self.runnerFatal = runnerFatal
345
351
  self.runnerFatalReason = runnerFatalReason
346
352
  self.completedSteps = completedSteps
@@ -390,4 +396,8 @@ struct SnapshotOptions {
390
396
  let depth: Int?
391
397
  let scope: String?
392
398
  let raw: Bool
399
+ /// Internal daemon ask: capture with this backend first regardless of channel
400
+ /// health ("private-ax"). Same-backend evidence probes (tap-outcome
401
+ /// corroboration) must be captured the way their baseline was.
402
+ var preferredBackend: String? = nil
393
403
  }
@@ -7,7 +7,6 @@ extension RunnerTests {
7
7
  final class ScreenRecorder {
8
8
  private let outputPath: String
9
9
  private let fps: Int32?
10
- private let maxSize: Int?
11
10
  private var effectiveFps: Int32 {
12
11
  max(1, fps ?? RunnerTests.defaultRecordingFps)
13
12
  }
@@ -26,10 +25,9 @@ extension RunnerTests {
26
25
  private var startedSession = false
27
26
  private var startError: Error?
28
27
 
29
- init(outputPath: String, fps: Int32?, maxSize: Int?) {
28
+ init(outputPath: String, fps: Int32?) {
30
29
  self.outputPath = outputPath
31
30
  self.fps = fps
32
- self.maxSize = maxSize
33
31
  }
34
32
 
35
33
  func start(captureFrame: @escaping () -> RunnerImage?) throws {
@@ -50,7 +48,7 @@ extension RunnerTests {
50
48
  while Date() < bootstrapDeadline {
51
49
  if let image = captureFrame(), let cgImage = runnerCGImage(from: image) {
52
50
  bootstrapImage = image
53
- dimensions = scaledDimensions(width: cgImage.width, height: cgImage.height)
51
+ dimensions = CGSize(width: cgImage.width, height: cgImage.height)
54
52
  break
55
53
  }
56
54
  Thread.sleep(forTimeInterval: 0.05)
@@ -261,23 +259,5 @@ extension RunnerTests {
261
259
  return pixelBuffer
262
260
  }
263
261
 
264
- private func scaledDimensions(width: Int, height: Int) -> CGSize {
265
- guard let maxSize, maxSize > 0 else {
266
- return CGSize(width: width, height: height)
267
- }
268
- let longest = max(width, height)
269
- guard longest > maxSize else {
270
- return CGSize(width: width, height: height)
271
- }
272
- let scale = Double(maxSize) / Double(longest)
273
- return CGSize(
274
- width: scaledEvenDimension(width, scale: scale),
275
- height: scaledEvenDimension(height, scale: scale)
276
- )
277
- }
278
-
279
- private func scaledEvenDimension(_ value: Int, scale: Double) -> Int {
280
- max(2, Int((Double(value) * scale / 2.0).rounded()) * 2)
281
- }
282
262
  }
283
263
  }