agent-device 0.20.5 → 0.20.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/README.md +5 -3
  2. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.apk → agent-device-android-ime-helper-0.20.7.apk} +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.7.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.manifest.json → agent-device-android-ime-helper-0.20.7.manifest.json} +4 -4
  5. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.apk → agent-device-android-snapshot-helper-0.20.7.apk} +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.7.apk.sha256 +1 -0
  7. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.manifest.json → agent-device-android-snapshot-helper-0.20.7.manifest.json} +6 -6
  8. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +48 -0
  9. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +108 -1
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +630 -89
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSelectorMatchPolicy.swift +78 -0
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +126 -187
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +32 -0
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +212 -0
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +149 -10
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +161 -32
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CoordinateTextEntryTests.swift +4 -0
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +36 -28
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +23 -5
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +10 -0
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +0 -67
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +18 -1
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +2 -22
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SelectorMatchPolicyTests.swift +4 -0
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +2 -1
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +145 -20
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +0 -20
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +4 -0
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +234 -0
  31. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +106 -448
  32. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +4 -0
  33. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +602 -0
  34. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +18 -0
  35. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +72 -0
  36. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +89 -0
  37. package/dist/src/agent-browser-tool.js +1 -1
  38. package/dist/src/agent-device-client.js +2 -2
  39. package/dist/src/android-adb.js +1 -1
  40. package/dist/src/android.js +1 -1
  41. package/dist/src/app-helpers.js +1 -1
  42. package/dist/src/app-lifecycle.js +7 -7
  43. package/dist/src/app-lifecycle2.js +2 -0
  44. package/dist/src/app-log-files.js +1 -0
  45. package/dist/src/app-log-reconnect.js +1 -0
  46. package/dist/src/app-log-runtime.js +4 -0
  47. package/dist/src/app-resolution.js +2 -1
  48. package/dist/src/apps.js +5 -5
  49. package/dist/src/audio-probe.js +1 -1
  50. package/dist/src/auth-session.js +1 -0
  51. package/dist/src/auth.js +2 -0
  52. package/dist/src/boot-diagnostics.js +2 -2
  53. package/dist/src/cli-config.js +1 -1
  54. package/dist/src/cli-grammar.js +1 -0
  55. package/dist/src/cli-help.js +204 -254
  56. package/dist/src/cli-output.js +1 -0
  57. package/dist/src/cli-runner.js +1 -2
  58. package/dist/src/cli.js +6 -16
  59. package/dist/src/client-metro.js +1 -1
  60. package/dist/src/cloud-artifacts.d.ts +2 -137
  61. package/dist/src/command-metadata.js +1 -1
  62. package/dist/src/command-schema.js +3 -3
  63. package/dist/src/connection-runtime.js +1 -0
  64. package/dist/src/connection.js +5 -0
  65. package/dist/src/context.js +1 -1
  66. package/dist/src/daemon-shutdown-report.js +1 -0
  67. package/dist/src/daemon.js +2 -0
  68. package/dist/src/debug-symbols.js +3 -3
  69. package/dist/src/device-claim-inspection.js +1 -0
  70. package/dist/src/device-input-state.js +2 -2
  71. package/dist/src/device.js +1 -1
  72. package/dist/src/device2.js +2 -0
  73. package/dist/src/find.js +1 -1
  74. package/dist/src/finders.js +1 -1
  75. package/dist/src/generic-settle.js +1 -0
  76. package/dist/src/generic.js +1 -1
  77. package/dist/src/harmonyos.js +1 -0
  78. package/dist/src/helper.js +1 -1
  79. package/dist/src/host-process.js +2 -2
  80. package/dist/src/ime-lifecycle.js +7 -7
  81. package/dist/src/index.d.ts +69 -18
  82. package/dist/src/input-actions.js +4 -4
  83. package/dist/src/input-actions2.js +1 -1
  84. package/dist/src/input-actions3.js +1 -0
  85. package/dist/src/install-artifact.js +1 -5
  86. package/dist/src/install-artifact2.js +1 -1
  87. package/dist/src/install-source2.js +1 -1
  88. package/dist/src/interaction-runtime.js +1 -0
  89. package/dist/src/interaction-snapshot.js +1 -1
  90. package/dist/src/interaction-touch-reference-frame.js +1 -0
  91. package/dist/src/interaction.js +1 -1
  92. package/dist/src/interactor.js +2 -2
  93. package/dist/src/interactor2.js +1 -1
  94. package/dist/src/internal/bin.js +2 -2
  95. package/dist/src/internal/daemon.js +9 -16
  96. package/dist/src/inventory.js +5 -0
  97. package/dist/src/inventory2.js +1 -0
  98. package/dist/src/inventory3.js +2 -0
  99. package/dist/src/inventory4.js +1 -0
  100. package/dist/src/inventory5.js +1 -0
  101. package/dist/src/inventory6.js +1 -0
  102. package/dist/src/lease-context.js +1 -1
  103. package/dist/src/lease.js +1 -1
  104. package/dist/src/limrun-runtime-dependencies.js +1 -0
  105. package/dist/src/limrun.d.ts +10 -104
  106. package/dist/src/limrun.js +1 -1
  107. package/dist/src/linux.js +1 -1
  108. package/dist/src/logcat.js +1 -1
  109. package/dist/src/manifest.js +1 -1
  110. package/dist/src/owner-identity.js +1 -0
  111. package/dist/src/physical-device-control.js +1 -1
  112. package/dist/src/platform-runtime-app-log-android-transport.js +1 -0
  113. package/dist/src/platform-runtime-app-log-process.js +1 -0
  114. package/dist/src/platform-runtime-host.js +1 -0
  115. package/dist/src/platform-runtime-network-web-transport.js +1 -0
  116. package/dist/src/platform-runtime-operation-host.js +5 -0
  117. package/dist/src/platform-runtime-toolchain-host.js +1 -0
  118. package/dist/src/platform-runtime.js +1 -0
  119. package/dist/src/prepare-kind.js +15 -0
  120. package/dist/src/process-entry.js +1 -0
  121. package/dist/src/process-exit.js +1 -0
  122. package/dist/src/process-lock.js +1 -1
  123. package/dist/src/provider-webdriver.js +1 -0
  124. package/dist/src/provider.js +1 -0
  125. package/dist/src/proxy.js +2 -0
  126. package/dist/src/react-native.js +1 -1
  127. package/dist/src/record-trace.js +1 -1
  128. package/dist/src/registry.js +25 -24
  129. package/dist/src/remote-config.js +1 -1
  130. package/dist/src/remote-config2.js +1 -1
  131. package/dist/src/replay.js +3 -0
  132. package/dist/src/reporting.js +4 -4
  133. package/dist/src/runner-client.js +1 -1
  134. package/dist/src/runner-disposal.js +1 -1
  135. package/dist/src/runner-lease.js +1 -1
  136. package/dist/src/runtime.js +3 -3
  137. package/dist/src/runtime2.js +4 -0
  138. package/dist/src/runtime3.js +3 -0
  139. package/dist/src/runtime4.js +1 -0
  140. package/dist/src/runtime5.js +1 -0
  141. package/dist/src/runtime6.js +1 -0
  142. package/dist/src/runtime7.js +1 -0
  143. package/dist/src/screenshot-result.js +4 -4
  144. package/dist/src/screenshot.js +1 -0
  145. package/dist/src/sdk-android-adb.d.ts +20 -2
  146. package/dist/src/sdk-android-adb.js +2 -2
  147. package/dist/src/sdk-batch-runner.js +3 -3
  148. package/dist/src/sdk-contracts.d.ts +47 -6
  149. package/dist/src/sdk-contracts.js +1 -1
  150. package/dist/src/sdk-device.js +1 -1
  151. package/dist/src/sdk-finders.d.ts +4 -20
  152. package/dist/src/sdk-finders.js +1 -1
  153. package/dist/src/sdk-remote-config.js +1 -1
  154. package/dist/src/sdk-selectors.d.ts +601 -0
  155. package/dist/src/sdk-selectors.js +1 -1
  156. package/dist/src/selector-runtime.js +1 -1
  157. package/dist/src/selector-vocabulary.js +1 -1
  158. package/dist/src/selectors.d.ts +1 -46
  159. package/dist/src/selectors.js +1 -1
  160. package/dist/src/server.js +3 -3
  161. package/dist/src/session-target-evidence.js +1 -1
  162. package/dist/src/session.js +10 -9
  163. package/dist/src/shared.js +2 -0
  164. package/dist/src/shell-quote.js +1 -0
  165. package/dist/src/simulator.js +1 -1
  166. package/dist/src/snapshot-runtime.js +1 -0
  167. package/dist/src/snapshot-session.js +1 -0
  168. package/dist/src/snapshot.js +1 -1
  169. package/dist/src/snapshot2.js +1 -1
  170. package/dist/src/src.js +1 -1
  171. package/dist/src/src2.js +2 -0
  172. package/dist/src/src3.js +1 -0
  173. package/dist/src/text-surface.js +1 -0
  174. package/dist/src/timing-safe-equal.js +1 -0
  175. package/dist/src/tool-provider.js +3 -3
  176. package/dist/src/tool-provider2.js +1 -1
  177. package/dist/src/tool-provider3.js +1 -1
  178. package/dist/src/update-check.js +1 -1
  179. package/dist/src/version.js +1 -1
  180. package/dist/src/web.js +1 -1
  181. package/package.json +72 -50
  182. package/server.json +3 -3
  183. package/skills/agent-device/SKILL.md +6 -31
  184. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.5.apk.sha256 +0 -1
  185. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.5.apk.sha256 +0 -1
  186. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -225
  187. package/dist/src/app-log-request-scope.js +0 -1
  188. package/dist/src/devices.js +0 -1
  189. package/dist/src/devices2.js +0 -1
  190. package/dist/src/devices3.js +0 -1
  191. package/dist/src/devices4.js +0 -1
  192. package/dist/src/interaction.d.ts +0 -230
  193. package/dist/src/provider-limrun-runtime.js +0 -1
  194. package/dist/src/register-builtins.js +0 -1
  195. package/dist/src/screenshot-geometry.js +0 -1
  196. package/dist/src/viewport-dimension.js +0 -14
@@ -152,6 +152,17 @@ extension RunnerTests {
152
152
  )
153
153
  }
154
154
 
155
+ private func plannedGestureResponse(
156
+ plan: RunnerGesturePlan,
157
+ timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
158
+ outcome: RunnerInteractionOutcome
159
+ ) -> Response {
160
+ if let response = unsupportedResponse(for: outcome) {
161
+ return response
162
+ }
163
+ return gestureResponse(message: plan.intent, timing: timing)
164
+ }
165
+
155
166
 
156
167
  func execute(command: Command) throws -> Response {
157
168
  if command.command == .status {
@@ -572,13 +583,26 @@ extension RunnerTests {
572
583
  }
573
584
  }
574
585
 
575
- private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
576
- let options = SnapshotOptions(
586
+ /// Pure command→options projection, extracted so the runner unit bundle can
587
+ /// prove the decoded wire field actually reaches capture options (#1634 P2).
588
+ static func snapshotOptions(from command: Command) -> SnapshotOptions {
589
+ let customActions = command.customActions ?? false
590
+ return SnapshotOptions(
577
591
  interactiveOnly: command.interactiveOnly ?? false,
578
592
  depth: command.depth,
579
593
  scope: command.scope,
580
- raw: command.raw ?? false
594
+ raw: command.raw ?? false,
595
+ // Custom actions are only readable through the private AX client, so
596
+ // asking for them pins that backend rather than silently returning a
597
+ // capture that structurally cannot carry them. An explicit pin wins.
598
+ preferredBackend: command.preferredBackend
599
+ ?? (customActions ? SnapshotBackendKind.privateAX.rawValue : nil),
600
+ customActions: customActions
581
601
  )
602
+ }
603
+
604
+ private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
605
+ let options = Self.snapshotOptions(from: command)
582
606
  do {
583
607
  let payload: DataPayload
584
608
  if options.raw {
@@ -713,24 +737,18 @@ extension RunnerTests {
713
737
  if let requestedFps = command.fps, (requestedFps < minRecordingFps || requestedFps > maxRecordingFps) {
714
738
  return Response(ok: false, error: ErrorPayload(message: "recordStart fps must be between \(minRecordingFps) and \(maxRecordingFps)"))
715
739
  }
716
- if let requestedMaxSize = command.maxSize, requestedMaxSize < 1 {
717
- return Response(ok: false, error: ErrorPayload(message: "recordStart maxSize must be a positive integer"))
718
- }
719
740
  do {
720
741
  let resolvedOutPath = resolveRecordingOutPath(requestedOutPath)
721
742
  let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps)
722
- let maxSizeLabel = command.maxSize.map(String.init) ?? "native"
723
743
  NSLog(
724
- "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@ maxSize=%@",
744
+ "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@",
725
745
  requestedOutPath,
726
746
  resolvedOutPath,
727
- fpsLabel,
728
- maxSizeLabel
747
+ fpsLabel
729
748
  )
730
749
  let recorder = ScreenRecorder(
731
750
  outputPath: resolvedOutPath,
732
- fps: command.fps.map { Int32($0) },
733
- maxSize: command.maxSize
751
+ fps: command.fps.map { Int32($0) }
734
752
  )
735
753
  try recorder.start { [weak self] in
736
754
  return self?.captureRunnerFrame()
@@ -859,12 +877,15 @@ extension RunnerTests {
859
877
  return .context(ActiveCommandContext(app: activeApp))
860
878
  }
861
879
 
862
- private func executeOnMainPrepared(
880
+ func executeOnMainPrepared(
863
881
  command: Command,
864
882
  activeApp: XCUIApplication,
865
883
  alertDeadline: Date? = nil
866
884
  ) throws -> Response {
867
885
  var activeApp = activeApp
886
+ if command.command != .tap && command.command != .type && !isReadOnlyCommand(command) {
887
+ clearRememberedTextEntryTap()
888
+ }
868
889
  switch command.command {
869
890
  case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime:
870
891
  return Response(
@@ -893,6 +914,7 @@ extension RunnerTests {
893
914
  expectedPoint: expectedPoint
894
915
  )
895
916
  if match.isAmbiguous {
917
+ clearRememberedTextEntryTap()
896
918
  return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements"))
897
919
  }
898
920
  if let element = match.element {
@@ -911,6 +933,7 @@ extension RunnerTests {
911
933
  elementFrame: frame,
912
934
  windowFrame: onScreenWindowFrame(app: activeApp)
913
935
  ) {
936
+ clearRememberedTextEntryTap()
914
937
  return Response(ok: false, error: ErrorPayload(
915
938
  code: "ELEMENT_OFFSCREEN",
916
939
  message: "element resolved off-screen at (\(Int(frame.midX)), \(Int(frame.midY)))"))
@@ -946,6 +969,7 @@ extension RunnerTests {
946
969
  if isTextEntry {
947
970
  waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
948
971
  }
972
+ rememberTextEntryTap(isTextEntry ? element : nil)
949
973
  return gestureResponse(
950
974
  message: match.usedNonHittableFallback
951
975
  ? "tapped via non-hittable coordinate fallback"
@@ -976,11 +1000,13 @@ extension RunnerTests {
976
1000
  return activateElement(app: activeApp, element: element, action: "tap by selector")
977
1001
  }
978
1002
  if let response = unsupportedResponse(for: outcome) {
1003
+ clearRememberedTextEntryTap()
979
1004
  return response
980
1005
  }
981
1006
  if isTextEntry {
982
1007
  waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
983
1008
  }
1009
+ rememberTextEntryTap(isTextEntry ? element : nil)
984
1010
  return gestureResponse(
985
1011
  message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" : "tapped",
986
1012
  timing: timing,
@@ -992,9 +1018,27 @@ extension RunnerTests {
992
1018
  : nil
993
1019
  )
994
1020
  }
1021
+ clearRememberedTextEntryTap()
995
1022
  return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found"))
996
1023
  }
997
1024
  if let x = command.x, let y = command.y {
1025
+ let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(
1026
+ bundleId: currentBundleId
1027
+ )
1028
+ let xCTestTextInputProbeSkipped = !shouldProbeCoordinateTapTextInput(
1029
+ xCTestChannelPenalized: xCTestChannelPenalized
1030
+ )
1031
+ let textInput: XCUIElement?
1032
+ if !xCTestTextInputProbeSkipped {
1033
+ textInput = textInputAt(app: activeApp, x: x, y: y)
1034
+ } else {
1035
+ // A process-scoped tap cannot authorize later typing without concrete element identity.
1036
+ textInput = nil
1037
+ NSLog(
1038
+ "AGENT_DEVICE_RUNNER_COORDINATE_TAP_TEXT_INPUT_PROBE_SKIPPED bundle=%@",
1039
+ currentBundleId ?? ""
1040
+ )
1041
+ }
998
1042
  var fallback: GestureFallback?
999
1043
  if command.synthesized == true {
1000
1044
  let policyKind = SynthesizedGesturePolicyKind.coordinateTap
@@ -1007,6 +1051,7 @@ extension RunnerTests {
1007
1051
  }
1008
1052
  if case .performed = outcome {
1009
1053
  logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false)
1054
+ rememberTextEntryTap(textInput)
1010
1055
  return gestureResponse(message: "tapped", timing: timing)
1011
1056
  }
1012
1057
  logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true)
@@ -1015,8 +1060,10 @@ extension RunnerTests {
1015
1060
  let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
1016
1061
  let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) }
1017
1062
  if let response = unsupportedResponse(for: outcome) {
1063
+ clearRememberedTextEntryTap()
1018
1064
  return response
1019
1065
  }
1066
+ rememberTextEntryTap(textInput)
1020
1067
  return gestureResponse(
1021
1068
  message: "tapped",
1022
1069
  timing: timing,
@@ -1024,6 +1071,7 @@ extension RunnerTests {
1024
1071
  fallback: fallback
1025
1072
  )
1026
1073
  }
1074
+ clearRememberedTextEntryTap()
1027
1075
  return Response(ok: false, error: ErrorPayload(message: "tap requires a selector or x/y"))
1028
1076
  case .mouseClick:
1029
1077
  guard let x = command.x, let y = command.y else {
@@ -1348,7 +1396,7 @@ extension RunnerTests {
1348
1396
  ok: false,
1349
1397
  error: ErrorPayload(
1350
1398
  code: "UNSUPPORTED_OPERATION",
1351
- message: "Unable to dismiss the iOS keyboard without a safe native dismiss control",
1399
+ message: "Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)",
1352
1400
  hint:
1353
1401
  "The on-screen keyboard usually does not block agent-device interactions: press the next target directly instead of retrying dismiss. If that press fails or reports no visible effect, scroll the target into view, or use keyboard enter to press the return key when submission is wanted."
1354
1402
  )
@@ -1360,7 +1408,8 @@ extension RunnerTests {
1360
1408
  message: "keyboardDismiss",
1361
1409
  visible: result.visible,
1362
1410
  wasVisible: result.wasVisible,
1363
- dismissed: result.dismissed
1411
+ dismissed: result.dismissed,
1412
+ keyboardDismissMechanism: result.mechanism?.rawValue
1364
1413
  )
1365
1414
  )
1366
1415
  case .keyboardReturn:
@@ -1404,7 +1453,8 @@ extension RunnerTests {
1404
1453
  error: ErrorPayload(code: "INVALID_ARGS", message: validationError)
1405
1454
  )
1406
1455
  }
1407
- if plannedGestureExecution(for: plan) == .fastSwipe {
1456
+ switch plannedGestureExecution(for: plan) {
1457
+ case .fastSwipe:
1408
1458
  // Validation above guarantees a non-empty, single-pointer path for this execution kind.
1409
1459
  let first = plan.pointers[0].samples.first!.point
1410
1460
  let last = plan.pointers[0].samples.last!.point
@@ -1422,14 +1472,12 @@ extension RunnerTests {
1422
1472
  synthesizedProfile: .fastSwipe
1423
1473
  )
1424
1474
  )
1475
+ case .sampled:
1476
+ let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
1477
+ sampledPlannedGesture(app: activeApp, plan: plan)
1478
+ }
1479
+ return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome)
1425
1480
  }
1426
- let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
1427
- sampledPlannedGesture(app: activeApp, plan: plan)
1428
- }
1429
- if let response = unsupportedResponse(for: outcome) {
1430
- return response
1431
- }
1432
- return gestureResponse(message: plan.intent, timing: timing)
1433
1481
  case .gestureViewport:
1434
1482
  let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
1435
1483
  guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else {
@@ -1684,11 +1732,11 @@ extension RunnerTests {
1684
1732
  return message == "scrolled" ? "scroll" : "drag"
1685
1733
  }
1686
1734
 
1687
- private func currentXCTestFailureCount() -> Int {
1735
+ func currentXCTestFailureCount() -> Int {
1688
1736
  return testRun?.failureCount ?? 0
1689
1737
  }
1690
1738
 
1691
- private func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool {
1739
+ func didRecordXCTestFailure(since failureCountBefore: Int) -> Bool {
1692
1740
  return currentXCTestFailureCount() > failureCountBefore
1693
1741
  }
1694
1742
 
@@ -1705,12 +1753,12 @@ extension RunnerTests {
1705
1753
  error: ErrorPayload(
1706
1754
  code: "XCTEST_RECORDED_FAILURE",
1707
1755
  message: "XCTest recorded a failure while executing \(command.command.rawValue); the action may not have been performed.",
1708
- hint: "The iOS runner session will be restarted. Retry after a fresh snapshot, or use screenshot plus coordinate commands when the accessibility tree is unavailable."
1756
+ hint: "The iOS runner session was invalidated. Re-observe with a fresh snapshot before retrying; if the accessibility tree is unavailable, use screenshot plus coordinate commands instead of retrying the tap blindly."
1709
1757
  )
1710
1758
  )
1711
1759
  }
1712
1760
 
1713
- private func runnerCommandFixture(_ json: String) throws -> Command {
1761
+ func runnerCommandFixture(_ json: String) throws -> Command {
1714
1762
  try JSONDecoder().decode(Command.self, from: Data(json.utf8))
1715
1763
  }
1716
1764
 
@@ -1789,16 +1837,70 @@ extension RunnerTests {
1789
1837
  return XCUIApplication(bundleIdentifier: bundleId)
1790
1838
  }
1791
1839
 
1792
- private func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
1840
+ func executeTypeCommand(activeApp: XCUIApplication, command: Command) -> Response {
1793
1841
  guard let text = command.text else {
1794
1842
  return Response(ok: false, error: ErrorPayload(message: "type requires text"))
1795
1843
  }
1796
1844
  let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0
1797
1845
  let textEntryMode = resolveTextEntryMode(command)
1798
1846
  let target: TextEntryTarget
1847
+ var resolvedCoordinateContext: SynthesizedCoordinateContext?
1799
1848
  var maestroNonHittableCoordinateFallbackUsed: Bool?
1849
+ if command.allowNonHittableCoordinateFallback == true,
1850
+ command.x != nil,
1851
+ command.y != nil
1852
+ {
1853
+ // The shared runtime has already resolved this node as non-hittable and
1854
+ // deliberately selected Maestro's coordinate compatibility route.
1855
+ maestroNonHittableCoordinateFallbackUsed = true
1856
+ }
1800
1857
  let focusStartedAt = Date()
1801
- if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
1858
+ #if os(iOS)
1859
+ let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
1860
+ var resolvedCoordinateTarget: TextEntryTarget?
1861
+ if Self.shouldUseResolvedCoordinateTextEntryRoute(
1862
+ repairMode: textEntryMode,
1863
+ hasX: command.x != nil,
1864
+ hasY: command.y != nil,
1865
+ xCTestChannelPenalized: xCTestChannelPenalized
1866
+ ), let x = command.x, let y = command.y {
1867
+ let policyKind = SynthesizedGesturePolicyKind.coordinateTap
1868
+ let context = synthesizedCoordinateContext(
1869
+ app: activeApp,
1870
+ policy: synthesizedGesturePolicy(policyKind)
1871
+ )
1872
+ let (_, outcome) = performGesture(activeApp, idleTimeout: false) {
1873
+ synthesizedTapAt(app: activeApp, x: x, y: y, context: context)
1874
+ }
1875
+ if Self.shouldFallbackFromSynthesizedTextEntryFocus(outcome) {
1876
+ logSynthesizedGesturePolicyDecision(
1877
+ kind: policyKind,
1878
+ context: context,
1879
+ fallbackAttempted: true
1880
+ )
1881
+ } else {
1882
+ logSynthesizedGesturePolicyDecision(
1883
+ kind: policyKind,
1884
+ context: context,
1885
+ fallbackAttempted: false
1886
+ )
1887
+ resolvedCoordinateContext = context
1888
+ resolvedCoordinateTarget = TextEntryTarget(
1889
+ element: nil,
1890
+ refreshPoint: CGPoint(x: x, y: y),
1891
+ prefersFocusedElement: false
1892
+ )
1893
+ }
1894
+ }
1895
+ #else
1896
+ let xCTestChannelPenalized = false
1897
+ let resolvedCoordinateTarget: TextEntryTarget? = nil
1898
+ #endif
1899
+ if let resolvedCoordinateTarget {
1900
+ target = resolvedCoordinateTarget
1901
+ } else if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
1902
+ // Released daemons may still send selector-keyed type commands even though current
1903
+ // daemons resolve fill selectors through the runtime tree before reaching the runner.
1802
1904
  let match = findElement(
1803
1905
  app: activeApp,
1804
1906
  selectorKey: selectorKey,
@@ -1829,7 +1931,16 @@ extension RunnerTests {
1829
1931
  textEntryModeName(textEntryMode)
1830
1932
  )
1831
1933
  if textEntryMode == .replacement {
1832
- guard target.element != nil else {
1934
+ #if os(iOS)
1935
+ let canReplaceResolvedFirstResponder = Self.shouldUseSynthesizedFirstResponderReplacement(
1936
+ hasResolvedElement: target.element != nil,
1937
+ hasRefreshPoint: target.refreshPoint != nil,
1938
+ xCTestChannelPenalized: xCTestChannelPenalized
1939
+ )
1940
+ #else
1941
+ let canReplaceResolvedFirstResponder = false
1942
+ #endif
1943
+ guard target.element != nil || canReplaceResolvedFirstResponder else {
1833
1944
  let message =
1834
1945
  (command.x != nil && command.y != nil)
1835
1946
  ? "no text input found at the provided coordinates to clear"
@@ -1843,8 +1954,16 @@ extension RunnerTests {
1843
1954
  text: text,
1844
1955
  delaySeconds: delaySeconds,
1845
1956
  repairMode: textEntryMode,
1957
+ xCTestChannelPenalized: xCTestChannelPenalized,
1958
+ synthesizer: PrivateXCTestTextEntrySynthesizer(),
1846
1959
  commandId: command.commandId
1847
1960
  )
1961
+ if let failure = textResult.failure {
1962
+ return Response(
1963
+ ok: false,
1964
+ error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)
1965
+ )
1966
+ }
1848
1967
  if textResult.verified == false {
1849
1968
  let expected = textResult.expectedText ?? ""
1850
1969
  let observed = textResult.observedText ?? ""
@@ -1857,7 +1976,16 @@ extension RunnerTests {
1857
1976
  )
1858
1977
  }
1859
1978
  let point = target.refreshPoint
1860
- let frame = activeApp.frame
1979
+ let frame: CGRect
1980
+ if let resolvedCoordinateContext {
1981
+ frame = resolvedCoordinateContext.referenceFrame
1982
+ } else if point != nil {
1983
+ frame = activeApp.frame
1984
+ } else {
1985
+ // Bare `type` has no coordinate response to normalize. Avoid serializing the
1986
+ // application AX tree only to emit unused reference dimensions.
1987
+ frame = .zero
1988
+ }
1861
1989
  return Response(
1862
1990
  ok: true,
1863
1991
  data: DataPayload(
@@ -1866,7 +1994,8 @@ extension RunnerTests {
1866
1994
  y: point.map { Double($0.y) },
1867
1995
  referenceWidth: frame.isEmpty ? nil : Double(frame.width),
1868
1996
  referenceHeight: frame.isEmpty ? nil : Double(frame.height),
1869
- maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
1997
+ maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed,
1998
+ textEntryRoute: textResult.textEntryRoute
1870
1999
  )
1871
2000
  )
1872
2001
  }
@@ -129,7 +129,8 @@ extension RunnerTests {
129
129
  selectorKey: String,
130
130
  selectorValue: String,
131
131
  allowNonHittableFallback: Bool = false,
132
- expectedPoint: CGPoint? = nil
132
+ expectedPoint: CGPoint? = nil,
133
+ rawMatchPolicy: DirectSelectorRawMatchPolicy = .rejectDistinctMatches
133
134
  ) -> SelectorElementMatch {
134
135
  let value = selectorValue.trimmingCharacters(in: .whitespacesAndNewlines)
135
136
  guard !value.isEmpty else {
@@ -149,35 +150,32 @@ extension RunnerTests {
149
150
  return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false)
150
151
  }
151
152
 
152
- var matchedElement: XCUIElement?
153
- var nonHittableElement: XCUIElement?
154
153
  let matches = app.descendants(matching: .any).matching(predicate).allElementsBoundByIndex
155
- for element in matches where element.exists {
156
- if let expectedPoint, !element.frame.contains(expectedPoint) {
157
- continue
158
- }
159
- if !element.isHittable {
160
- if allowNonHittableFallback && hasTappableFrame(app: app, element: element) {
161
- guard nonHittableElement == nil else {
162
- return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false)
163
- }
164
- nonHittableElement = element
165
- }
166
- continue
167
- }
168
- guard matchedElement == nil else {
169
- return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false)
170
- }
171
- matchedElement = element
154
+ .filter(\.exists)
155
+ let facts = matches.map { element in
156
+ SelectorCandidateFacts(
157
+ isHittable: element.isHittable,
158
+ hasTappableFrame: hasTappableFrame(app: app, element: element),
159
+ containsExpectedPoint: expectedPoint.map(element.frame.contains) ?? true
160
+ )
172
161
  }
173
- if let matchedElement {
174
- return SelectorElementMatch(element: matchedElement, isAmbiguous: false, usedNonHittableFallback: false)
162
+ switch classifyDirectSelectorCandidates(
163
+ facts,
164
+ allowNonHittableFallback: allowNonHittableFallback,
165
+ filtersByExpectedPoint: expectedPoint != nil,
166
+ rawMatchPolicy: rawMatchPolicy
167
+ ) {
168
+ case .noMatch:
169
+ return SelectorElementMatch(element: nil, isAmbiguous: false, usedNonHittableFallback: false)
170
+ case .ambiguous:
171
+ return SelectorElementMatch(element: nil, isAmbiguous: true, usedNonHittableFallback: false)
172
+ case let .selected(index, usedNonHittableFallback):
173
+ return SelectorElementMatch(
174
+ element: matches[index],
175
+ isAmbiguous: false,
176
+ usedNonHittableFallback: usedNonHittableFallback
177
+ )
175
178
  }
176
- return SelectorElementMatch(
177
- element: nonHittableElement,
178
- isAmbiguous: false,
179
- usedNonHittableFallback: nonHittableElement != nil
180
- )
181
179
  }
182
180
 
183
181
  // Maestro-compat gate for the non-hittable coordinate fallback: an element
@@ -209,7 +207,17 @@ extension RunnerTests {
209
207
  }
210
208
 
211
209
  func queryElement(app: XCUIApplication, selectorKey: String, selectorValue: String) -> Response {
212
- let match = findElement(app: app, selectorKey: selectorKey, selectorValue: selectorValue)
210
+ // querySelector is a read — it backs get/is/wait and the offscreen-refusal
211
+ // double-check, none of which mutate. The fail-closed raw-match rule exists
212
+ // to stop a mutation acting on an unseen duplicate; applying it here would
213
+ // instead turn a decorative non-hittable duplicate into an AMBIGUOUS_MATCH
214
+ // for readers that previously resolved the hittable element.
215
+ let match = findElement(
216
+ app: app,
217
+ selectorKey: selectorKey,
218
+ selectorValue: selectorValue,
219
+ rawMatchPolicy: .preferHittableMatch
220
+ )
213
221
  if match.isAmbiguous {
214
222
  return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements"))
215
223
  }
@@ -21,23 +21,34 @@ private enum KeyboardDismissObservationTiming {
21
21
  static let settleRequiredConsecutiveMatches: Int = 3
22
22
  }
23
23
 
24
+ // The mechanism that actually resigned the keyboard, disclosed to the caller
25
+ // (#1598) so a response never claims "dismissed" without saying how — a
26
+ // safe-area tap has a very different reliability/side-effect profile than
27
+ // tapping the keyboard's own Done key, and callers need to know which one
28
+ // fired.
29
+ enum RunnerKeyboardDismissMechanism: String {
30
+ case dismissKey
31
+ }
32
+
24
33
  extension RunnerTests {
25
34
  func isKeyboardVisible(app: XCUIApplication) -> Bool {
26
35
  return visibleKeyboardFrame(app: app) != nil
27
36
  }
28
37
 
29
- func dismissKeyboard(app: XCUIApplication) -> (wasVisible: Bool, dismissed: Bool, visible: Bool) {
38
+ func dismissKeyboard(
39
+ app: XCUIApplication
40
+ ) -> (wasVisible: Bool, dismissed: Bool, visible: Bool, mechanism: RunnerKeyboardDismissMechanism?) {
30
41
  let keyboard = app.keyboards.firstMatch
31
42
  let wasVisible = isKeyboardVisible(app: app)
32
43
  guard wasVisible else {
33
- return (wasVisible: false, dismissed: false, visible: false)
44
+ return (wasVisible: false, dismissed: false, visible: false, mechanism: nil)
34
45
  }
35
46
 
36
47
  #if os(tvOS)
37
48
  _ = pressTvRemote(.menu)
38
49
  sleepFor(0.2)
39
50
  let visible = isKeyboardVisible(app: app)
40
- return (wasVisible: true, dismissed: !visible, visible: visible)
51
+ return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey)
41
52
  #else
42
53
  if tapKeyboardDismissControl(app: app) {
43
54
  _ = keyboard.waitForNonExistence(timeout: KeyboardDismissObservationTiming.timeout)
@@ -47,13 +58,20 @@ extension RunnerTests {
47
58
  requiredConsecutiveMatches: KeyboardDismissObservationTiming.settleRequiredConsecutiveMatches
48
59
  )
49
60
  let visible = isKeyboardVisible(app: app)
50
- return (wasVisible: true, dismissed: !visible, visible: visible)
61
+ return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey)
51
62
  }
52
63
 
53
- return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app))
64
+ // #1606 review P1 (twice): generic background-tap dismissal is
65
+ // deliberately UNSUPPORTED. No geometry or role query can prove a
66
+ // coordinate is side-effect-free — a full-screen unlabeled Pressable is
67
+ // indistinguishable from an inert backdrop, so a "safe-area" tap can
68
+ // navigate or submit while reporting a successful dismiss. The dismiss
69
+ // key is the only mechanism the runner can vouch for.
70
+ return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app), mechanism: nil)
54
71
  #endif
55
72
  }
56
73
 
74
+
57
75
  // AX-free on purpose (screenshot bytes, not the accessibility tree) so it holds
58
76
  // under the same AX degradation the synthesized gesture lane is built to survive.
59
77
  // Bounded and self-terminating: returns as soon as `requiredConsecutiveMatches`
@@ -86,6 +86,7 @@ extension RunnerTests {
86
86
  currentApp = app
87
87
  currentBundleId = nil
88
88
  currentAppProcessIdentifier = nil
89
+ clearRememberedTextEntryTap()
89
90
  snapshotXCTestPenaltyWarmupExemptionPending = false
90
91
  }
91
92
 
@@ -96,11 +97,17 @@ extension RunnerTests {
96
97
  currentApp = nil
97
98
  currentBundleId = nil
98
99
  currentAppProcessIdentifier = nil
100
+ clearRememberedTextEntryTap()
99
101
  snapshotXCTestPenaltyWarmupExemptionPending = false
100
102
  }
101
103
 
102
104
  func resetTargetAfterExternalRelaunch() -> Response {
103
105
  invalidateCachedTarget(reason: "external_app_relaunch")
106
+ // The app process is replaced, but the retained runner survives. Clear
107
+ // process-bound capture state explicitly because invalidation drops the
108
+ // old PID before refreshCachedTargetIfProcessChanged can observe it.
109
+ clearSnapshotXCTestChannelPenalty(reason: "external_app_relaunch")
110
+ clearPrivateAXAcceptedDepth(reason: "external_app_relaunch")
104
111
  needsFirstInteractionDelay = true
105
112
  return Response(ok: true, data: DataPayload(message: "target reset"))
106
113
  }
@@ -121,7 +128,9 @@ extension RunnerTests {
121
128
  )
122
129
  currentApp = candidate
123
130
  currentAppProcessIdentifier = observedProcessIdentifier
131
+ clearRememberedTextEntryTap()
124
132
  clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
133
+ clearPrivateAXAcceptedDepth(reason: "target_process_changed")
125
134
  snapshotXCTestPenaltyWarmupExemptionPending = true
126
135
  needsFirstInteractionDelay = true
127
136
  }
@@ -194,6 +203,7 @@ extension RunnerTests {
194
203
  currentApp = target
195
204
  currentBundleId = bundleId
196
205
  currentAppProcessIdentifier = Self.processIdentifier(of: target)
206
+ clearRememberedTextEntryTap()
197
207
  snapshotXCTestPenaltyWarmupExemptionPending = false
198
208
  needsFirstInteractionDelay = true
199
209
  return target
@@ -3,71 +3,4 @@ import XCTest
3
3
 
4
4
  extension RunnerTests {
5
5
 
6
- func testCachedTargetRefreshRequiresChangedPositiveProcessIdentity() {
7
- XCTAssertFalse(
8
- Self.shouldRefreshCachedTarget(
9
- cachedProcessIdentifier: nil,
10
- observedProcessIdentifier: 42
11
- )
12
- )
13
- XCTAssertFalse(
14
- Self.shouldRefreshCachedTarget(
15
- cachedProcessIdentifier: 42,
16
- observedProcessIdentifier: 42
17
- )
18
- )
19
- XCTAssertTrue(
20
- Self.shouldRefreshCachedTarget(
21
- cachedProcessIdentifier: 41,
22
- observedProcessIdentifier: 42
23
- )
24
- )
25
- }
26
-
27
- func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() {
28
- snapshotXCTestPenaltyWarmupExemptionPending = true
29
-
30
- XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption())
31
- XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption())
32
- }
33
-
34
- func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() {
35
- penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test")
36
- XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
37
-
38
- clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
39
-
40
- XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
41
- }
42
-
43
- func testCachedTargetInvalidationClearsProcessBoundState() {
44
- currentApp = app
45
- currentBundleId = "com.example.app"
46
- currentAppProcessIdentifier = 42
47
- snapshotXCTestPenaltyWarmupExemptionPending = true
48
-
49
- invalidateCachedTarget(reason: "unit_test")
50
-
51
- XCTAssertNil(currentApp)
52
- XCTAssertNil(currentBundleId)
53
- XCTAssertNil(currentAppProcessIdentifier)
54
- XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
55
- }
56
-
57
- func testTargetResetInvalidatesProcessBoundStateWithoutRestartingRunner() {
58
- currentApp = app
59
- currentBundleId = "com.example.app"
60
- currentAppProcessIdentifier = 42
61
- snapshotXCTestPenaltyWarmupExemptionPending = true
62
- needsFirstInteractionDelay = false
63
-
64
- let response = resetTargetAfterExternalRelaunch()
65
-
66
- XCTAssertTrue(response.ok)
67
- XCTAssertNil(currentApp)
68
- XCTAssertNil(currentBundleId)
69
- XCTAssertNil(currentAppProcessIdentifier)
70
- XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
71
- XCTAssertTrue(needsFirstInteractionDelay)
72
- }
73
6
  }