agent-device 0.20.5 → 0.20.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.apk → agent-device-android-ime-helper-0.20.6.apk} +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.6.apk.sha256 +1 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.manifest.json → agent-device-android-ime-helper-0.20.6.manifest.json} +4 -4
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.apk → agent-device-android-snapshot-helper-0.20.6.apk} +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.6.apk.sha256 +1 -0
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.manifest.json → agent-device-android-snapshot-helper-0.20.6.manifest.json} +6 -6
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +28 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +41 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +298 -92
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +126 -187
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +32 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +212 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +118 -8
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +134 -27
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +23 -5
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +10 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +0 -67
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +11 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +2 -22
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +48 -11
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +0 -20
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +177 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +105 -448
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +4 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +587 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +18 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +72 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +89 -0
- package/dist/src/agent-device-client.js +2 -2
- package/dist/src/android.js +1 -1
- package/dist/src/app-log-request-scope.js +1 -1
- package/dist/src/app-resolution.js +1 -1
- package/dist/src/apps.js +5 -5
- package/dist/src/cli-config.js +1 -1
- package/dist/src/cli-help.js +34 -24
- package/dist/src/cli-runner.js +1 -1
- package/dist/src/cli.js +15 -15
- package/dist/src/command-metadata.js +1 -1
- package/dist/src/command-schema.js +3 -3
- package/dist/src/context.js +1 -1
- package/dist/src/debug-symbols.js +1 -1
- package/dist/src/find.js +1 -1
- package/dist/src/finders.js +1 -1
- package/dist/src/generic-settle.js +1 -0
- package/dist/src/generic.js +1 -1
- package/dist/src/ime-lifecycle.js +3 -3
- package/dist/src/index.d.ts +27 -7
- package/dist/src/input-actions.js +4 -4
- package/dist/src/interaction-runtime.js +1 -0
- package/dist/src/interaction-snapshot.js +1 -1
- package/dist/src/interaction-touch-reference-frame.js +1 -0
- package/dist/src/interaction.js +1 -1
- package/dist/src/interactor.js +2 -2
- package/dist/src/interactor2.js +1 -1
- package/dist/src/internal/bin.js +2 -2
- package/dist/src/internal/daemon.js +15 -15
- package/dist/src/lease-context.js +1 -1
- package/dist/src/lease.js +1 -1
- package/dist/src/limrun.d.ts +23 -2
- package/dist/src/linux.js +1 -1
- package/dist/src/process-exit.js +1 -0
- package/dist/src/react-native.js +1 -1
- package/dist/src/record-trace.js +1 -1
- package/dist/src/register-builtins.js +1 -1
- package/dist/src/registry.js +25 -24
- package/dist/src/remote-config.js +1 -1
- package/dist/src/remote-config2.js +1 -1
- package/dist/src/reporting.js +5 -5
- package/dist/src/runner-client.js +1 -1
- package/dist/src/runner-disposal.js +1 -1
- package/dist/src/runner-lease.js +1 -1
- package/dist/src/runtime.js +3 -3
- package/dist/src/screenshot-geometry.js +1 -1
- package/dist/src/screenshot-result.js +4 -4
- package/dist/src/sdk-android-adb.d.ts +20 -2
- package/dist/src/sdk-android-adb.js +1 -1
- package/dist/src/sdk-batch-runner.js +3 -3
- package/dist/src/sdk-contracts.d.ts +17 -2
- package/dist/src/sdk-finders.d.ts +4 -20
- package/dist/src/sdk-finders.js +1 -1
- package/dist/src/{interaction.d.ts → sdk-selectors.d.ts} +98 -3
- package/dist/src/sdk-selectors.js +1 -1
- package/dist/src/selector-runtime.js +1 -1
- package/dist/src/selector-vocabulary.js +1 -1
- package/dist/src/selectors.d.ts +1 -46
- package/dist/src/selectors.js +1 -1
- package/dist/src/server.js +3 -3
- package/dist/src/session-target-evidence.js +1 -1
- package/dist/src/session.js +9 -9
- package/dist/src/snapshot2.js +1 -1
- package/dist/src/text-surface.js +1 -0
- package/dist/src/viewport-dimension.js +13 -12
- package/dist/src/web.js +1 -1
- package/package.json +41 -35
- package/server.json +3 -3
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.5.apk.sha256 +0 -1
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.5.apk.sha256 +0 -1
- package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -225
|
@@ -152,6 +152,17 @@ extension RunnerTests {
|
|
|
152
152
|
)
|
|
153
153
|
}
|
|
154
154
|
|
|
155
|
+
private func plannedGestureResponse(
|
|
156
|
+
plan: RunnerGesturePlan,
|
|
157
|
+
timing: (gestureStartUptimeMs: Double, gestureEndUptimeMs: Double),
|
|
158
|
+
outcome: RunnerInteractionOutcome
|
|
159
|
+
) -> Response {
|
|
160
|
+
if let response = unsupportedResponse(for: outcome) {
|
|
161
|
+
return response
|
|
162
|
+
}
|
|
163
|
+
return gestureResponse(message: plan.intent, timing: timing)
|
|
164
|
+
}
|
|
165
|
+
|
|
155
166
|
|
|
156
167
|
func execute(command: Command) throws -> Response {
|
|
157
168
|
if command.command == .status {
|
|
@@ -572,13 +583,20 @@ extension RunnerTests {
|
|
|
572
583
|
}
|
|
573
584
|
}
|
|
574
585
|
|
|
575
|
-
|
|
576
|
-
|
|
586
|
+
/// Pure command→options projection, extracted so the runner unit bundle can
|
|
587
|
+
/// prove the decoded wire field actually reaches capture options (#1634 P2).
|
|
588
|
+
static func snapshotOptions(from command: Command) -> SnapshotOptions {
|
|
589
|
+
SnapshotOptions(
|
|
577
590
|
interactiveOnly: command.interactiveOnly ?? false,
|
|
578
591
|
depth: command.depth,
|
|
579
592
|
scope: command.scope,
|
|
580
|
-
raw: command.raw ?? false
|
|
593
|
+
raw: command.raw ?? false,
|
|
594
|
+
preferredBackend: command.preferredBackend
|
|
581
595
|
)
|
|
596
|
+
}
|
|
597
|
+
|
|
598
|
+
private func executeSnapshotPrepared(command: Command, activeApp: XCUIApplication) throws -> Response {
|
|
599
|
+
let options = Self.snapshotOptions(from: command)
|
|
582
600
|
do {
|
|
583
601
|
let payload: DataPayload
|
|
584
602
|
if options.raw {
|
|
@@ -713,24 +731,18 @@ extension RunnerTests {
|
|
|
713
731
|
if let requestedFps = command.fps, (requestedFps < minRecordingFps || requestedFps > maxRecordingFps) {
|
|
714
732
|
return Response(ok: false, error: ErrorPayload(message: "recordStart fps must be between \(minRecordingFps) and \(maxRecordingFps)"))
|
|
715
733
|
}
|
|
716
|
-
if let requestedMaxSize = command.maxSize, requestedMaxSize < 1 {
|
|
717
|
-
return Response(ok: false, error: ErrorPayload(message: "recordStart maxSize must be a positive integer"))
|
|
718
|
-
}
|
|
719
734
|
do {
|
|
720
735
|
let resolvedOutPath = resolveRecordingOutPath(requestedOutPath)
|
|
721
736
|
let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps)
|
|
722
|
-
let maxSizeLabel = command.maxSize.map(String.init) ?? "native"
|
|
723
737
|
NSLog(
|
|
724
|
-
"AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@
|
|
738
|
+
"AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@",
|
|
725
739
|
requestedOutPath,
|
|
726
740
|
resolvedOutPath,
|
|
727
|
-
fpsLabel
|
|
728
|
-
maxSizeLabel
|
|
741
|
+
fpsLabel
|
|
729
742
|
)
|
|
730
743
|
let recorder = ScreenRecorder(
|
|
731
744
|
outputPath: resolvedOutPath,
|
|
732
|
-
fps: command.fps.map { Int32($0) }
|
|
733
|
-
maxSize: command.maxSize
|
|
745
|
+
fps: command.fps.map { Int32($0) }
|
|
734
746
|
)
|
|
735
747
|
try recorder.start { [weak self] in
|
|
736
748
|
return self?.captureRunnerFrame()
|
|
@@ -865,6 +877,9 @@ extension RunnerTests {
|
|
|
865
877
|
alertDeadline: Date? = nil
|
|
866
878
|
) throws -> Response {
|
|
867
879
|
var activeApp = activeApp
|
|
880
|
+
if command.command != .tap && command.command != .type && !isReadOnlyCommand(command) {
|
|
881
|
+
clearRememberedTextEntryTap()
|
|
882
|
+
}
|
|
868
883
|
switch command.command {
|
|
869
884
|
case .status, .activate, .terminate, .targetReset, .shutdown, .recordStart, .recordStop, .uptime:
|
|
870
885
|
return Response(
|
|
@@ -893,6 +908,7 @@ extension RunnerTests {
|
|
|
893
908
|
expectedPoint: expectedPoint
|
|
894
909
|
)
|
|
895
910
|
if match.isAmbiguous {
|
|
911
|
+
clearRememberedTextEntryTap()
|
|
896
912
|
return Response(ok: false, error: ErrorPayload(code: "AMBIGUOUS_MATCH", message: "selector matched multiple elements"))
|
|
897
913
|
}
|
|
898
914
|
if let element = match.element {
|
|
@@ -911,6 +927,7 @@ extension RunnerTests {
|
|
|
911
927
|
elementFrame: frame,
|
|
912
928
|
windowFrame: onScreenWindowFrame(app: activeApp)
|
|
913
929
|
) {
|
|
930
|
+
clearRememberedTextEntryTap()
|
|
914
931
|
return Response(ok: false, error: ErrorPayload(
|
|
915
932
|
code: "ELEMENT_OFFSCREEN",
|
|
916
933
|
message: "element resolved off-screen at (\(Int(frame.midX)), \(Int(frame.midY)))"))
|
|
@@ -946,6 +963,7 @@ extension RunnerTests {
|
|
|
946
963
|
if isTextEntry {
|
|
947
964
|
waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
|
|
948
965
|
}
|
|
966
|
+
rememberTextEntryTap(isTextEntry ? element : nil)
|
|
949
967
|
return gestureResponse(
|
|
950
968
|
message: match.usedNonHittableFallback
|
|
951
969
|
? "tapped via non-hittable coordinate fallback"
|
|
@@ -976,11 +994,13 @@ extension RunnerTests {
|
|
|
976
994
|
return activateElement(app: activeApp, element: element, action: "tap by selector")
|
|
977
995
|
}
|
|
978
996
|
if let response = unsupportedResponse(for: outcome) {
|
|
997
|
+
clearRememberedTextEntryTap()
|
|
979
998
|
return response
|
|
980
999
|
}
|
|
981
1000
|
if isTextEntry {
|
|
982
1001
|
waitForTextEntryReadinessAfterTap(app: activeApp, element: element)
|
|
983
1002
|
}
|
|
1003
|
+
rememberTextEntryTap(isTextEntry ? element : nil)
|
|
984
1004
|
return gestureResponse(
|
|
985
1005
|
message: match.usedNonHittableFallback ? "tapped via non-hittable coordinate fallback" : "tapped",
|
|
986
1006
|
timing: timing,
|
|
@@ -992,9 +1012,11 @@ extension RunnerTests {
|
|
|
992
1012
|
: nil
|
|
993
1013
|
)
|
|
994
1014
|
}
|
|
1015
|
+
clearRememberedTextEntryTap()
|
|
995
1016
|
return Response(ok: false, error: ErrorPayload(code: "ELEMENT_NOT_FOUND", message: "element not found"))
|
|
996
1017
|
}
|
|
997
1018
|
if let x = command.x, let y = command.y {
|
|
1019
|
+
let textInput = textInputAt(app: activeApp, x: x, y: y)
|
|
998
1020
|
var fallback: GestureFallback?
|
|
999
1021
|
if command.synthesized == true {
|
|
1000
1022
|
let policyKind = SynthesizedGesturePolicyKind.coordinateTap
|
|
@@ -1007,6 +1029,7 @@ extension RunnerTests {
|
|
|
1007
1029
|
}
|
|
1008
1030
|
if case .performed = outcome {
|
|
1009
1031
|
logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: false)
|
|
1032
|
+
rememberTextEntryTap(textInput)
|
|
1010
1033
|
return gestureResponse(message: "tapped", timing: timing)
|
|
1011
1034
|
}
|
|
1012
1035
|
logSynthesizedGesturePolicyDecision(kind: policyKind, context: context, fallbackAttempted: true)
|
|
@@ -1015,8 +1038,10 @@ extension RunnerTests {
|
|
|
1015
1038
|
let touchFrame = resolvedTouchVisualizationFrame(app: activeApp, x: x, y: y)
|
|
1016
1039
|
let (timing, outcome) = performGesture(activeApp) { tapAt(app: activeApp, x: x, y: y) }
|
|
1017
1040
|
if let response = unsupportedResponse(for: outcome) {
|
|
1041
|
+
clearRememberedTextEntryTap()
|
|
1018
1042
|
return response
|
|
1019
1043
|
}
|
|
1044
|
+
rememberTextEntryTap(textInput)
|
|
1020
1045
|
return gestureResponse(
|
|
1021
1046
|
message: "tapped",
|
|
1022
1047
|
timing: timing,
|
|
@@ -1024,6 +1049,7 @@ extension RunnerTests {
|
|
|
1024
1049
|
fallback: fallback
|
|
1025
1050
|
)
|
|
1026
1051
|
}
|
|
1052
|
+
clearRememberedTextEntryTap()
|
|
1027
1053
|
return Response(ok: false, error: ErrorPayload(message: "tap requires a selector or x/y"))
|
|
1028
1054
|
case .mouseClick:
|
|
1029
1055
|
guard let x = command.x, let y = command.y else {
|
|
@@ -1348,7 +1374,7 @@ extension RunnerTests {
|
|
|
1348
1374
|
ok: false,
|
|
1349
1375
|
error: ErrorPayload(
|
|
1350
1376
|
code: "UNSUPPORTED_OPERATION",
|
|
1351
|
-
message: "Unable to dismiss the iOS keyboard
|
|
1377
|
+
message: "Unable to dismiss the iOS keyboard: the keyboard exposes no dismiss key, and background taps are never attempted (no tap outside the keyboard can be proven side-effect-free)",
|
|
1352
1378
|
hint:
|
|
1353
1379
|
"The on-screen keyboard usually does not block agent-device interactions: press the next target directly instead of retrying dismiss. If that press fails or reports no visible effect, scroll the target into view, or use keyboard enter to press the return key when submission is wanted."
|
|
1354
1380
|
)
|
|
@@ -1360,7 +1386,8 @@ extension RunnerTests {
|
|
|
1360
1386
|
message: "keyboardDismiss",
|
|
1361
1387
|
visible: result.visible,
|
|
1362
1388
|
wasVisible: result.wasVisible,
|
|
1363
|
-
dismissed: result.dismissed
|
|
1389
|
+
dismissed: result.dismissed,
|
|
1390
|
+
keyboardDismissMechanism: result.mechanism?.rawValue
|
|
1364
1391
|
)
|
|
1365
1392
|
)
|
|
1366
1393
|
case .keyboardReturn:
|
|
@@ -1404,7 +1431,8 @@ extension RunnerTests {
|
|
|
1404
1431
|
error: ErrorPayload(code: "INVALID_ARGS", message: validationError)
|
|
1405
1432
|
)
|
|
1406
1433
|
}
|
|
1407
|
-
|
|
1434
|
+
switch plannedGestureExecution(for: plan) {
|
|
1435
|
+
case .fastSwipe:
|
|
1408
1436
|
// Validation above guarantees a non-empty, single-pointer path for this execution kind.
|
|
1409
1437
|
let first = plan.pointers[0].samples.first!.point
|
|
1410
1438
|
let last = plan.pointers[0].samples.last!.point
|
|
@@ -1422,14 +1450,12 @@ extension RunnerTests {
|
|
|
1422
1450
|
synthesizedProfile: .fastSwipe
|
|
1423
1451
|
)
|
|
1424
1452
|
)
|
|
1453
|
+
case .sampled:
|
|
1454
|
+
let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
|
|
1455
|
+
sampledPlannedGesture(app: activeApp, plan: plan)
|
|
1456
|
+
}
|
|
1457
|
+
return plannedGestureResponse(plan: plan, timing: timing, outcome: outcome)
|
|
1425
1458
|
}
|
|
1426
|
-
let (timing, outcome) = performGesture(activeApp, idleTimeout: false) {
|
|
1427
|
-
sampledPlannedGesture(app: activeApp, plan: plan)
|
|
1428
|
-
}
|
|
1429
|
-
if let response = unsupportedResponse(for: outcome) {
|
|
1430
|
-
return response
|
|
1431
|
-
}
|
|
1432
|
-
return gestureResponse(message: plan.intent, timing: timing)
|
|
1433
1459
|
case .gestureViewport:
|
|
1434
1460
|
let frame = resolvedTouchReferenceFrame(app: activeApp, appFrame: activeApp.frame)
|
|
1435
1461
|
guard !frame.isNull, !frame.isInfinite, !frame.isEmpty else {
|
|
@@ -1705,7 +1731,7 @@ extension RunnerTests {
|
|
|
1705
1731
|
error: ErrorPayload(
|
|
1706
1732
|
code: "XCTEST_RECORDED_FAILURE",
|
|
1707
1733
|
message: "XCTest recorded a failure while executing \(command.command.rawValue); the action may not have been performed.",
|
|
1708
|
-
hint: "The iOS runner session
|
|
1734
|
+
hint: "The iOS runner session was invalidated. Re-observe with a fresh snapshot before retrying; if the accessibility tree is unavailable, use screenshot plus coordinate commands instead of retrying the tap blindly."
|
|
1709
1735
|
)
|
|
1710
1736
|
)
|
|
1711
1737
|
}
|
|
@@ -1796,9 +1822,63 @@ extension RunnerTests {
|
|
|
1796
1822
|
let delaySeconds = Double(max(command.delayMs ?? 0, 0)) / 1000.0
|
|
1797
1823
|
let textEntryMode = resolveTextEntryMode(command)
|
|
1798
1824
|
let target: TextEntryTarget
|
|
1825
|
+
var resolvedCoordinateContext: SynthesizedCoordinateContext?
|
|
1799
1826
|
var maestroNonHittableCoordinateFallbackUsed: Bool?
|
|
1827
|
+
if command.allowNonHittableCoordinateFallback == true,
|
|
1828
|
+
command.x != nil,
|
|
1829
|
+
command.y != nil
|
|
1830
|
+
{
|
|
1831
|
+
// The shared runtime has already resolved this node as non-hittable and
|
|
1832
|
+
// deliberately selected Maestro's coordinate compatibility route.
|
|
1833
|
+
maestroNonHittableCoordinateFallbackUsed = true
|
|
1834
|
+
}
|
|
1800
1835
|
let focusStartedAt = Date()
|
|
1801
|
-
|
|
1836
|
+
#if os(iOS)
|
|
1837
|
+
let xCTestChannelPenalized = isSnapshotXCTestChannelPenalized(bundleId: currentBundleId)
|
|
1838
|
+
var resolvedCoordinateTarget: TextEntryTarget?
|
|
1839
|
+
if Self.shouldUseResolvedCoordinateTextEntryRoute(
|
|
1840
|
+
repairMode: textEntryMode,
|
|
1841
|
+
hasX: command.x != nil,
|
|
1842
|
+
hasY: command.y != nil,
|
|
1843
|
+
xCTestChannelPenalized: xCTestChannelPenalized
|
|
1844
|
+
), let x = command.x, let y = command.y {
|
|
1845
|
+
let policyKind = SynthesizedGesturePolicyKind.coordinateTap
|
|
1846
|
+
let context = synthesizedCoordinateContext(
|
|
1847
|
+
app: activeApp,
|
|
1848
|
+
policy: synthesizedGesturePolicy(policyKind)
|
|
1849
|
+
)
|
|
1850
|
+
let (_, outcome) = performGesture(activeApp, idleTimeout: false) {
|
|
1851
|
+
synthesizedTapAt(app: activeApp, x: x, y: y, context: context)
|
|
1852
|
+
}
|
|
1853
|
+
if Self.shouldFallbackFromSynthesizedTextEntryFocus(outcome) {
|
|
1854
|
+
logSynthesizedGesturePolicyDecision(
|
|
1855
|
+
kind: policyKind,
|
|
1856
|
+
context: context,
|
|
1857
|
+
fallbackAttempted: true
|
|
1858
|
+
)
|
|
1859
|
+
} else {
|
|
1860
|
+
logSynthesizedGesturePolicyDecision(
|
|
1861
|
+
kind: policyKind,
|
|
1862
|
+
context: context,
|
|
1863
|
+
fallbackAttempted: false
|
|
1864
|
+
)
|
|
1865
|
+
resolvedCoordinateContext = context
|
|
1866
|
+
resolvedCoordinateTarget = TextEntryTarget(
|
|
1867
|
+
element: nil,
|
|
1868
|
+
refreshPoint: CGPoint(x: x, y: y),
|
|
1869
|
+
prefersFocusedElement: false
|
|
1870
|
+
)
|
|
1871
|
+
}
|
|
1872
|
+
}
|
|
1873
|
+
#else
|
|
1874
|
+
let xCTestChannelPenalized = false
|
|
1875
|
+
let resolvedCoordinateTarget: TextEntryTarget? = nil
|
|
1876
|
+
#endif
|
|
1877
|
+
if let resolvedCoordinateTarget {
|
|
1878
|
+
target = resolvedCoordinateTarget
|
|
1879
|
+
} else if let selectorKey = command.selectorKey, let selectorValue = command.selectorValue {
|
|
1880
|
+
// Released daemons may still send selector-keyed type commands even though current
|
|
1881
|
+
// daemons resolve fill selectors through the runtime tree before reaching the runner.
|
|
1802
1882
|
let match = findElement(
|
|
1803
1883
|
app: activeApp,
|
|
1804
1884
|
selectorKey: selectorKey,
|
|
@@ -1829,7 +1909,16 @@ extension RunnerTests {
|
|
|
1829
1909
|
textEntryModeName(textEntryMode)
|
|
1830
1910
|
)
|
|
1831
1911
|
if textEntryMode == .replacement {
|
|
1832
|
-
|
|
1912
|
+
#if os(iOS)
|
|
1913
|
+
let canReplaceResolvedFirstResponder = Self.shouldUseSynthesizedFirstResponderReplacement(
|
|
1914
|
+
hasResolvedElement: target.element != nil,
|
|
1915
|
+
hasRefreshPoint: target.refreshPoint != nil,
|
|
1916
|
+
xCTestChannelPenalized: xCTestChannelPenalized
|
|
1917
|
+
)
|
|
1918
|
+
#else
|
|
1919
|
+
let canReplaceResolvedFirstResponder = false
|
|
1920
|
+
#endif
|
|
1921
|
+
guard target.element != nil || canReplaceResolvedFirstResponder else {
|
|
1833
1922
|
let message =
|
|
1834
1923
|
(command.x != nil && command.y != nil)
|
|
1835
1924
|
? "no text input found at the provided coordinates to clear"
|
|
@@ -1843,8 +1932,16 @@ extension RunnerTests {
|
|
|
1843
1932
|
text: text,
|
|
1844
1933
|
delaySeconds: delaySeconds,
|
|
1845
1934
|
repairMode: textEntryMode,
|
|
1935
|
+
xCTestChannelPenalized: xCTestChannelPenalized,
|
|
1936
|
+
synthesizer: PrivateXCTestTextEntrySynthesizer(),
|
|
1846
1937
|
commandId: command.commandId
|
|
1847
1938
|
)
|
|
1939
|
+
if let failure = textResult.failure {
|
|
1940
|
+
return Response(
|
|
1941
|
+
ok: false,
|
|
1942
|
+
error: ErrorPayload(code: failure.rawValue, message: failure.message, hint: failure.hint)
|
|
1943
|
+
)
|
|
1944
|
+
}
|
|
1848
1945
|
if textResult.verified == false {
|
|
1849
1946
|
let expected = textResult.expectedText ?? ""
|
|
1850
1947
|
let observed = textResult.observedText ?? ""
|
|
@@ -1857,7 +1954,16 @@ extension RunnerTests {
|
|
|
1857
1954
|
)
|
|
1858
1955
|
}
|
|
1859
1956
|
let point = target.refreshPoint
|
|
1860
|
-
let frame
|
|
1957
|
+
let frame: CGRect
|
|
1958
|
+
if let resolvedCoordinateContext {
|
|
1959
|
+
frame = resolvedCoordinateContext.referenceFrame
|
|
1960
|
+
} else if point != nil {
|
|
1961
|
+
frame = activeApp.frame
|
|
1962
|
+
} else {
|
|
1963
|
+
// Bare `type` has no coordinate response to normalize. Avoid serializing the
|
|
1964
|
+
// application AX tree only to emit unused reference dimensions.
|
|
1965
|
+
frame = .zero
|
|
1966
|
+
}
|
|
1861
1967
|
return Response(
|
|
1862
1968
|
ok: true,
|
|
1863
1969
|
data: DataPayload(
|
|
@@ -1866,7 +1972,8 @@ extension RunnerTests {
|
|
|
1866
1972
|
y: point.map { Double($0.y) },
|
|
1867
1973
|
referenceWidth: frame.isEmpty ? nil : Double(frame.width),
|
|
1868
1974
|
referenceHeight: frame.isEmpty ? nil : Double(frame.height),
|
|
1869
|
-
maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed
|
|
1975
|
+
maestroNonHittableCoordinateFallbackUsed: maestroNonHittableCoordinateFallbackUsed,
|
|
1976
|
+
textEntryRoute: textResult.textEntryRoute
|
|
1870
1977
|
)
|
|
1871
1978
|
)
|
|
1872
1979
|
}
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift
CHANGED
|
@@ -21,23 +21,34 @@ private enum KeyboardDismissObservationTiming {
|
|
|
21
21
|
static let settleRequiredConsecutiveMatches: Int = 3
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
// The mechanism that actually resigned the keyboard, disclosed to the caller
|
|
25
|
+
// (#1598) so a response never claims "dismissed" without saying how — a
|
|
26
|
+
// safe-area tap has a very different reliability/side-effect profile than
|
|
27
|
+
// tapping the keyboard's own Done key, and callers need to know which one
|
|
28
|
+
// fired.
|
|
29
|
+
enum RunnerKeyboardDismissMechanism: String {
|
|
30
|
+
case dismissKey
|
|
31
|
+
}
|
|
32
|
+
|
|
24
33
|
extension RunnerTests {
|
|
25
34
|
func isKeyboardVisible(app: XCUIApplication) -> Bool {
|
|
26
35
|
return visibleKeyboardFrame(app: app) != nil
|
|
27
36
|
}
|
|
28
37
|
|
|
29
|
-
func dismissKeyboard(
|
|
38
|
+
func dismissKeyboard(
|
|
39
|
+
app: XCUIApplication
|
|
40
|
+
) -> (wasVisible: Bool, dismissed: Bool, visible: Bool, mechanism: RunnerKeyboardDismissMechanism?) {
|
|
30
41
|
let keyboard = app.keyboards.firstMatch
|
|
31
42
|
let wasVisible = isKeyboardVisible(app: app)
|
|
32
43
|
guard wasVisible else {
|
|
33
|
-
return (wasVisible: false, dismissed: false, visible: false)
|
|
44
|
+
return (wasVisible: false, dismissed: false, visible: false, mechanism: nil)
|
|
34
45
|
}
|
|
35
46
|
|
|
36
47
|
#if os(tvOS)
|
|
37
48
|
_ = pressTvRemote(.menu)
|
|
38
49
|
sleepFor(0.2)
|
|
39
50
|
let visible = isKeyboardVisible(app: app)
|
|
40
|
-
return (wasVisible: true, dismissed: !visible, visible: visible)
|
|
51
|
+
return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey)
|
|
41
52
|
#else
|
|
42
53
|
if tapKeyboardDismissControl(app: app) {
|
|
43
54
|
_ = keyboard.waitForNonExistence(timeout: KeyboardDismissObservationTiming.timeout)
|
|
@@ -47,13 +58,20 @@ extension RunnerTests {
|
|
|
47
58
|
requiredConsecutiveMatches: KeyboardDismissObservationTiming.settleRequiredConsecutiveMatches
|
|
48
59
|
)
|
|
49
60
|
let visible = isKeyboardVisible(app: app)
|
|
50
|
-
return (wasVisible: true, dismissed: !visible, visible: visible)
|
|
61
|
+
return (wasVisible: true, dismissed: !visible, visible: visible, mechanism: visible ? nil : .dismissKey)
|
|
51
62
|
}
|
|
52
63
|
|
|
53
|
-
|
|
64
|
+
// #1606 review P1 (twice): generic background-tap dismissal is
|
|
65
|
+
// deliberately UNSUPPORTED. No geometry or role query can prove a
|
|
66
|
+
// coordinate is side-effect-free — a full-screen unlabeled Pressable is
|
|
67
|
+
// indistinguishable from an inert backdrop, so a "safe-area" tap can
|
|
68
|
+
// navigate or submit while reporting a successful dismiss. The dismiss
|
|
69
|
+
// key is the only mechanism the runner can vouch for.
|
|
70
|
+
return (wasVisible: true, dismissed: false, visible: isKeyboardVisible(app: app), mechanism: nil)
|
|
54
71
|
#endif
|
|
55
72
|
}
|
|
56
73
|
|
|
74
|
+
|
|
57
75
|
// AX-free on purpose (screenshot bytes, not the accessibility tree) so it holds
|
|
58
76
|
// under the same AX degradation the synthesized gesture lane is built to survive.
|
|
59
77
|
// Bounded and self-terminating: returns as soon as `requiredConsecutiveMatches`
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift
CHANGED
|
@@ -86,6 +86,7 @@ extension RunnerTests {
|
|
|
86
86
|
currentApp = app
|
|
87
87
|
currentBundleId = nil
|
|
88
88
|
currentAppProcessIdentifier = nil
|
|
89
|
+
clearRememberedTextEntryTap()
|
|
89
90
|
snapshotXCTestPenaltyWarmupExemptionPending = false
|
|
90
91
|
}
|
|
91
92
|
|
|
@@ -96,11 +97,17 @@ extension RunnerTests {
|
|
|
96
97
|
currentApp = nil
|
|
97
98
|
currentBundleId = nil
|
|
98
99
|
currentAppProcessIdentifier = nil
|
|
100
|
+
clearRememberedTextEntryTap()
|
|
99
101
|
snapshotXCTestPenaltyWarmupExemptionPending = false
|
|
100
102
|
}
|
|
101
103
|
|
|
102
104
|
func resetTargetAfterExternalRelaunch() -> Response {
|
|
103
105
|
invalidateCachedTarget(reason: "external_app_relaunch")
|
|
106
|
+
// The app process is replaced, but the retained runner survives. Clear
|
|
107
|
+
// process-bound capture state explicitly because invalidation drops the
|
|
108
|
+
// old PID before refreshCachedTargetIfProcessChanged can observe it.
|
|
109
|
+
clearSnapshotXCTestChannelPenalty(reason: "external_app_relaunch")
|
|
110
|
+
clearPrivateAXAcceptedDepth(reason: "external_app_relaunch")
|
|
104
111
|
needsFirstInteractionDelay = true
|
|
105
112
|
return Response(ok: true, data: DataPayload(message: "target reset"))
|
|
106
113
|
}
|
|
@@ -121,7 +128,9 @@ extension RunnerTests {
|
|
|
121
128
|
)
|
|
122
129
|
currentApp = candidate
|
|
123
130
|
currentAppProcessIdentifier = observedProcessIdentifier
|
|
131
|
+
clearRememberedTextEntryTap()
|
|
124
132
|
clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
|
|
133
|
+
clearPrivateAXAcceptedDepth(reason: "target_process_changed")
|
|
125
134
|
snapshotXCTestPenaltyWarmupExemptionPending = true
|
|
126
135
|
needsFirstInteractionDelay = true
|
|
127
136
|
}
|
|
@@ -194,6 +203,7 @@ extension RunnerTests {
|
|
|
194
203
|
currentApp = target
|
|
195
204
|
currentBundleId = bundleId
|
|
196
205
|
currentAppProcessIdentifier = Self.processIdentifier(of: target)
|
|
206
|
+
clearRememberedTextEntryTap()
|
|
197
207
|
snapshotXCTestPenaltyWarmupExemptionPending = false
|
|
198
208
|
needsFirstInteractionDelay = true
|
|
199
209
|
return target
|
|
@@ -3,71 +3,4 @@ import XCTest
|
|
|
3
3
|
|
|
4
4
|
extension RunnerTests {
|
|
5
5
|
|
|
6
|
-
func testCachedTargetRefreshRequiresChangedPositiveProcessIdentity() {
|
|
7
|
-
XCTAssertFalse(
|
|
8
|
-
Self.shouldRefreshCachedTarget(
|
|
9
|
-
cachedProcessIdentifier: nil,
|
|
10
|
-
observedProcessIdentifier: 42
|
|
11
|
-
)
|
|
12
|
-
)
|
|
13
|
-
XCTAssertFalse(
|
|
14
|
-
Self.shouldRefreshCachedTarget(
|
|
15
|
-
cachedProcessIdentifier: 42,
|
|
16
|
-
observedProcessIdentifier: 42
|
|
17
|
-
)
|
|
18
|
-
)
|
|
19
|
-
XCTAssertTrue(
|
|
20
|
-
Self.shouldRefreshCachedTarget(
|
|
21
|
-
cachedProcessIdentifier: 41,
|
|
22
|
-
observedProcessIdentifier: 42
|
|
23
|
-
)
|
|
24
|
-
)
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
func testSnapshotPenaltyWarmupExemptionIsConsumedOnce() {
|
|
28
|
-
snapshotXCTestPenaltyWarmupExemptionPending = true
|
|
29
|
-
|
|
30
|
-
XCTAssertTrue(consumeSnapshotXCTestPenaltyWarmupExemption())
|
|
31
|
-
XCTAssertFalse(consumeSnapshotXCTestPenaltyWarmupExemption())
|
|
32
|
-
}
|
|
33
|
-
|
|
34
|
-
func testSnapshotPenaltyCanBeClearedAcrossTargetProcessReplacement() {
|
|
35
|
-
penalizeSnapshotXCTestChannel(bundleId: "com.example.app", reason: "test")
|
|
36
|
-
XCTAssertTrue(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
|
|
37
|
-
|
|
38
|
-
clearSnapshotXCTestChannelPenalty(reason: "target_process_changed")
|
|
39
|
-
|
|
40
|
-
XCTAssertFalse(isSnapshotXCTestChannelPenalized(bundleId: "com.example.app"))
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
func testCachedTargetInvalidationClearsProcessBoundState() {
|
|
44
|
-
currentApp = app
|
|
45
|
-
currentBundleId = "com.example.app"
|
|
46
|
-
currentAppProcessIdentifier = 42
|
|
47
|
-
snapshotXCTestPenaltyWarmupExemptionPending = true
|
|
48
|
-
|
|
49
|
-
invalidateCachedTarget(reason: "unit_test")
|
|
50
|
-
|
|
51
|
-
XCTAssertNil(currentApp)
|
|
52
|
-
XCTAssertNil(currentBundleId)
|
|
53
|
-
XCTAssertNil(currentAppProcessIdentifier)
|
|
54
|
-
XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
func testTargetResetInvalidatesProcessBoundStateWithoutRestartingRunner() {
|
|
58
|
-
currentApp = app
|
|
59
|
-
currentBundleId = "com.example.app"
|
|
60
|
-
currentAppProcessIdentifier = 42
|
|
61
|
-
snapshotXCTestPenaltyWarmupExemptionPending = true
|
|
62
|
-
needsFirstInteractionDelay = false
|
|
63
|
-
|
|
64
|
-
let response = resetTargetAfterExternalRelaunch()
|
|
65
|
-
|
|
66
|
-
XCTAssertTrue(response.ok)
|
|
67
|
-
XCTAssertNil(currentApp)
|
|
68
|
-
XCTAssertNil(currentBundleId)
|
|
69
|
-
XCTAssertNil(currentAppProcessIdentifier)
|
|
70
|
-
XCTAssertFalse(snapshotXCTestPenaltyWarmupExemptionPending)
|
|
71
|
-
XCTAssertTrue(needsFirstInteractionDelay)
|
|
72
|
-
}
|
|
73
6
|
}
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift
CHANGED
|
@@ -136,8 +136,8 @@ struct Command: Codable {
|
|
|
136
136
|
let gesturePlan: RunnerGesturePlan?
|
|
137
137
|
let outPath: String?
|
|
138
138
|
let fps: Int?
|
|
139
|
-
let maxSize: Int?
|
|
140
139
|
let interactiveOnly: Bool?
|
|
140
|
+
let preferredBackend: String?
|
|
141
141
|
let depth: Int?
|
|
142
142
|
let scope: String?
|
|
143
143
|
let raw: Bool?
|
|
@@ -257,11 +257,13 @@ struct DataPayload: Codable {
|
|
|
257
257
|
let visible: Bool?
|
|
258
258
|
let wasVisible: Bool?
|
|
259
259
|
let dismissed: Bool?
|
|
260
|
+
let keyboardDismissMechanism: String?
|
|
260
261
|
let orientation: String?
|
|
261
262
|
let gestureFallback: String?
|
|
262
263
|
let gestureFallbackMessage: String?
|
|
263
264
|
let gestureFallbackHint: String?
|
|
264
265
|
let maestroNonHittableCoordinateFallbackUsed: Bool?
|
|
266
|
+
let textEntryRoute: String?
|
|
265
267
|
let runnerFatal: Bool?
|
|
266
268
|
let runnerFatalReason: String?
|
|
267
269
|
let completedSteps: Int?
|
|
@@ -297,11 +299,13 @@ struct DataPayload: Codable {
|
|
|
297
299
|
visible: Bool? = nil,
|
|
298
300
|
wasVisible: Bool? = nil,
|
|
299
301
|
dismissed: Bool? = nil,
|
|
302
|
+
keyboardDismissMechanism: String? = nil,
|
|
300
303
|
orientation: String? = nil,
|
|
301
304
|
gestureFallback: String? = nil,
|
|
302
305
|
gestureFallbackMessage: String? = nil,
|
|
303
306
|
gestureFallbackHint: String? = nil,
|
|
304
307
|
maestroNonHittableCoordinateFallbackUsed: Bool? = nil,
|
|
308
|
+
textEntryRoute: String? = nil,
|
|
305
309
|
runnerFatal: Bool? = nil,
|
|
306
310
|
runnerFatalReason: String? = nil,
|
|
307
311
|
completedSteps: Int? = nil,
|
|
@@ -336,11 +340,13 @@ struct DataPayload: Codable {
|
|
|
336
340
|
self.visible = visible
|
|
337
341
|
self.wasVisible = wasVisible
|
|
338
342
|
self.dismissed = dismissed
|
|
343
|
+
self.keyboardDismissMechanism = keyboardDismissMechanism
|
|
339
344
|
self.orientation = orientation
|
|
340
345
|
self.gestureFallback = gestureFallback
|
|
341
346
|
self.gestureFallbackMessage = gestureFallbackMessage
|
|
342
347
|
self.gestureFallbackHint = gestureFallbackHint
|
|
343
348
|
self.maestroNonHittableCoordinateFallbackUsed = maestroNonHittableCoordinateFallbackUsed
|
|
349
|
+
self.textEntryRoute = textEntryRoute
|
|
344
350
|
self.runnerFatal = runnerFatal
|
|
345
351
|
self.runnerFatalReason = runnerFatalReason
|
|
346
352
|
self.completedSteps = completedSteps
|
|
@@ -390,4 +396,8 @@ struct SnapshotOptions {
|
|
|
390
396
|
let depth: Int?
|
|
391
397
|
let scope: String?
|
|
392
398
|
let raw: Bool
|
|
399
|
+
/// Internal daemon ask: capture with this backend first regardless of channel
|
|
400
|
+
/// health ("private-ax"). Same-backend evidence probes (tap-outcome
|
|
401
|
+
/// corroboration) must be captured the way their baseline was.
|
|
402
|
+
var preferredBackend: String? = nil
|
|
393
403
|
}
|
|
@@ -7,7 +7,6 @@ extension RunnerTests {
|
|
|
7
7
|
final class ScreenRecorder {
|
|
8
8
|
private let outputPath: String
|
|
9
9
|
private let fps: Int32?
|
|
10
|
-
private let maxSize: Int?
|
|
11
10
|
private var effectiveFps: Int32 {
|
|
12
11
|
max(1, fps ?? RunnerTests.defaultRecordingFps)
|
|
13
12
|
}
|
|
@@ -26,10 +25,9 @@ extension RunnerTests {
|
|
|
26
25
|
private var startedSession = false
|
|
27
26
|
private var startError: Error?
|
|
28
27
|
|
|
29
|
-
init(outputPath: String, fps: Int32
|
|
28
|
+
init(outputPath: String, fps: Int32?) {
|
|
30
29
|
self.outputPath = outputPath
|
|
31
30
|
self.fps = fps
|
|
32
|
-
self.maxSize = maxSize
|
|
33
31
|
}
|
|
34
32
|
|
|
35
33
|
func start(captureFrame: @escaping () -> RunnerImage?) throws {
|
|
@@ -50,7 +48,7 @@ extension RunnerTests {
|
|
|
50
48
|
while Date() < bootstrapDeadline {
|
|
51
49
|
if let image = captureFrame(), let cgImage = runnerCGImage(from: image) {
|
|
52
50
|
bootstrapImage = image
|
|
53
|
-
dimensions =
|
|
51
|
+
dimensions = CGSize(width: cgImage.width, height: cgImage.height)
|
|
54
52
|
break
|
|
55
53
|
}
|
|
56
54
|
Thread.sleep(forTimeInterval: 0.05)
|
|
@@ -261,23 +259,5 @@ extension RunnerTests {
|
|
|
261
259
|
return pixelBuffer
|
|
262
260
|
}
|
|
263
261
|
|
|
264
|
-
private func scaledDimensions(width: Int, height: Int) -> CGSize {
|
|
265
|
-
guard let maxSize, maxSize > 0 else {
|
|
266
|
-
return CGSize(width: width, height: height)
|
|
267
|
-
}
|
|
268
|
-
let longest = max(width, height)
|
|
269
|
-
guard longest > maxSize else {
|
|
270
|
-
return CGSize(width: width, height: height)
|
|
271
|
-
}
|
|
272
|
-
let scale = Double(maxSize) / Double(longest)
|
|
273
|
-
return CGSize(
|
|
274
|
-
width: scaledEvenDimension(width, scale: scale),
|
|
275
|
-
height: scaledEvenDimension(height, scale: scale)
|
|
276
|
-
)
|
|
277
|
-
}
|
|
278
|
-
|
|
279
|
-
private func scaledEvenDimension(_ value: Int, scale: Double) -> Int {
|
|
280
|
-
max(2, Int((Double(value) * scale / 2.0).rounded()) * 2)
|
|
281
|
-
}
|
|
282
262
|
}
|
|
283
263
|
}
|