agent-device 0.21.30 → 0.21.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.30.apk → agent-device-android-ime-helper-0.21.31.apk} +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.31.apk.sha256 +1 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.30.manifest.json → agent-device-android-ime-helper-0.21.31.manifest.json} +4 -4
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.31.apk +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.31.apk.sha256 +1 -0
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.30.manifest.json → agent-device-android-snapshot-helper-0.21.31.manifest.json} +7 -7
- package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/BackgroundInteraction.swift +20 -6
- package/apple/snapshot-bridge/README.md +16 -3
- package/apple/snapshot-bridge/SnapshotBridgeRuntime.m +33 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +110 -8
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +20 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +4 -4
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +1 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +30 -4
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +14 -4
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +81 -41
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +40 -6
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +5 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +74 -10
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +33 -4
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +2 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +23 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +158 -6
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +165 -70
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryConfirmation.swift +2 -3
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +49 -27
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryReadiness.swift +17 -9
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputProbe.swift +30 -28
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +87 -87
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +4 -3
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +13 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +45 -0
- package/dist/src/absence-observation-errors.js +1 -1
- package/dist/src/adapter.js +1 -1
- package/dist/src/adb-executor.js +1 -1
- package/dist/src/adb-failure.js +2 -2
- package/dist/src/agent-browser-lifecycle.js +1 -1
- package/dist/src/agent-browser-process-record.js +1 -1
- package/dist/src/agent-browser-provider.js +1 -1
- package/dist/src/agent-browser-tool.js +1 -1
- package/dist/src/agent-device-client.js +1 -1
- package/dist/src/alert-tree-driver.js +1 -1
- package/dist/src/alert.js +1 -1
- package/dist/src/alert2.js +1 -1
- package/dist/src/android.js +1 -1
- package/dist/src/app-backend.js +1 -1
- package/dist/src/app-launch.js +1 -1
- package/dist/src/app-lifecycle.js +4 -4
- package/dist/src/app-log-runtime2.js +3 -3
- package/dist/src/app-log-session-resource.js +1 -1
- package/dist/src/app-resolution-facade.js +1 -1
- package/dist/src/app-resolution.js +2 -2
- package/dist/src/apple-runner-platform.js +1 -1
- package/dist/src/application-lifecycle-interaction.js +1 -1
- package/dist/src/artifact-tracking.js +1 -1
- package/dist/src/automation-helper-artifact.js +1 -1
- package/dist/src/automation-helper.js +1 -1
- package/dist/src/back-mode.js +1 -1
- package/dist/src/boot-diagnostics.js +1 -1
- package/dist/src/browser-launch.js +1 -1
- package/dist/src/cli-config.js +1 -1
- package/dist/src/cli-help.js +35 -29
- package/dist/src/cli-runner.js +1 -1
- package/dist/src/cli.js +3 -3
- package/dist/src/client-metro-companion.js +1 -1
- package/dist/src/client-metro.js +1 -1
- package/dist/src/client-types.d.ts +3 -3
- package/dist/src/cloud-artifacts.d.ts +5 -4
- package/dist/src/code-signature-cache.js +1 -1
- package/dist/src/command-metadata.js +10 -10
- package/dist/src/command-tools.js +2 -2
- package/dist/src/command.js +1 -1
- package/dist/src/composed-inventory-gateways.js +1 -1
- package/dist/src/connection-runtime.js +1 -1
- package/dist/src/connection.js +5 -5
- package/dist/src/contracts.js +1 -1
- package/dist/src/daemon-client-ensure.js +1 -1
- package/dist/src/daemon-client-lease-beat.js +1 -1
- package/dist/src/daemon-client-lifecycle.js +1 -1
- package/dist/src/daemon-client-liveness-probe.js +1 -1
- package/dist/src/daemon-client.js +1 -1
- package/dist/src/daemon-command-registry.js +1 -1
- package/dist/src/daemon-composition.js +1 -1
- package/dist/src/daemon-policy-file.js +1 -1
- package/dist/src/daemon-registration-owner.js +3 -3
- package/dist/src/daemon-resolution.js +1 -1
- package/dist/src/daemon-startup-diagnostics.js +1 -1
- package/dist/src/debug-symbols-facade.js +3 -3
- package/dist/src/device-claim-recovery.js +1 -1
- package/dist/src/device-claims.js +1 -1
- package/dist/src/device-inventory-context.js +1 -1
- package/dist/src/device-readiness-runtime.js +1 -1
- package/dist/src/device-selection-resolver.js +1 -1
- package/dist/src/device.js +1 -1
- package/dist/src/devicectl.js +1 -1
- package/dist/src/diagnostics.js +1 -1
- package/dist/src/display-inventory.js +1 -1
- package/dist/src/doctor-facade.js +1 -1
- package/dist/src/durable-capture-resource.js +1 -1
- package/dist/src/durable-resource.js +1 -1
- package/dist/src/engine-eval-script.js +1 -1
- package/dist/src/error.js +1 -1
- package/dist/src/explicit-session-resolution.js +1 -1
- package/dist/src/expo-dev-client-launch.js +1 -1
- package/dist/src/file.js +1 -1
- package/dist/src/find.js +1 -1
- package/dist/src/finders.js +1 -1
- package/dist/src/gesture-admission.js +1 -1
- package/dist/src/gesture-plan.js +1 -1
- package/dist/src/gesture-runtime.js +1 -1
- package/dist/src/hdc.js +1 -1
- package/dist/src/helper.js +2 -2
- package/dist/src/host.js +1 -1
- package/dist/src/host2.js +1 -1
- package/dist/src/host3.js +1 -1
- package/dist/src/host4.js +1 -1
- package/dist/src/host5.js +1 -0
- package/dist/src/human-control.js +1 -1
- package/dist/src/ime-rebind.js +1 -1
- package/dist/src/ime-state.js +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/input-actions.js +1 -1
- package/dist/src/input-actions2.js +1 -1
- package/dist/src/install-source-network-transport.js +1 -1
- package/dist/src/install-source2.js +1 -1
- package/dist/src/interaction-positionals.js +1 -1
- package/dist/src/interaction.js +1 -1
- package/dist/src/interaction2.js +1 -1
- package/dist/src/interactor-operation-catalog.js +1 -1
- package/dist/src/interactor.js +5 -5
- package/dist/src/interactor2.js +1 -1
- package/dist/src/interactor3.js +1 -1
- package/dist/src/interactor4.js +1 -1
- package/dist/src/interactor5.js +1 -1
- package/dist/src/interactors.js +1 -1
- package/dist/src/internal/bin.js +1 -1
- package/dist/src/internal/daemon.js +1 -1
- package/dist/src/internal/png-worker.js +1 -1
- package/dist/src/inventory.js +1 -1
- package/dist/src/inventory2.js +1 -1
- package/dist/src/inventory6.js +1 -1
- package/dist/src/ios-node-interpretation.js +1 -1
- package/dist/src/ios-settings.js +1 -1
- package/dist/src/ios-snapshot-acquisition.js +1 -1
- package/dist/src/ios-snapshot-engine.js +1 -1
- package/dist/src/ios-snapshot-runtime.js +1 -1
- package/dist/src/keyboard-occlusion.js +1 -1
- package/dist/src/keyed-lock.js +1 -1
- package/dist/src/launch-confirmation.js +1 -1
- package/dist/src/launch-services-registration.js +4 -4
- package/dist/src/lease.js +1 -1
- package/dist/src/legacy.js +1 -1
- package/dist/src/limrun-runtime-dependencies.js +1 -1
- package/dist/src/local-interactor-operation-set.js +1 -1
- package/dist/src/mechanics.js +12 -12
- package/dist/src/mobile-snapshot-semantics.js +1 -1
- package/dist/src/observability.js +1 -1
- package/dist/src/output-path.js +1 -1
- package/dist/src/parameterized-recorded-fill.js +1 -1
- package/dist/src/perf-facade.js +1 -1
- package/dist/src/perf-runtime-operation-builder.js +1 -1
- package/dist/src/perf-target.js +1 -1
- package/dist/src/physical-device-control.js +1 -1
- package/dist/src/platform-runtime-android-adb-host.js +1 -1
- package/dist/src/platform-runtime-claim-recovery.js +1 -1
- package/dist/src/platform-runtime-daemon-services.js +1 -1
- package/dist/src/platform-runtime-host.js +1 -1
- package/dist/src/platform-runtime-open-target.js +1 -1
- package/dist/src/platform-runtime-operation-host.js +5 -5
- package/dist/src/platform-runtime-runtime-hints.js +2 -2
- package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -1
- package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
- package/dist/src/platform-runtime-unavailable.js +1 -1
- package/dist/src/platform-runtime.js +1 -1
- package/dist/src/platform-runtime2.js +1 -1
- package/dist/src/png-resize.js +1 -1
- package/dist/src/png-worker-client.js +1 -1
- package/dist/src/process-exit.js +1 -1
- package/dist/src/provider-device-runtime.d.ts +28 -0
- package/dist/src/provider-plugin-catalog.js +2 -0
- package/dist/src/provider-plugin.d.ts +1 -1
- package/dist/src/provider-webdriver.js +1 -1
- package/dist/src/proxy.js +1 -1
- package/dist/src/react-native-overlay2.js +1 -1
- package/dist/src/react-native.js +1 -1
- package/dist/src/react-profile-command.js +1 -1
- package/dist/src/record-runtime.js +1 -1
- package/dist/src/record-trace.js +1 -1
- package/dist/src/recording-scope.js +1 -1
- package/dist/src/recording.js +1 -1
- package/dist/src/registry.js +1 -1
- package/dist/src/remote-config-core.js +1 -1
- package/dist/src/replay-backend-host.js +1 -1
- package/dist/src/replay-divergence.js +1 -1
- package/dist/src/replay-maestro-backend.js +3 -3
- package/dist/src/replay-script-selection.js +1 -1
- package/dist/src/replay2.js +1 -1
- package/dist/src/request-runtime-binding.js +1 -1
- package/dist/src/request.js +1 -1
- package/dist/src/result-serialization.js +1 -1
- package/dist/src/retry.js +1 -1
- package/dist/src/runner-cache-retention.js +1 -1
- package/dist/src/runner-client.js +1 -1
- package/dist/src/runner-dev-tools-security.js +1 -1
- package/dist/src/runner-device-readiness.js +1 -1
- package/dist/src/runner-device-set.js +1 -1
- package/dist/src/runner-error-classification.js +2 -2
- package/dist/src/runner-exchange.js +1 -1
- package/dist/src/runner-lease.js +1 -1
- package/dist/src/runner-operations-facade.js +1 -1
- package/dist/src/runner-retention.js +1 -1
- package/dist/src/runner-selector-query.js +1 -1
- package/dist/src/runner-start-budget.js +1 -1
- package/dist/src/runner-startup-transport.js +1 -1
- package/dist/src/runner-toolchain-probe.js +1 -1
- package/dist/src/runner.js +2 -2
- package/dist/src/runtime-admission.js +1 -1
- package/dist/src/runtime-deployment.js +1 -1
- package/dist/src/runtime-factory.js +1 -1
- package/dist/src/runtime-session.js +1 -1
- package/dist/src/runtime.js +4 -4
- package/dist/src/runtime10.js +2 -2
- package/dist/src/runtime2.js +1 -1
- package/dist/src/runtime3.js +5 -5
- package/dist/src/runtime4.js +1 -1
- package/dist/src/runtime5.js +1 -1
- package/dist/src/runtime6.js +1 -1
- package/dist/src/runtime7.js +1 -1
- package/dist/src/runtime8.js +1 -1
- package/dist/src/runtime9.js +2 -2
- package/dist/src/screen-recording-transport.js +1 -1
- package/dist/src/screenshot-density.js +1 -1
- package/dist/src/screenshot-runtime.js +1 -1
- package/dist/src/screenshot3.js +1 -1
- package/dist/src/sdk-batch-runner.js +2 -2
- package/dist/src/sdk-batch.js +2 -2
- package/dist/src/sdk-contracts.d.ts +9 -0
- package/dist/src/sdk-contracts.js +2 -2
- package/dist/src/sdk-device.js +1 -1
- package/dist/src/sdk-finders.js +1 -1
- package/dist/src/sdk-io.js +5 -5
- package/dist/src/sdk-remote-config.js +1 -1
- package/dist/src/sdk-selectors.js +1 -1
- package/dist/src/selection.js +0 -0
- package/dist/src/selector-pipeline-policy.js +1 -1
- package/dist/src/selector-runtime.js +1 -1
- package/dist/src/selectors.js +1 -1
- package/dist/src/server-startup.js +1 -1
- package/dist/src/session-paths.js +1 -1
- package/dist/src/session-replay-vars.js +1 -1
- package/dist/src/session-runtime.js +1 -1
- package/dist/src/session-store.js +3 -3
- package/dist/src/session.js +1 -1
- package/dist/src/session2.js +5 -5
- package/dist/src/settings-bluetooth.js +1 -1
- package/dist/src/settings.js +1 -1
- package/dist/src/settings2.js +1 -1
- package/dist/src/settle-observation-capture.js +1 -1
- package/dist/src/simctl-settings.js +1 -1
- package/dist/src/simctl.js +1 -1
- package/dist/src/simulator-boot.js +1 -1
- package/dist/src/simulator-state.js +1 -1
- package/dist/src/simulator.js +1 -1
- package/dist/src/snapshot-chrome.js +1 -1
- package/dist/src/snapshot-desktop-projection.js +1 -1
- package/dist/src/snapshot-helper-install.js +6 -6
- package/dist/src/snapshot-node-lookup.js +1 -1
- package/dist/src/snapshot-occlusion.js +1 -1
- package/dist/src/snapshot-runtime-binding.js +1 -1
- package/dist/src/snapshot-runtime-capture-input.js +1 -1
- package/dist/src/snapshot-runtime.js +1 -1
- package/dist/src/snapshot-state.js +1 -1
- package/dist/src/snapshot.js +1 -1
- package/dist/src/snapshot3.js +1 -1
- package/dist/src/src.js +1 -1
- package/dist/src/src3.js +1 -1
- package/dist/src/src4.js +1 -1
- package/dist/src/store.js +1 -1
- package/dist/src/tap-keyboard-occlusion.js +1 -1
- package/dist/src/target-evidence-tree.js +1 -1
- package/dist/src/target-evidence.js +1 -1
- package/dist/src/telephony-runtime.js +1 -1
- package/dist/src/text-entry-focus.js +1 -1
- package/dist/src/tool-provider.js +3 -3
- package/dist/src/tool-provider2.js +1 -1
- package/dist/src/tool-provider3.js +1 -1
- package/dist/src/toolchain-probe.js +1 -1
- package/dist/src/touch-reference-frame.js +1 -1
- package/dist/src/touch-runtime.js +1 -1
- package/dist/src/touch-runtime2.js +1 -1
- package/dist/src/tree.js +1 -1
- package/dist/src/tv-remote-runtime.js +1 -1
- package/dist/src/tv-remote.js +2 -2
- package/dist/src/type-text-runtime.js +1 -1
- package/dist/src/update-check.js +1 -1
- package/dist/src/vega-executable.js +1 -1
- package/dist/src/video.js +1 -1
- package/dist/src/webdriver-source.js +1 -1
- package/package.json +7 -6
- package/server.json +2 -2
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.30.apk.sha256 +0 -1
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.30.apk +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.30.apk.sha256 +0 -1
- package/dist/src/physical-device-apps.js +0 -1
|
@@ -165,6 +165,63 @@ extension RunnerTests {
|
|
|
165
165
|
#endif
|
|
166
166
|
}
|
|
167
167
|
|
|
168
|
+
#if os(iOS)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
@MainActor
|
|
172
|
+
func synthesizedLongPressAt(
|
|
173
|
+
app: XCUIApplication,
|
|
174
|
+
x: Double,
|
|
175
|
+
y: Double,
|
|
176
|
+
durationMs: Double,
|
|
177
|
+
context: SynthesizedCoordinateContext?
|
|
178
|
+
) -> RunnerInteractionOutcome {
|
|
179
|
+
guard x.isFinite, y.isFinite, durationMs.isFinite, durationMs > 0 else {
|
|
180
|
+
return .unsupported(
|
|
181
|
+
message: "synthesized coordinate long press requires finite coordinates and a positive duration",
|
|
182
|
+
hint: "Retry with finite x and y values and a positive durationMs."
|
|
183
|
+
)
|
|
184
|
+
}
|
|
185
|
+
guard let context else {
|
|
186
|
+
return .unsupported(
|
|
187
|
+
message: "synthesized coordinate long press could not resolve an app window with a finite screen frame",
|
|
188
|
+
hint: "Retry after the app is foregrounded, or use a plain screenshot to choose coordinates."
|
|
189
|
+
)
|
|
190
|
+
}
|
|
191
|
+
let orientation = Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
|
|
192
|
+
let point = CoordinateSpaceRotation.native(
|
|
193
|
+
point: CGPoint(x: x, y: y),
|
|
194
|
+
in: context.referenceFrame,
|
|
195
|
+
interfaceOrientation: orientation
|
|
196
|
+
)
|
|
197
|
+
logSynthesizedDispatch(
|
|
198
|
+
kind: "longPress",
|
|
199
|
+
start: point,
|
|
200
|
+
end: nil,
|
|
201
|
+
referenceFrame: context.referenceFrame,
|
|
202
|
+
orientation: orientation
|
|
203
|
+
)
|
|
204
|
+
let samples: [[String: NSNumber]] = [0, durationMs].map { offsetMs in
|
|
205
|
+
[
|
|
206
|
+
"x": NSNumber(value: Double(point.x)),
|
|
207
|
+
"y": NSNumber(value: Double(point.y)),
|
|
208
|
+
"offsetMs": NSNumber(value: offsetMs),
|
|
209
|
+
]
|
|
210
|
+
}
|
|
211
|
+
if let message = RunnerSynthesizedGesture.synthesizeGesture(
|
|
212
|
+
withApplication: app,
|
|
213
|
+
resolvedWindow: context.resolvedWindow,
|
|
214
|
+
pointerSamples: [samples]
|
|
215
|
+
) {
|
|
216
|
+
return .unsupported(
|
|
217
|
+
message: message,
|
|
218
|
+
hint: "Private XCTest event synthesis is required for an AX-free long press on iOS; update Xcode if this persists."
|
|
219
|
+
)
|
|
220
|
+
}
|
|
221
|
+
return .performed
|
|
222
|
+
}
|
|
223
|
+
#endif
|
|
224
|
+
|
|
168
225
|
func keyboardAvoidingDragPoints(
|
|
169
226
|
app: XCUIApplication,
|
|
170
227
|
x: Double,
|
|
@@ -239,6 +296,21 @@ extension RunnerTests {
|
|
|
239
296
|
)
|
|
240
297
|
}
|
|
241
298
|
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
func touchVisualizationFrame(
|
|
302
|
+
x: Double,
|
|
303
|
+
y: Double,
|
|
304
|
+
context: SynthesizedCoordinateContext
|
|
305
|
+
) -> TouchVisualizationFrame {
|
|
306
|
+
TouchVisualizationFrame(
|
|
307
|
+
x: Double(context.referenceFrame.minX) + x,
|
|
308
|
+
y: Double(context.referenceFrame.minY) + y,
|
|
309
|
+
referenceWidth: Double(context.referenceFrame.width),
|
|
310
|
+
referenceHeight: Double(context.referenceFrame.height)
|
|
311
|
+
)
|
|
312
|
+
}
|
|
313
|
+
|
|
242
314
|
func resolvedDragVisualizationFrame(
|
|
243
315
|
app: XCUIApplication,
|
|
244
316
|
x: Double,
|
|
@@ -269,6 +341,27 @@ extension RunnerTests {
|
|
|
269
341
|
return CGRect(x: 0, y: 0, width: 0, height: 0)
|
|
270
342
|
}
|
|
271
343
|
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
@MainActor
|
|
348
|
+
func gestureViewportFrame(app: XCUIApplication) -> CGRect {
|
|
349
|
+
#if os(iOS)
|
|
350
|
+
guard let context = synthesizedCoordinateContext(
|
|
351
|
+
app: app,
|
|
352
|
+
policy: synthesizedGesturePolicy(.synthesizedDrag)
|
|
353
|
+
) else {
|
|
354
|
+
return frameAvoidingKeyboard(app: app, frame: app.frame)
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
return context.resolvedWindow == nil
|
|
358
|
+
? context.referenceFrame
|
|
359
|
+
: frameAvoidingKeyboard(app: app, frame: context.referenceFrame)
|
|
360
|
+
#else
|
|
361
|
+
return resolvedTouchReferenceFrame(app: app, appFrame: app.frame)
|
|
362
|
+
#endif
|
|
363
|
+
}
|
|
364
|
+
|
|
272
365
|
private func frameAvoidingKeyboard(app: XCUIApplication, frame: CGRect) -> CGRect {
|
|
273
366
|
#if os(iOS)
|
|
274
367
|
guard let keyboardFrame = visibleKeyboardFrame(app: app), !frame.isEmpty else {
|
|
@@ -384,15 +477,28 @@ extension RunnerTests {
|
|
|
384
477
|
@MainActor
|
|
385
478
|
func synthesizedCoordinateContext(
|
|
386
479
|
app: XCUIApplication,
|
|
387
|
-
policy: SynthesizedGesturePolicy
|
|
480
|
+
policy: SynthesizedGesturePolicy,
|
|
481
|
+
windowQueryRejected: Bool = false
|
|
388
482
|
) -> SynthesizedCoordinateContext? {
|
|
389
483
|
#if os(iOS)
|
|
390
|
-
|
|
391
|
-
let
|
|
392
|
-
|
|
484
|
+
var health = mainOwned.accessibilityHealth
|
|
485
|
+
let window: XCUIElement?
|
|
486
|
+
let referenceFrame: CGRect
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
let rejectedTreeSoleScreenFrame = windowQueryRejected ? soleScreenReferenceFrame(app: app) : nil
|
|
490
|
+
if rejectedTreeSoleScreenFrame == nil, let resolved = resolveRunnerWindowElement(app: app) {
|
|
491
|
+
window = resolved.window
|
|
492
|
+
referenceFrame = resolved.frame
|
|
493
|
+
} else if let soleScreenFrame = rejectedTreeSoleScreenFrame ?? soleScreenReferenceFrame(app: app) {
|
|
494
|
+
window = nil
|
|
495
|
+
referenceFrame = soleScreenFrame
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
health = .unavailable
|
|
499
|
+
} else {
|
|
393
500
|
return nil
|
|
394
501
|
}
|
|
395
|
-
let referenceFrame = resolved.frame
|
|
396
502
|
guard referenceFrame.width.isFinite, referenceFrame.height.isFinite,
|
|
397
503
|
referenceFrame.width > 0, referenceFrame.height > 0
|
|
398
504
|
else {
|
|
@@ -410,6 +516,52 @@ extension RunnerTests {
|
|
|
410
516
|
}
|
|
411
517
|
|
|
412
518
|
|
|
519
|
+
#if os(iOS)
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
|
|
525
|
+
@MainActor
|
|
526
|
+
func soleScreenReferenceFrame(app: XCUIApplication) -> CGRect? {
|
|
527
|
+
guard UIDevice.current.userInterfaceIdiom == .phone, XCUIScreen.screens.count == 1 else {
|
|
528
|
+
return nil
|
|
529
|
+
}
|
|
530
|
+
let screen = UIScreen.main
|
|
531
|
+
let scale = screen.nativeScale
|
|
532
|
+
guard scale > 0 else { return nil }
|
|
533
|
+
let frame = Self.soleScreenReferenceFrame(
|
|
534
|
+
nativePortraitSize: CGSize(
|
|
535
|
+
width: screen.nativeBounds.width / scale,
|
|
536
|
+
height: screen.nativeBounds.height / scale
|
|
537
|
+
),
|
|
538
|
+
interfaceOrientation: Int(RunnerSynthesizedGesture.interfaceOrientation(forApplication: app))
|
|
539
|
+
)
|
|
540
|
+
NSLog(
|
|
541
|
+
"AGENT_DEVICE_RUNNER_SYNTHESIZED_SOLE_SCREEN_ROUTE resolved=%@",
|
|
542
|
+
frame.map { NSCoder.string(for: $0) } ?? "none"
|
|
543
|
+
)
|
|
544
|
+
return frame
|
|
545
|
+
}
|
|
546
|
+
#endif
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
static func soleScreenReferenceFrame(
|
|
551
|
+
nativePortraitSize size: CGSize,
|
|
552
|
+
interfaceOrientation: Int
|
|
553
|
+
) -> CGRect? {
|
|
554
|
+
guard size.width > 0, size.height > 0 else { return nil }
|
|
555
|
+
switch interfaceOrientation {
|
|
556
|
+
case RunnerInterfaceOrientation.portrait, RunnerInterfaceOrientation.portraitUpsideDown:
|
|
557
|
+
return CGRect(origin: .zero, size: size)
|
|
558
|
+
case RunnerInterfaceOrientation.landscapeLeft, RunnerInterfaceOrientation.landscapeRight:
|
|
559
|
+
return CGRect(x: 0, y: 0, width: size.height, height: size.width)
|
|
560
|
+
default:
|
|
561
|
+
return nil
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
|
|
413
565
|
func keyboardAvoidingSynthesizedDragPoints(
|
|
414
566
|
app: XCUIApplication,
|
|
415
567
|
x: Double,
|
|
@@ -594,7 +746,7 @@ extension RunnerTests {
|
|
|
594
746
|
]
|
|
595
747
|
}
|
|
596
748
|
}
|
|
597
|
-
let resolvedWindow =
|
|
749
|
+
let resolvedWindow = resolveRunnerWindowElement(app: app)?.window
|
|
598
750
|
if let message = RunnerSynthesizedGesture.synthesizeGesture(
|
|
599
751
|
withApplication: app,
|
|
600
752
|
resolvedWindow: resolvedWindow,
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift
CHANGED
|
@@ -159,13 +159,17 @@ extension RunnerTests {
|
|
|
159
159
|
|
|
160
160
|
let inputAtRefreshPoint: TextInputAtPoint?
|
|
161
161
|
|
|
162
|
+
|
|
163
|
+
let postFocusRead: ResolvedTextEntryInput?
|
|
164
|
+
|
|
162
165
|
init(
|
|
163
166
|
element: XCUIElement?,
|
|
164
167
|
refreshPoint: CGPoint?,
|
|
165
168
|
prefersFocusedElement: Bool,
|
|
166
169
|
fromTapWitness: Bool = false,
|
|
167
170
|
boundIdentity: TextEntryInputIdentity? = nil,
|
|
168
|
-
inputAtRefreshPoint: TextInputAtPoint? = nil
|
|
171
|
+
inputAtRefreshPoint: TextInputAtPoint? = nil,
|
|
172
|
+
postFocusRead: ResolvedTextEntryInput? = nil
|
|
169
173
|
) {
|
|
170
174
|
self.element = element
|
|
171
175
|
self.refreshPoint = refreshPoint
|
|
@@ -173,32 +177,113 @@ extension RunnerTests {
|
|
|
173
177
|
self.fromTapWitness = fromTapWitness
|
|
174
178
|
self.boundIdentity = boundIdentity ?? inputAtRefreshPoint?.identity
|
|
175
179
|
self.inputAtRefreshPoint = inputAtRefreshPoint
|
|
180
|
+
self.postFocusRead = postFocusRead
|
|
176
181
|
}
|
|
177
182
|
|
|
178
|
-
func withElement(_ nextElement: XCUIElement?) -> TextEntryTarget {
|
|
179
|
-
guard let nextElement else {
|
|
180
|
-
return self
|
|
181
|
-
}
|
|
182
|
-
|
|
183
183
|
|
|
184
184
|
|
|
185
|
-
|
|
186
|
-
let
|
|
187
|
-
|
|
185
|
+
func withResolvedInput(_ input: ResolvedTextEntryInput?) -> TextEntryTarget {
|
|
186
|
+
guard let input else {
|
|
187
|
+
return self
|
|
188
|
+
}
|
|
189
|
+
let frame = input.snapshot?.frame ?? .zero
|
|
188
190
|
return TextEntryTarget(
|
|
189
|
-
element:
|
|
190
|
-
refreshPoint:
|
|
191
|
+
element: input.element,
|
|
192
|
+
refreshPoint: frame.isEmpty ? refreshPoint : CGPoint(x: frame.midX, y: frame.midY),
|
|
191
193
|
prefersFocusedElement: prefersFocusedElement,
|
|
192
194
|
fromTapWitness: fromTapWitness,
|
|
193
|
-
boundIdentity: boundIdentity ??
|
|
194
|
-
TextEntryInputIdentity(elementType: $0.elementType, identifier: $0.identifier)
|
|
195
|
-
},
|
|
195
|
+
boundIdentity: boundIdentity ?? input.identity,
|
|
196
196
|
inputAtRefreshPoint: inputAtRefreshPoint
|
|
197
197
|
)
|
|
198
198
|
}
|
|
199
199
|
}
|
|
200
200
|
|
|
201
201
|
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
struct ResolvedTextEntryInput {
|
|
208
|
+
let element: XCUIElement
|
|
209
|
+
|
|
210
|
+
let snapshot: XCUIElementSnapshot?
|
|
211
|
+
|
|
212
|
+
private init(element: XCUIElement, snapshot: XCUIElementSnapshot?) {
|
|
213
|
+
self.element = element
|
|
214
|
+
self.snapshot = snapshot
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
init?(reading element: XCUIElement) {
|
|
221
|
+
var exists = false
|
|
222
|
+
var snapshot: XCUIElementSnapshot?
|
|
223
|
+
if let exception = RunnerObjCExceptionCatcher.catchException({
|
|
224
|
+
exists = element.exists
|
|
225
|
+
snapshot = exists ? try? element.snapshot() : nil
|
|
226
|
+
}) {
|
|
227
|
+
NSLog("AGENT_DEVICE_RUNNER_TEXT_ENTRY_READ_IGNORED_EXCEPTION=%@", exception)
|
|
228
|
+
return nil
|
|
229
|
+
}
|
|
230
|
+
guard exists else { return nil }
|
|
231
|
+
self.init(element: element, snapshot: snapshot)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
init?(readingListed element: XCUIElement) {
|
|
237
|
+
let snapshot = try? element.snapshot()
|
|
238
|
+
guard snapshot != nil || element.exists else { return nil }
|
|
239
|
+
self.init(element: element, snapshot: snapshot)
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
var identity: TextEntryInputIdentity? {
|
|
243
|
+
snapshot.map { TextEntryInputIdentity(elementType: $0.elementType, identifier: $0.identifier) }
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
var attributes: any XCUIElementAttributes {
|
|
247
|
+
if let snapshot {
|
|
248
|
+
return snapshot
|
|
249
|
+
}
|
|
250
|
+
return element
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
struct TextEntryReader {
|
|
259
|
+
private(set) var target: TextEntryTarget
|
|
260
|
+
private var lastRead: ResolvedTextEntryInput?
|
|
261
|
+
private let resolve: (TextEntryTarget) -> ResolvedTextEntryInput?
|
|
262
|
+
|
|
263
|
+
init(target: TextEntryTarget, resolve: @escaping (TextEntryTarget) -> ResolvedTextEntryInput?) {
|
|
264
|
+
self.target = target.withResolvedInput(target.postFocusRead)
|
|
265
|
+
lastRead = target.postFocusRead
|
|
266
|
+
self.resolve = resolve
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
mutating func input() -> ResolvedTextEntryInput? {
|
|
270
|
+
if let lastRead {
|
|
271
|
+
return lastRead
|
|
272
|
+
}
|
|
273
|
+
let read = resolve(target)
|
|
274
|
+
target = target.withResolvedInput(read)
|
|
275
|
+
lastRead = read
|
|
276
|
+
return read
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
mutating func dispatch<T>(_ dispatch: () -> T) -> T {
|
|
281
|
+
lastRead = nil
|
|
282
|
+
return dispatch()
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
|
|
202
287
|
enum TextEntryInputProbe: Equatable {
|
|
203
288
|
case input(TextEntryInputIdentity)
|
|
204
289
|
|
|
@@ -226,9 +311,38 @@ extension RunnerTests {
|
|
|
226
311
|
boundIdentity?.isDistinguishable ?? true
|
|
227
312
|
}
|
|
228
313
|
|
|
314
|
+
|
|
315
|
+
enum TextEntryFocusEvidence {
|
|
316
|
+
|
|
317
|
+
case focusEngine
|
|
318
|
+
|
|
319
|
+
case focusedInput
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
case inputReportsKeyboardFocus(ResolvedTextEntryInput)
|
|
323
|
+
|
|
324
|
+
case keyboardAppeared
|
|
325
|
+
|
|
326
|
+
case focusQueryMatchesInput
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
var name: String {
|
|
330
|
+
switch self {
|
|
331
|
+
case .focusEngine: return "focus-engine"
|
|
332
|
+
case .focusedInput: return "focused-input"
|
|
333
|
+
case .inputReportsKeyboardFocus: return "input-reports-keyboard-focus"
|
|
334
|
+
case .keyboardAppeared: return "keyboard-appeared"
|
|
335
|
+
case .focusQueryMatchesInput: return "focus-query-matches-input"
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
229
340
|
struct TextEntryStabilization {
|
|
230
341
|
let element: XCUIElement?
|
|
231
|
-
|
|
342
|
+
|
|
343
|
+
let evidence: TextEntryFocusEvidence?
|
|
344
|
+
|
|
345
|
+
var focusConfirmed: Bool { evidence != nil }
|
|
232
346
|
}
|
|
233
347
|
|
|
234
348
|
struct TextEntryTapWitness {
|
|
@@ -241,7 +355,8 @@ extension RunnerTests {
|
|
|
241
355
|
}
|
|
242
356
|
}
|
|
243
357
|
|
|
244
|
-
|
|
358
|
+
|
|
359
|
+
func clearTextInput(app: XCUIApplication, _ input: ResolvedTextEntryInput) {
|
|
245
360
|
|
|
246
361
|
|
|
247
362
|
|
|
@@ -250,16 +365,17 @@ extension RunnerTests {
|
|
|
250
365
|
|
|
251
366
|
|
|
252
367
|
|
|
253
|
-
if let existing = editableTextValue(for:
|
|
368
|
+
if let existing = editableTextValue(for: input.attributes, treatingPlaceholderAsEmpty: true),
|
|
254
369
|
existing.isEmpty {
|
|
255
370
|
return
|
|
256
371
|
}
|
|
372
|
+
|
|
373
|
+
let count = estimatedDeleteCount(for: input.attributes)
|
|
257
374
|
#if !os(tvOS)
|
|
258
|
-
moveCaretToEnd(app: app,
|
|
375
|
+
moveCaretToEnd(app: app, input: input)
|
|
259
376
|
#endif
|
|
260
|
-
let count = estimatedDeleteCount(for: element)
|
|
261
377
|
let deletes = String(repeating: XCUIKeyboardKey.delete.rawValue, count: count)
|
|
262
|
-
element.typeText(deletes)
|
|
378
|
+
input.element.typeText(deletes)
|
|
263
379
|
}
|
|
264
380
|
|
|
265
381
|
func isTextEntryElement(_ element: XCUIElement) -> Bool {
|
|
@@ -285,33 +401,35 @@ extension RunnerTests {
|
|
|
285
401
|
|
|
286
402
|
|
|
287
403
|
|
|
288
|
-
func
|
|
404
|
+
func resolveTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? {
|
|
289
405
|
guard let boundIdentity = target.boundIdentity else {
|
|
290
|
-
return
|
|
406
|
+
return resolveUnboundTextEntryInput(app: app, target: target)
|
|
291
407
|
}
|
|
292
|
-
func
|
|
293
|
-
|
|
408
|
+
func boundInput(_ candidate: XCUIElement) -> ResolvedTextEntryInput? {
|
|
409
|
+
guard let input = ResolvedTextEntryInput(reading: candidate), input.identity == boundIdentity else { return nil }
|
|
410
|
+
return input
|
|
294
411
|
}
|
|
295
|
-
if target.prefersFocusedElement, let focused = focusedTextInput(app: app),
|
|
296
|
-
return
|
|
412
|
+
if target.prefersFocusedElement, let focused = focusedTextInput(app: app), let input = boundInput(focused) {
|
|
413
|
+
return input
|
|
297
414
|
}
|
|
298
|
-
if let element = target.element,
|
|
299
|
-
return
|
|
415
|
+
if let element = target.element, let input = boundInput(element) {
|
|
416
|
+
return input
|
|
300
417
|
}
|
|
301
|
-
if let input = target.inputAtRefreshPoint,
|
|
302
|
-
return
|
|
418
|
+
if let input = target.inputAtRefreshPoint, let read = textInputIfStillResolves(input) {
|
|
419
|
+
return read
|
|
303
420
|
}
|
|
304
421
|
if target.inputAtRefreshPoint == nil,
|
|
305
422
|
let refreshPoint = target.refreshPoint,
|
|
306
423
|
case .matches(let candidates) = probeTextInputs(app: app, point: refreshPoint),
|
|
307
|
-
let match = candidates.first(where:
|
|
424
|
+
let match = candidates.first(where: { $0.identity == boundIdentity }) {
|
|
308
425
|
return match
|
|
309
426
|
}
|
|
310
|
-
if let focused = focusedTextInput(app: app),
|
|
311
|
-
return
|
|
427
|
+
if let focused = focusedTextInput(app: app), let input = boundInput(focused) {
|
|
428
|
+
return input
|
|
312
429
|
}
|
|
313
|
-
if let byIdentifier = boundTextEntryInputQuery(app: app, identity: boundIdentity),
|
|
314
|
-
|
|
430
|
+
if let byIdentifier = boundTextEntryInputQuery(app: app, identity: boundIdentity),
|
|
431
|
+
let input = boundInput(byIdentifier) {
|
|
432
|
+
return input
|
|
315
433
|
}
|
|
316
434
|
return nil
|
|
317
435
|
}
|
|
@@ -341,7 +459,7 @@ extension RunnerTests {
|
|
|
341
459
|
case .absent:
|
|
342
460
|
return true
|
|
343
461
|
case .matches(let candidates):
|
|
344
|
-
return candidates.allSatisfy {
|
|
462
|
+
return candidates.allSatisfy { $0.identity != boundIdentity }
|
|
345
463
|
case .unavailable:
|
|
346
464
|
return false
|
|
347
465
|
}
|
|
@@ -356,21 +474,6 @@ extension RunnerTests {
|
|
|
356
474
|
}
|
|
357
475
|
|
|
358
476
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
func textEntrySnapshotFrame(_ element: XCUIElement) -> CGRect? {
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
safely("TEXT_ENTRY_SNAPSHOT_FRAME", { element.exists ? try? element.snapshot() : nil })?.frame
|
|
371
|
-
}
|
|
372
|
-
|
|
373
|
-
|
|
374
477
|
func probeTextEntryInput(_ element: XCUIElement) -> TextEntryInputProbe {
|
|
375
478
|
var probe = TextEntryInputProbe.unavailable
|
|
376
479
|
let (_, exception) = catchingObjCException(fallback: ()) {
|
|
@@ -385,30 +488,21 @@ extension RunnerTests {
|
|
|
385
488
|
}
|
|
386
489
|
|
|
387
490
|
|
|
388
|
-
private func
|
|
389
|
-
if target.prefersFocusedElement {
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
return element
|
|
395
|
-
}
|
|
396
|
-
} else {
|
|
397
|
-
if let element = target.element, element.exists {
|
|
398
|
-
return element
|
|
399
|
-
}
|
|
491
|
+
private func resolveUnboundTextEntryInput(app: XCUIApplication, target: TextEntryTarget) -> ResolvedTextEntryInput? {
|
|
492
|
+
if target.prefersFocusedElement, let focused = focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:)) {
|
|
493
|
+
return focused
|
|
494
|
+
}
|
|
495
|
+
if let element = target.element, let input = ResolvedTextEntryInput(reading: element) {
|
|
496
|
+
return input
|
|
400
497
|
}
|
|
401
498
|
if let refreshPoint = target.refreshPoint,
|
|
402
499
|
let refreshed = textInputAt(app: app, x: refreshPoint.x, y: refreshPoint.y) {
|
|
403
500
|
return refreshed
|
|
404
501
|
}
|
|
405
|
-
|
|
406
|
-
return focused
|
|
407
|
-
}
|
|
408
|
-
return nil
|
|
502
|
+
return focusedTextInput(app: app).flatMap(ResolvedTextEntryInput.init(reading:))
|
|
409
503
|
}
|
|
410
504
|
|
|
411
|
-
private func moveCaretToEnd(app: XCUIApplication,
|
|
505
|
+
private func moveCaretToEnd(app: XCUIApplication, input: ResolvedTextEntryInput) {
|
|
412
506
|
#if os(tvOS)
|
|
413
507
|
return
|
|
414
508
|
#else
|
|
@@ -423,7 +517,8 @@ extension RunnerTests {
|
|
|
423
517
|
|
|
424
518
|
|
|
425
519
|
|
|
426
|
-
|
|
520
|
+
|
|
521
|
+
guard let frame = input.snapshot?.frame, !frame.isEmpty else {
|
|
427
522
|
return
|
|
428
523
|
}
|
|
429
524
|
_ = tapAt(
|
|
@@ -434,7 +529,7 @@ extension RunnerTests {
|
|
|
434
529
|
#endif
|
|
435
530
|
}
|
|
436
531
|
|
|
437
|
-
private func estimatedDeleteCount(for element:
|
|
532
|
+
private func estimatedDeleteCount(for element: any XCUIElementAttributes) -> Int {
|
|
438
533
|
let valueText = normalizedElementText(element.value)
|
|
439
534
|
let base = valueText.isEmpty ? 24 : (valueText.count + 8)
|
|
440
535
|
return max(24, min(120, base))
|
|
@@ -125,9 +125,8 @@ extension RunnerTests {
|
|
|
125
125
|
|
|
126
126
|
|
|
127
127
|
|
|
128
|
-
func textEntryObservation(for
|
|
129
|
-
guard let
|
|
130
|
-
let snapshot = try? element.snapshot(),
|
|
128
|
+
func textEntryObservation(for input: ResolvedTextEntryInput?) -> TextEntryObservation? {
|
|
129
|
+
guard let snapshot = input?.snapshot,
|
|
131
130
|
let value = editableTextValue(for: snapshot, treatingPlaceholderAsEmpty: true)
|
|
132
131
|
else {
|
|
133
132
|
return nil
|