agent-device 0.21.12 → 0.21.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.12.apk → agent-device-android-ime-helper-0.21.14.apk} +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.14.apk.sha256 +1 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.12.manifest.json → agent-device-android-ime-helper-0.21.14.manifest.json} +4 -4
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.14.apk +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.14.apk.sha256 +1 -0
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.12.manifest.json → agent-device-android-snapshot-helper-0.21.14.manifest.json} +6 -6
- package/apple/macos-helper/Package.swift +4 -0
- package/apple/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +3 -6
- package/apple/macos-helper/Tests/AgentDeviceMacOSHelperTests/ScreenshotResponseTests.swift +15 -0
- package/apple/snapshot-bridge/SnapshotBridgeRuntime.m +47 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +122 -42
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/Info.plist +23 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner.xcodeproj/project.pbxproj +17 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAppScreenCapture.swift +21 -13
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +1 -162
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +0 -9
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +0 -56
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSystemSurfaceHostPolicy.swift +0 -48
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTapPointPolicy.swift +0 -50
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +33 -586
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Alert.swift +40 -24
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AlertObservation.swift +2 -3
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+BlockingSystemModalResolution.swift +0 -36
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandDispatch.swift +630 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +34 -2256
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandJournal.swift +1 -338
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Environment.swift +0 -12
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+GestureExecution.swift +160 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +0 -24
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +0 -142
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +90 -39
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+MainThreadWork.swift +78 -106
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +23 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +162 -113
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +0 -209
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +86 -10
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +271 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollGesture.swift +1 -200
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SequenceExecution.swift +20 -227
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Snapshot.swift +78 -295
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotAcquisition.swift +40 -27
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotBackendCapabilities.swift +0 -78
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +85 -581
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCaptureTarget.swift +106 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotExecution.swift +142 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTiming.swift +49 -78
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedCommitDeadline.swift +0 -44
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +73 -114
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +1 -12
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +11 -94
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SystemModal.swift +2 -10
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +1 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryFocus.swift +0 -35
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextInputCandidatePolicy.swift +0 -31
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +2 -18
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Transport.swift +34 -52
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TvRemote.swift +0 -22
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +135 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +22 -6
- package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +140 -20
- package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotCoordinateSpace.swift +16 -19
- package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotGeometry.swift +21 -9
- package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotModels.swift +59 -10
- package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotPresentationInvariant.swift +12 -10
- package/dist/apple/snapshot-presentation/Sources/AgentDeviceSnapshotPresentation/SnapshotVisibilityFold.swift +13 -8
- package/dist/src/absence-observation-errors.js +1 -1
- package/dist/src/adapter.js +1 -1
- package/dist/src/adb-executor.js +1 -1
- package/dist/src/adb-failure.js +1 -1
- package/dist/src/adb-transport.js +1 -1
- package/dist/src/agent-browser-provider.js +1 -1
- package/dist/src/agent-browser-tool.js +1 -1
- package/dist/src/agent-device-client.js +2 -2
- package/dist/src/ai-sdk.js +1 -1
- package/dist/src/alert-runtime.js +1 -1
- package/dist/src/android-adb.d.ts +1 -1
- package/dist/src/app-catalog.js +1 -1
- package/dist/src/app-deployment-resolution.js +1 -1
- package/dist/src/app-inventory.js +1 -1
- package/dist/src/app-launch.js +2 -2
- package/dist/src/app-lifecycle.js +1 -1
- package/dist/src/app-lifecycle2.js +1 -1
- package/dist/src/app-log-files.js +1 -1
- package/dist/src/app-log-runtime2.js +3 -3
- package/dist/src/app-resolution.js +2 -2
- package/dist/src/apple-runner-platform.js +1 -1
- package/dist/src/application-lifecycle-interaction.js +1 -1
- package/dist/src/archive.js +1 -1
- package/dist/src/artifact-download.js +1 -1
- package/dist/src/auth-session.js +1 -1
- package/dist/src/auth.js +1 -1
- package/dist/src/back-mode.js +1 -1
- package/dist/src/backend-snapshot-options.js +1 -1
- package/dist/src/boot-diagnostics.js +1 -1
- package/dist/src/capture-disclosure.js +1 -1
- package/dist/src/capture.js +1 -1
- package/dist/src/cli-config.js +1 -1
- package/dist/src/cli-help.js +8 -7
- package/dist/src/cli-runner.js +1 -1
- package/dist/src/cli.js +4 -4
- package/dist/src/click-button.js +1 -1
- package/dist/src/client-metro.js +1 -1
- package/dist/src/client-types.d.ts +4 -5
- package/dist/src/clipboard-shell-response.js +1 -1
- package/dist/src/code-signature.js +1 -1
- package/dist/src/command-schema.js +3 -3
- package/dist/src/command-tools.js +3 -3
- package/dist/src/connection-runtime.js +1 -1
- package/dist/src/connection.js +1 -1
- package/dist/src/contracts.js +1 -1
- package/dist/src/daemon-client-lifecycle.js +1 -1
- package/dist/src/daemon-resolution.js +1 -1
- package/dist/src/daemon.js +1 -1
- package/dist/src/debug-symbols-facade.js +1 -1
- package/dist/src/device-claim-rule.js +2 -2
- package/dist/src/device-input-state.d.ts +1 -1
- package/dist/src/device-inventory-context.js +1 -1
- package/dist/src/device-rotation.js +1 -1
- package/dist/src/device-selection-resolver.js +1 -1
- package/dist/src/device-session.js +1 -1
- package/dist/src/device-shell.js +1 -1
- package/dist/src/device2.js +1 -1
- package/dist/src/devicectl.js +1 -1
- package/dist/src/diagnostics.js +1 -1
- package/dist/src/dispatch-resolve.js +1 -1
- package/dist/src/display-inventory.js +1 -1
- package/dist/src/doctor-facade.js +1 -1
- package/dist/src/doctor2.js +1 -1
- package/dist/src/engine-eval-script.js +1 -1
- package/dist/src/error.js +1 -1
- package/dist/src/exec.js +1 -1
- package/dist/src/file.js +1 -1
- package/dist/src/find.js +1 -1
- package/dist/src/format.js +1 -1
- package/dist/src/gesture-plan-types.d.ts +38 -0
- package/dist/src/gesture-plan.js +1 -1
- package/dist/src/gesture-runtime.js +1 -1
- package/dist/src/harmonyos.js +1 -1
- package/dist/src/helper.js +2 -2
- package/dist/src/host.js +1 -1
- package/dist/src/human-control-contract.js +1 -1
- package/dist/src/human-control.js +1 -1
- package/dist/src/index.js +1 -1
- package/dist/src/input-actions.js +1 -1
- package/dist/src/input-actions2.js +1 -1
- package/dist/src/install-artifact-facade.js +1 -1
- package/dist/src/install-source-network-transport.js +1 -1
- package/dist/src/install-source2.js +1 -1
- package/dist/src/interaction-positionals.js +1 -1
- package/dist/src/interaction-ref-policy.js +1 -1
- package/dist/src/interaction2.js +1 -1
- package/dist/src/interactor-operation-binding.js +1 -1
- package/dist/src/interactor-operation-catalog.js +1 -1
- package/dist/src/interactor.js +4 -4
- package/dist/src/interactor2.js +1 -1
- package/dist/src/interactors.js +1 -1
- package/dist/src/internal/bin.js +1 -1
- package/dist/src/internal/daemon.js +2 -2
- package/dist/src/internal/png-worker.js +1 -1
- package/dist/src/internal/run-script-http-child.js +1 -1
- package/dist/src/inventory.js +1 -1
- package/dist/src/inventory2.js +1 -1
- package/dist/src/inventory3.js +1 -1
- package/dist/src/inventory5.js +1 -1
- package/dist/src/ios-snapshot-acquisition.js +1 -1
- package/dist/src/ios-snapshot-runtime.js +1 -1
- package/dist/src/lease.js +1 -1
- package/dist/src/limrun-download-file.js +1 -1
- package/dist/src/limrun-runtime-dependencies.js +1 -1
- package/dist/src/limrun.d.ts +435 -4
- package/dist/src/linux.js +1 -1
- package/dist/src/local-interactor-operation-set.js +1 -1
- package/dist/src/location-coordinates.js +1 -1
- package/dist/src/managed-device-scope.js +1 -1
- package/dist/src/mechanics.js +5 -5
- package/dist/src/mobile-snapshot-semantics.js +1 -1
- package/dist/src/multitouch-support.js +1 -1
- package/dist/src/native-build-cache.js +1 -0
- package/dist/src/observation.js +1 -1
- package/dist/src/parameterized-recorded-fill.js +1 -1
- package/dist/src/perf-facade.js +1 -2
- package/dist/src/perf-runtime-operation-builder.js +1 -1
- package/dist/src/perf-runtime-plan.js +1 -1
- package/dist/src/perf-target.js +2 -0
- package/dist/src/perf.js +1 -1
- package/dist/src/physical-device-apps.js +1 -0
- package/dist/src/physical-device-control.js +1 -1
- package/dist/src/platform-request-scope.js +1 -1
- package/dist/src/platform-runtime-host.js +1 -1
- package/dist/src/platform-runtime-managed-owner.js +1 -1
- package/dist/src/platform-runtime-open-target.js +1 -1
- package/dist/src/platform-runtime-operation-host.js +2 -2
- package/dist/src/platform-runtime-runtime-hints.js +1 -1
- package/dist/src/platform-runtime-screen-recording-apple-simulator-host.js +1 -1
- package/dist/src/platform-runtime-screen-recording-finalizer-host.js +1 -1
- package/dist/src/platform-runtime-screen-recording-process-host.js +1 -1
- package/dist/src/platform-runtime-unavailable.js +1 -1
- package/dist/src/platform-runtime.js +1 -1
- package/dist/src/platform-runtime2.js +1 -1
- package/dist/src/png-crop-bytes.js +1 -1
- package/dist/src/png-size.js +1 -1
- package/dist/src/png-worker-client.js +1 -1
- package/dist/src/png.js +1 -1
- package/dist/src/prepare-kind.js +1 -1
- package/dist/src/provider-session-ownership.js +1 -1
- package/dist/src/provider-webdriver.js +1 -1
- package/dist/src/proxy.js +1 -1
- package/dist/src/quality-warnings.js +1 -1
- package/dist/src/react-native.js +1 -1
- package/dist/src/record-runtime.js +1 -1
- package/dist/src/recording.js +1 -1
- package/dist/src/register-builtins.js +1 -1
- package/dist/src/registry.js +35 -35
- package/dist/src/remote-config2.js +1 -1
- package/dist/src/replay-divergence.js +1 -1
- package/dist/src/replay.js +1 -1
- package/dist/src/reporting.js +1 -1
- package/dist/src/request-admission.js +1 -1
- package/dist/src/request-cancel.js +1 -1
- package/dist/src/request-runtime-binding.js +1 -1
- package/dist/src/retry.js +1 -1
- package/dist/src/runner-cache-metadata.js +1 -1
- package/dist/src/runner-client.js +1 -1
- package/dist/src/runner-dev-tools-security.js +1 -1
- package/dist/src/runner-device-readiness.js +1 -1
- package/dist/src/runner-disposal.js +1 -1
- package/dist/src/runner-error-classification.js +3 -0
- package/dist/src/runner-operations-facade.js +1 -1
- package/dist/src/runner-presentation.js +1 -1
- package/dist/src/runner.js +1 -1
- package/dist/src/runtime-transport-hints.js +1 -1
- package/dist/src/runtime.js +3 -3
- package/dist/src/runtime10.js +2 -2
- package/dist/src/runtime2.js +2 -2
- package/dist/src/runtime3.js +1 -1
- package/dist/src/runtime4.js +4 -4
- package/dist/src/runtime5.js +1 -1
- package/dist/src/runtime6.js +1 -1
- package/dist/src/runtime7.js +1 -1
- package/dist/src/runtime8.js +1 -1
- package/dist/src/runtime9.js +1 -1
- package/dist/src/screen-recording-admission-ledger.js +1 -1
- package/dist/src/screen-recording-completion.js +1 -1
- package/dist/src/screen-recording-options.js +1 -1
- package/dist/src/screen-recording-resource-recovery.js +1 -1
- package/dist/src/screen-recording-transport.js +1 -1
- package/dist/src/screenshot-density.js +1 -1
- package/dist/src/screenshot-image.js +1 -1
- package/dist/src/screenshot-runtime.js +1 -1
- package/dist/src/screenshot.js +1 -1
- package/dist/src/screenshot3.js +1 -1
- package/dist/src/script.js +1 -1
- package/dist/src/scroll-command.d.ts +377 -0
- package/dist/src/scroll-command.js +1 -1
- package/dist/src/scroll-gesture.js +1 -1
- package/dist/src/sdk-batch-runner.js +2 -2
- package/dist/src/sdk-batch.js +2 -2
- package/dist/src/sdk-contracts.d.ts +28 -0
- package/dist/src/sdk-contracts.js +1 -1
- package/dist/src/sdk-device.js +1 -1
- package/dist/src/sdk-io.js +1 -1
- package/dist/src/sdk-remote-config.js +1 -1
- package/dist/src/sdk-selectors.d.ts +23 -726
- package/dist/src/selection.js +0 -0
- package/dist/src/selector-runtime.js +1 -1
- package/dist/src/semantic-index.js +1 -1
- package/dist/src/server.js +1 -1
- package/dist/src/session-allocation.js +1 -1
- package/dist/src/session-observation.js +1 -1
- package/dist/src/session-routing.js +1 -1
- package/dist/src/session-runtime-admission.js +1 -1
- package/dist/src/session-snapshot.js +1 -1
- package/dist/src/session-store.js +2 -2
- package/dist/src/session2.js +3 -3
- package/dist/src/settings.js +1 -1
- package/dist/src/simctl-facade.js +1 -1
- package/dist/src/simctl.js +1 -1
- package/dist/src/simulator-state.js +1 -1
- package/dist/src/simulator.js +1 -1
- package/dist/src/snapshot-runtime-binding.js +1 -1
- package/dist/src/snapshot-runtime.js +1 -1
- package/dist/src/snapshot-state.js +1 -1
- package/dist/src/snapshot2.js +1 -1
- package/dist/src/snapshot3.js +1 -1
- package/dist/src/snapshot4.js +1 -1
- package/dist/src/src.js +1 -1
- package/dist/src/src7.js +1 -1
- package/dist/src/src8.js +2 -2
- package/dist/src/string-enum.js +1 -1
- package/dist/src/takeover.js +1 -1
- package/dist/src/text-entry-focus.js +1 -1
- package/dist/src/tool-provider-facade.js +1 -1
- package/dist/src/tool-provider.js +2 -2
- package/dist/src/tool-provider2.js +1 -1
- package/dist/src/touch-runtime.js +1 -1
- package/dist/src/transport.js +1 -1
- package/dist/src/tree.js +1 -1
- package/dist/src/tv-remote.js +1 -1
- package/dist/src/type-text-runtime.js +1 -1
- package/dist/src/update-check.js +1 -1
- package/dist/src/verified-file.js +1 -1
- package/dist/src/version.js +1 -1
- package/dist/src/video.js +1 -1
- package/dist/src/viewport.js +1 -1
- package/dist/src/web.js +1 -1
- package/dist/src/web2.js +1 -1
- package/dist/src/webdriver-source.js +1 -1
- package/package.json +3 -2
- package/server.json +2 -2
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.12.apk.sha256 +0 -1
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.12.apk +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.12.apk.sha256 +0 -1
- package/dist/src/app-inventory.d.ts +0 -4
- package/dist/src/apps-simctl.js +0 -1
- package/dist/src/cloud-artifacts.d.ts +0 -61
- package/dist/src/runner-contract.js +0 -3
- package/dist/src/snapshot-process.js +0 -1
|
@@ -31,8 +31,10 @@ final class RunnerTests: XCTestCase {
|
|
|
31
31
|
|
|
32
32
|
|
|
33
33
|
|
|
34
|
+
|
|
34
35
|
enum RunnerWireErrorCode {
|
|
35
36
|
static let mainThreadTimeout = "MAIN_THREAD_TIMEOUT"
|
|
37
|
+
static let appNotRunning = "APP_NOT_RUNNING"
|
|
36
38
|
}
|
|
37
39
|
|
|
38
40
|
static let springboardBundleId = "com.apple.springboard"
|
|
@@ -52,6 +54,8 @@ final class RunnerTests: XCTestCase {
|
|
|
52
54
|
let commandExecutionQueue = DispatchQueue(label: "agent-device.runner.commands")
|
|
53
55
|
let app = XCUIApplication()
|
|
54
56
|
lazy var springboard = XCUIApplication(bundleIdentifier: Self.springboardBundleId)
|
|
57
|
+
|
|
58
|
+
|
|
55
59
|
var currentApp: XCUIApplication?
|
|
56
60
|
var currentBundleId: String?
|
|
57
61
|
var currentAppProcessIdentifier: Int?
|
|
@@ -75,11 +79,22 @@ final class RunnerTests: XCTestCase {
|
|
|
75
79
|
let xctestIdleKeepaliveInterval: TimeInterval = 60.0
|
|
76
80
|
let minRecordingFps = 1
|
|
77
81
|
let maxRecordingFps = 120
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
let recordingFrameCaptureTimeout: TimeInterval = 1
|
|
78
86
|
var needsPostSnapshotInteractionDelay = false
|
|
79
87
|
|
|
80
88
|
|
|
81
89
|
|
|
82
90
|
|
|
91
|
+
var lastLoggedFastAppGuardLine: String?
|
|
92
|
+
var lastLoggedGesturePolicyLines: [SynthesizedGesturePolicyKind: String] = [:]
|
|
93
|
+
var runnerMarkerWriter: (String) -> Void = { NSLog("%@", $0) }
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
|
|
83
98
|
var firstInteractionReadyUptime: TimeInterval?
|
|
84
99
|
var runnerAccessibilityHealth: RunnerAccessibilityHealth = .unknown
|
|
85
100
|
var activeRecording: ScreenRecorder?
|
|
@@ -95,6 +110,9 @@ final class RunnerTests: XCTestCase {
|
|
|
95
110
|
|
|
96
111
|
let mainThreadWorkLock = NSLock()
|
|
97
112
|
var abandonedMainThreadWorkCount = 0
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
var mainThreadWorkInFlightCount = 0
|
|
98
116
|
var abandonedMainThreadWorkSince: Date?
|
|
99
117
|
|
|
100
118
|
|
|
@@ -107,7 +125,7 @@ final class RunnerTests: XCTestCase {
|
|
|
107
125
|
var snapshotXCTestChannelPenaltyBundleId: String?
|
|
108
126
|
var snapshotXCTestChannelPenaltyUntil = Date.distantPast
|
|
109
127
|
let snapshotXCTestChannelPenaltyDuration: TimeInterval = 120
|
|
110
|
-
|
|
128
|
+
let snapshotXCTestPenaltyWarmupExemption = SnapshotXCTestPenaltyWarmupExemption()
|
|
111
129
|
|
|
112
130
|
|
|
113
131
|
|
|
@@ -189,6 +207,9 @@ final class RunnerTests: XCTestCase {
|
|
|
189
207
|
|
|
190
208
|
|
|
191
209
|
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
|
|
192
213
|
|
|
193
214
|
|
|
194
215
|
|
|
@@ -263,11 +284,6 @@ final class RunnerTests: XCTestCase {
|
|
|
263
284
|
|
|
264
285
|
@MainActor
|
|
265
286
|
func testCommand() throws {
|
|
266
|
-
if RunnerEnv.isTruthy("AGENT_DEVICE_RUNNER_NOOP_STARTUP") {
|
|
267
|
-
NSLog("AGENT_DEVICE_RUNNER_NOOP_STARTUP=1")
|
|
268
|
-
return
|
|
269
|
-
}
|
|
270
|
-
|
|
271
287
|
doneExpectation = expectation(description: "agent-device command handled")
|
|
272
288
|
NSLog("AGENT_DEVICE_RUNNER_HEADLESS_STARTUP=1")
|
|
273
289
|
let desiredPort = RunnerEnv.resolvePort()
|
|
@@ -11,6 +11,13 @@ let minimumPinchVisibility: CFTimeInterval = 0.5
|
|
|
11
11
|
let swipeVisibilityTail: CFTimeInterval = 0.16
|
|
12
12
|
let trailOpacityKeyTimes: [NSNumber] = [0.0, 0.08, 0.62, 1.0]
|
|
13
13
|
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
let compositedBlackFrameLuma: Double = 1.0
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
let sourceVisibleFrameLuma: Double = 2.0
|
|
20
|
+
|
|
14
21
|
struct GestureEnvelope: Decodable {
|
|
15
22
|
let events: [GestureEvent]
|
|
16
23
|
}
|
|
@@ -129,7 +136,7 @@ func run() throws {
|
|
|
129
136
|
|
|
130
137
|
|
|
131
138
|
|
|
132
|
-
let presetName = exportPresetName(
|
|
139
|
+
let presetName = try exportPresetName(compatibleWith: composition)
|
|
133
140
|
let exporter = try makeRecordingExporter(
|
|
134
141
|
composition,
|
|
135
142
|
presetName: presetName,
|
|
@@ -141,17 +148,19 @@ func run() throws {
|
|
|
141
148
|
timeoutMessage: "Touch overlay export timed out.",
|
|
142
149
|
failureMessage: "Touch overlay export failed."
|
|
143
150
|
)
|
|
151
|
+
try verifyCompositedOverlay(
|
|
152
|
+
input: inputURL,
|
|
153
|
+
output: outputURL,
|
|
154
|
+
expectedRenderSize: renderSize
|
|
155
|
+
)
|
|
144
156
|
}
|
|
145
157
|
|
|
146
158
|
func parseArguments(
|
|
147
159
|
_ arguments: [String]
|
|
148
|
-
) throws -> (inputPath: String, outputPath: String, eventsPath: String
|
|
160
|
+
) throws -> (inputPath: String, outputPath: String, eventsPath: String) {
|
|
149
161
|
var inputPath: String?
|
|
150
162
|
var outputPath: String?
|
|
151
163
|
var eventsPath: String?
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
var exportQuality: ExportQuality = .medium
|
|
155
164
|
var index = 0
|
|
156
165
|
|
|
157
166
|
while index < arguments.count {
|
|
@@ -168,11 +177,13 @@ func parseArguments(
|
|
|
168
177
|
eventsPath = try recordingOptionValue(arguments, nextIndex, "--events")
|
|
169
178
|
index += 2
|
|
170
179
|
case "--quality":
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
|
|
171
183
|
let rawValue = try recordingOptionValue(arguments, nextIndex, "--quality")
|
|
172
|
-
guard
|
|
184
|
+
guard ExportQuality(rawValue: rawValue) != nil else {
|
|
173
185
|
throw RecordingScriptError.invalidArgs("--quality must be one of: medium, high")
|
|
174
186
|
}
|
|
175
|
-
exportQuality = parsed
|
|
176
187
|
index += 2
|
|
177
188
|
default:
|
|
178
189
|
throw RecordingScriptError.invalidArgs("Unknown argument: \(argument)")
|
|
@@ -184,24 +195,22 @@ func parseArguments(
|
|
|
184
195
|
"Usage: recording-overlay.swift --input <video> --output <video> --events <json> [--quality <medium|high>]"
|
|
185
196
|
)
|
|
186
197
|
}
|
|
187
|
-
return (inputPath, outputPath, eventsPath
|
|
198
|
+
return (inputPath, outputPath, eventsPath)
|
|
188
199
|
}
|
|
189
200
|
|
|
190
|
-
func exportPresetName(
|
|
191
|
-
for exportQuality: ExportQuality,
|
|
192
|
-
compatibleWith asset: AVAsset
|
|
193
|
-
) -> String {
|
|
194
|
-
switch exportQuality {
|
|
195
|
-
case .high:
|
|
196
|
-
return AVAssetExportPresetHighestQuality
|
|
197
|
-
case .medium:
|
|
198
201
|
|
|
199
202
|
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
func exportPresetName(compatibleWith asset: AVAsset) throws -> String {
|
|
208
|
+
guard AVAssetExportSession.exportPresets(compatibleWith: asset).contains(AVAssetExportPresetHighestQuality) else {
|
|
209
|
+
throw RecordingScriptError.exportFailed(
|
|
210
|
+
"No geometry-preserving export preset is available; refusing to rescale the capture."
|
|
211
|
+
)
|
|
204
212
|
}
|
|
213
|
+
return AVAssetExportPresetHighestQuality
|
|
205
214
|
}
|
|
206
215
|
|
|
207
216
|
func resolvedRenderSize(for track: AVAssetTrack) -> CGSize {
|
|
@@ -209,6 +218,117 @@ func resolvedRenderSize(for track: AVAssetTrack) -> CGSize {
|
|
|
209
218
|
return CGSize(width: abs(transformed.width), height: abs(transformed.height))
|
|
210
219
|
}
|
|
211
220
|
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
func verifyCompositedOverlay(input: URL, output: URL, expectedRenderSize: CGSize) throws {
|
|
226
|
+
let composited = AVURLAsset(url: output)
|
|
227
|
+
guard let track = composited.tracks(withMediaType: .video).first else {
|
|
228
|
+
throw RecordingScriptError.exportFailed("Touch overlay export produced no video track.")
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
let producedSize = resolvedRenderSize(for: track)
|
|
232
|
+
if !renderSizeMatches(producedSize, expectedRenderSize) {
|
|
233
|
+
throw RecordingScriptError.exportFailed(
|
|
234
|
+
"Touch overlay export changed the track geometry: \(Int(producedSize.width))x\(Int(producedSize.height)) instead of \(Int(expectedRenderSize.width))x\(Int(expectedRenderSize.height))."
|
|
235
|
+
)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
guard let compositedLuma = meanFrameLuma(of: composited) else {
|
|
240
|
+
throw RecordingScriptError.exportFailed("Touch overlay export produced no decodable frames.")
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
let source = AVURLAsset(url: input)
|
|
244
|
+
let sourceDuration = CMTimeGetSeconds(source.duration)
|
|
245
|
+
let producedDuration = CMTimeGetSeconds(composited.duration)
|
|
246
|
+
if sourceDuration.isFinite, sourceDuration > 1, producedDuration.isFinite,
|
|
247
|
+
producedDuration < sourceDuration * 0.5
|
|
248
|
+
{
|
|
249
|
+
throw RecordingScriptError.exportFailed(
|
|
250
|
+
"Touch overlay export truncated the track to \(Int(producedDuration))s of \(Int(sourceDuration))s."
|
|
251
|
+
)
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
if compositedLuma <= compositedBlackFrameLuma,
|
|
255
|
+
let sourceLuma = meanFrameLuma(of: source),
|
|
256
|
+
sourceLuma > sourceVisibleFrameLuma
|
|
257
|
+
{
|
|
258
|
+
throw RecordingScriptError.exportFailed(
|
|
259
|
+
"Touch overlay export produced an all-black track while the raw capture had visible content."
|
|
260
|
+
)
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
func renderSizeMatches(_ produced: CGSize, _ expected: CGSize) -> Bool {
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
let tolerance: CGFloat = 32
|
|
268
|
+
return abs(produced.width - expected.width) <= tolerance
|
|
269
|
+
&& abs(produced.height - expected.height) <= tolerance
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
func meanFrameLuma(of asset: AVURLAsset, sampleCount: Int = 5) -> Double? {
|
|
273
|
+
let duration = CMTimeGetSeconds(asset.duration)
|
|
274
|
+
guard duration.isFinite, duration > 0 else { return nil }
|
|
275
|
+
let generator = AVAssetImageGenerator(asset: asset)
|
|
276
|
+
generator.appliesPreferredTrackTransform = true
|
|
277
|
+
generator.requestedTimeToleranceBefore = .positiveInfinity
|
|
278
|
+
generator.requestedTimeToleranceAfter = .positiveInfinity
|
|
279
|
+
|
|
280
|
+
generator.maximumSize = CGSize(width: 64, height: 64)
|
|
281
|
+
|
|
282
|
+
var total = 0.0
|
|
283
|
+
var sampled = 0
|
|
284
|
+
for index in 0..<sampleCount {
|
|
285
|
+
let fraction = (Double(index) + 0.5) / Double(sampleCount)
|
|
286
|
+
let time = CMTime(seconds: fraction * duration, preferredTimescale: 600)
|
|
287
|
+
guard let image = try? generator.copyCGImage(at: time, actualTime: nil),
|
|
288
|
+
let luma = frameMeanLuma(image)
|
|
289
|
+
else { continue }
|
|
290
|
+
total += luma
|
|
291
|
+
sampled += 1
|
|
292
|
+
}
|
|
293
|
+
guard sampled > 0 else { return nil }
|
|
294
|
+
return total / Double(sampled)
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
func frameMeanLuma(_ image: CGImage) -> Double? {
|
|
298
|
+
let width = image.width
|
|
299
|
+
let height = image.height
|
|
300
|
+
guard width > 0, height > 0 else { return nil }
|
|
301
|
+
let bytesPerRow = width * 4
|
|
302
|
+
var pixels = [UInt8](repeating: 0, count: bytesPerRow * height)
|
|
303
|
+
let drawn = pixels.withUnsafeMutableBytes { raw -> Bool in
|
|
304
|
+
guard
|
|
305
|
+
let context = CGContext(
|
|
306
|
+
data: raw.baseAddress,
|
|
307
|
+
width: width,
|
|
308
|
+
height: height,
|
|
309
|
+
bitsPerComponent: 8,
|
|
310
|
+
bytesPerRow: bytesPerRow,
|
|
311
|
+
space: CGColorSpaceCreateDeviceRGB(),
|
|
312
|
+
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
|
|
313
|
+
)
|
|
314
|
+
else { return false }
|
|
315
|
+
context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
|
|
316
|
+
return true
|
|
317
|
+
}
|
|
318
|
+
guard drawn else { return nil }
|
|
319
|
+
|
|
320
|
+
var total = 0.0
|
|
321
|
+
var count = 0
|
|
322
|
+
for offset in stride(from: 0, to: bytesPerRow * height, by: 4) {
|
|
323
|
+
total += 0.299 * Double(pixels[offset])
|
|
324
|
+
+ 0.587 * Double(pixels[offset + 1])
|
|
325
|
+
+ 0.114 * Double(pixels[offset + 2])
|
|
326
|
+
count += 1
|
|
327
|
+
}
|
|
328
|
+
guard count > 0 else { return nil }
|
|
329
|
+
return total / Double(count)
|
|
330
|
+
}
|
|
331
|
+
|
|
212
332
|
func resolvedFrameDuration(for track: AVAssetTrack) -> CMTime {
|
|
213
333
|
let minFrameDuration = track.minFrameDuration
|
|
214
334
|
if minFrameDuration.isValid && !minFrameDuration.isIndefinite && minFrameDuration.seconds > 0 {
|
|
@@ -105,7 +105,7 @@ public enum CoordinateSpaceRotation {
|
|
|
105
105
|
|
|
106
106
|
public enum SnapshotGeometrySpace: Equatable {
|
|
107
107
|
case appOrientation
|
|
108
|
-
case deviceNative(appFrame:
|
|
108
|
+
case deviceNative(appFrame: SnapshotViewport.Box, interfaceOrientation: Int)
|
|
109
109
|
|
|
110
110
|
public static let quarterTurnTolerance: Double = 1
|
|
111
111
|
|
|
@@ -116,22 +116,26 @@ public enum SnapshotGeometrySpace: Equatable {
|
|
|
116
116
|
case .deviceNative(let appFrame, let interfaceOrientation):
|
|
117
117
|
return CoordinateSpaceRotation.oriented(
|
|
118
118
|
rect: reportedFrame,
|
|
119
|
-
in: appFrame,
|
|
119
|
+
in: appFrame.rect,
|
|
120
120
|
interfaceOrientation: interfaceOrientation
|
|
121
121
|
)
|
|
122
122
|
}
|
|
123
123
|
}
|
|
124
124
|
|
|
125
|
+
/// Only a `.reported` viewport carries an orientation, so only it can anchor a rotation.
|
|
125
126
|
public static func space(
|
|
126
127
|
reportedBySurfaceHost isSurfaceHost: Bool,
|
|
127
128
|
reportedFrame: CGRect,
|
|
128
129
|
inheritedFrom inherited: SnapshotGeometrySpace,
|
|
129
|
-
|
|
130
|
-
interfaceOrientation: Int
|
|
130
|
+
viewport: SnapshotViewport
|
|
131
131
|
) -> SnapshotGeometrySpace {
|
|
132
132
|
guard isSurfaceHost else { return inherited }
|
|
133
|
-
guard
|
|
134
|
-
|
|
133
|
+
guard case .reported(let appFrame, let interfaceOrientation) = viewport,
|
|
134
|
+
namesQuarterTurn(interfaceOrientation),
|
|
135
|
+
isQuarterTurned(reportedFrame, relativeTo: appFrame.rect)
|
|
136
|
+
else {
|
|
137
|
+
return .appOrientation
|
|
138
|
+
}
|
|
135
139
|
return .deviceNative(appFrame: appFrame, interfaceOrientation: interfaceOrientation)
|
|
136
140
|
}
|
|
137
141
|
|
|
@@ -145,7 +149,7 @@ public enum SnapshotGeometrySpace: Equatable {
|
|
|
145
149
|
}
|
|
146
150
|
|
|
147
151
|
private static func isQuarterTurned(_ frame: CGRect, relativeTo appFrame: CGRect) -> Bool {
|
|
148
|
-
guard
|
|
152
|
+
guard SnapshotGeometry.isPositiveFinite(frame),
|
|
149
153
|
abs(appFrame.width - appFrame.height) > quarterTurnTolerance
|
|
150
154
|
else {
|
|
151
155
|
return false
|
|
@@ -153,19 +157,12 @@ public enum SnapshotGeometrySpace: Equatable {
|
|
|
153
157
|
return abs(frame.width - appFrame.height) <= quarterTurnTolerance
|
|
154
158
|
&& abs(frame.height - appFrame.width) <= quarterTurnTolerance
|
|
155
159
|
}
|
|
156
|
-
|
|
157
|
-
private static func isPlottable(_ frame: CGRect) -> Bool {
|
|
158
|
-
frame.origin.x.isFinite && frame.origin.y.isFinite
|
|
159
|
-
&& frame.width.isFinite && frame.height.isFinite
|
|
160
|
-
&& frame.width > 0 && frame.height > 0
|
|
161
|
-
}
|
|
162
160
|
}
|
|
163
161
|
|
|
164
162
|
extension SnapshotGeometrySpace {
|
|
165
163
|
public static func normalized(
|
|
166
164
|
nodes: [RawAXNode],
|
|
167
|
-
viewport:
|
|
168
|
-
interfaceOrientation: Int
|
|
165
|
+
viewport: SnapshotViewport
|
|
169
166
|
) -> [RawAXNode] {
|
|
170
167
|
let carriers = SnapshotVisibilityFold.visibilityExemptCarrierTypes
|
|
171
168
|
var spaces = [SnapshotGeometrySpace](repeating: .appOrientation, count: nodes.count)
|
|
@@ -180,16 +177,16 @@ extension SnapshotGeometrySpace {
|
|
|
180
177
|
),
|
|
181
178
|
reportedFrame: node.rect.cgRect,
|
|
182
179
|
inheritedFrom: parentIndex.map { spaces[$0] } ?? .appOrientation,
|
|
183
|
-
|
|
184
|
-
interfaceOrientation: interfaceOrientation
|
|
180
|
+
viewport: viewport
|
|
185
181
|
)
|
|
186
182
|
spaces[position] = nodeSpace
|
|
187
183
|
let frame = nodeSpace.orientedFrame(of: node.rect.cgRect)
|
|
188
184
|
result.append(
|
|
189
185
|
node.replacing(
|
|
190
186
|
rect: SnapshotRect(frame),
|
|
191
|
-
hittable: node.parentIndex
|
|
192
|
-
|
|
187
|
+
hittable: node.parentIndex == nil
|
|
188
|
+
? false
|
|
189
|
+
: SnapshotGeometry.isGeometricallyActionable(
|
|
193
190
|
enabled: node.enabled,
|
|
194
191
|
frame: frame,
|
|
195
192
|
viewport: viewport
|
|
@@ -2,14 +2,23 @@ import Foundation
|
|
|
2
2
|
import CoreGraphics
|
|
3
3
|
|
|
4
4
|
public enum SnapshotGeometry {
|
|
5
|
+
/// Twin of `isPositiveFiniteRect` in `packages/kernel/src/rect.ts`. `CGRect.infinite` is built
|
|
6
|
+
/// from finite components, so it is refused by identity.
|
|
7
|
+
public static func isPositiveFinite(_ rect: CGRect) -> Bool {
|
|
8
|
+
!rect.isInfinite
|
|
9
|
+
&& rect.origin.x.isFinite && rect.origin.y.isFinite
|
|
10
|
+
&& rect.size.width.isFinite && rect.size.height.isFinite
|
|
11
|
+
&& rect.size.width > 0 && rect.size.height > 0
|
|
12
|
+
}
|
|
13
|
+
|
|
5
14
|
public static func effectiveFrame(
|
|
6
15
|
reportedFrame: CGRect,
|
|
7
|
-
viewport:
|
|
16
|
+
viewport: SnapshotViewport,
|
|
8
17
|
ancestorClip: CGRect?
|
|
9
18
|
) -> CGRect {
|
|
10
19
|
var frame = reportedFrame
|
|
11
|
-
if
|
|
12
|
-
frame = clipped(frame, to:
|
|
20
|
+
if let box = viewport.rect {
|
|
21
|
+
frame = clipped(frame, to: box)
|
|
13
22
|
}
|
|
14
23
|
if let ancestorClip {
|
|
15
24
|
frame = clipped(frame, to: ancestorClip)
|
|
@@ -34,15 +43,18 @@ public enum SnapshotGeometry {
|
|
|
34
43
|
)
|
|
35
44
|
}
|
|
36
45
|
|
|
46
|
+
/// The shared `hittable` predicate (#1933), twin of `isGeometricallyActionable` in
|
|
47
|
+
/// `packages/kernel/src/rect.ts`, with `CGRect.contains`'s half-open right and bottom edges.
|
|
48
|
+
/// `nil` when only containment is left to decide and the capture has no viewport box: the node's
|
|
49
|
+
/// `hittable` is then absent on the wire, as it is on the host bridge (#2891).
|
|
37
50
|
public static func isGeometricallyActionable(
|
|
38
51
|
enabled: Bool,
|
|
39
52
|
frame: CGRect,
|
|
40
|
-
viewport:
|
|
41
|
-
) -> Bool {
|
|
42
|
-
guard enabled,
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
return viewport.contains(center)
|
|
53
|
+
viewport: SnapshotViewport
|
|
54
|
+
) -> Bool? {
|
|
55
|
+
guard enabled, isPositiveFinite(frame) else { return false }
|
|
56
|
+
guard let box = viewport.rect else { return nil }
|
|
57
|
+
return box.contains(CGPoint(x: frame.midX, y: frame.midY))
|
|
46
58
|
}
|
|
47
59
|
|
|
48
60
|
private static func clipped(_ frame: CGRect, to clip: CGRect) -> CGRect {
|
|
@@ -38,7 +38,8 @@ public struct RawAXNode: Equatable {
|
|
|
38
38
|
public let enabled: Bool
|
|
39
39
|
public let focused: Bool?
|
|
40
40
|
public let selected: Bool?
|
|
41
|
-
|
|
41
|
+
/// Geometric actionability; `nil` when the capture has no viewport box to decide it against.
|
|
42
|
+
public var hittable: Bool?
|
|
42
43
|
public let depth: Int
|
|
43
44
|
public let parentIndex: Int?
|
|
44
45
|
public let hiddenContentAbove: Bool?
|
|
@@ -55,7 +56,7 @@ public struct RawAXNode: Equatable {
|
|
|
55
56
|
enabled: Bool,
|
|
56
57
|
focused: Bool?,
|
|
57
58
|
selected: Bool?,
|
|
58
|
-
hittable: Bool
|
|
59
|
+
hittable: Bool?,
|
|
59
60
|
depth: Int,
|
|
60
61
|
parentIndex: Int?,
|
|
61
62
|
hiddenContentAbove: Bool?,
|
|
@@ -79,7 +80,7 @@ public struct RawAXNode: Equatable {
|
|
|
79
80
|
self.actions = actions
|
|
80
81
|
}
|
|
81
82
|
|
|
82
|
-
func replacing(rect: SnapshotRect, hittable: Bool) -> RawAXNode {
|
|
83
|
+
func replacing(rect: SnapshotRect, hittable: Bool?) -> RawAXNode {
|
|
83
84
|
var updated = self
|
|
84
85
|
updated.rect = rect
|
|
85
86
|
updated.hittable = hittable
|
|
@@ -153,15 +154,65 @@ public struct PresentationOptions: Equatable {
|
|
|
153
154
|
}
|
|
154
155
|
}
|
|
155
156
|
|
|
157
|
+
/// What a capture knows about the viewport hosting its tree: the three cases of the host's
|
|
158
|
+
/// `IosViewportEvidence` (#2891). No rectangle stands for "unknown".
|
|
159
|
+
public enum SnapshotViewport: Equatable {
|
|
160
|
+
/// A box `SnapshotGeometry.isPositiveFinite` accepted. Only the factories below construct one.
|
|
161
|
+
public struct Box: Equatable {
|
|
162
|
+
public let rect: CGRect
|
|
163
|
+
|
|
164
|
+
init(positiveFinite rect: CGRect) {
|
|
165
|
+
self.rect = rect
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/// The platform's box for the app's surface, with the interface orientation read in the same hop.
|
|
170
|
+
/// Only this case can anchor a rotation (#2612).
|
|
171
|
+
case reported(Box, interfaceOrientation: Int)
|
|
172
|
+
/// A box the capture inferred from its own root element. It clips and contains, and carries no
|
|
173
|
+
/// orientation, so it cannot anchor a rotation.
|
|
174
|
+
case derived(Box)
|
|
175
|
+
case missing(reason: MissingReason)
|
|
176
|
+
|
|
177
|
+
public enum MissingReason: Equatable {
|
|
178
|
+
/// The read was skipped or raised.
|
|
179
|
+
case notProvided
|
|
180
|
+
/// The box read is one `SnapshotGeometry.isPositiveFinite` refuses.
|
|
181
|
+
case invalid
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
public var rect: CGRect? {
|
|
185
|
+
switch self {
|
|
186
|
+
case .reported(let box, _), .derived(let box):
|
|
187
|
+
return box.rect
|
|
188
|
+
case .missing:
|
|
189
|
+
return nil
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
public static func reported(
|
|
194
|
+
box: CGRect,
|
|
195
|
+
interfaceOrientation: Int = RunnerInterfaceOrientation.unknown
|
|
196
|
+
) -> SnapshotViewport {
|
|
197
|
+
SnapshotGeometry.isPositiveFinite(box)
|
|
198
|
+
? .reported(Box(positiveFinite: box), interfaceOrientation: interfaceOrientation)
|
|
199
|
+
: .missing(reason: .invalid)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
public static func derived(box: CGRect) -> SnapshotViewport {
|
|
203
|
+
SnapshotGeometry.isPositiveFinite(box)
|
|
204
|
+
? .derived(Box(positiveFinite: box))
|
|
205
|
+
: .missing(reason: .invalid)
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
|
|
156
209
|
public struct SnapshotAcquisition {
|
|
157
210
|
public let hint: CaptureHint
|
|
158
211
|
public var nodes: [RawAXNode]
|
|
159
212
|
public let truncated: Bool
|
|
160
213
|
public let effectiveDepth: Int?
|
|
161
214
|
public var customActions: SnapshotCustomActionCoverage?
|
|
162
|
-
public let viewport:
|
|
163
|
-
/// The app's interface orientation, consumed by the one `normalized` pass; `unknown` turns nothing.
|
|
164
|
-
public let interfaceOrientation: Int
|
|
215
|
+
public let viewport: SnapshotViewport
|
|
165
216
|
|
|
166
217
|
public init(
|
|
167
218
|
hint: CaptureHint,
|
|
@@ -169,8 +220,7 @@ public struct SnapshotAcquisition {
|
|
|
169
220
|
truncated: Bool,
|
|
170
221
|
effectiveDepth: Int?,
|
|
171
222
|
customActions: SnapshotCustomActionCoverage? = nil,
|
|
172
|
-
viewport:
|
|
173
|
-
interfaceOrientation: Int = 0
|
|
223
|
+
viewport: SnapshotViewport
|
|
174
224
|
) {
|
|
175
225
|
self.hint = hint
|
|
176
226
|
self.nodes = nodes
|
|
@@ -178,7 +228,6 @@ public struct SnapshotAcquisition {
|
|
|
178
228
|
self.effectiveDepth = effectiveDepth
|
|
179
229
|
self.customActions = customActions
|
|
180
230
|
self.viewport = viewport
|
|
181
|
-
self.interfaceOrientation = interfaceOrientation
|
|
182
231
|
}
|
|
183
232
|
|
|
184
233
|
public func replacingNodes(_ nodes: [RawAXNode]) -> SnapshotAcquisition {
|
|
@@ -226,7 +275,7 @@ public struct PresentedNode: Codable, Equatable {
|
|
|
226
275
|
public let enabled: Bool
|
|
227
276
|
public let focused: Bool?
|
|
228
277
|
public let selected: Bool?
|
|
229
|
-
public let hittable: Bool
|
|
278
|
+
public let hittable: Bool?
|
|
230
279
|
public let depth: Int
|
|
231
280
|
public let parentIndex: Int?
|
|
232
281
|
public let hiddenContentAbove: Bool?
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import Foundation
|
|
2
2
|
import CoreGraphics
|
|
3
3
|
|
|
4
|
+
/// With no viewport box the cumulative clip has no root, so containment has nothing to violate.
|
|
4
5
|
public enum SnapshotPresentationInvariant {
|
|
5
6
|
struct ValidationStats: Equatable {
|
|
6
7
|
let parentClipLookups: Int
|
|
@@ -12,7 +13,7 @@ public enum SnapshotPresentationInvariant {
|
|
|
12
13
|
|
|
13
14
|
public static func validateRegular(
|
|
14
15
|
_ nodes: [SnapshotPresentationNode],
|
|
15
|
-
viewport:
|
|
16
|
+
viewport: SnapshotViewport,
|
|
16
17
|
policy: SnapshotVisibilityFold.Policy
|
|
17
18
|
) throws {
|
|
18
19
|
_ = try validateRegularWithStats(nodes, viewport: viewport, policy: policy)
|
|
@@ -20,24 +21,25 @@ public enum SnapshotPresentationInvariant {
|
|
|
20
21
|
|
|
21
22
|
static func validateRegularWithStats(
|
|
22
23
|
_ nodes: [SnapshotPresentationNode],
|
|
23
|
-
viewport:
|
|
24
|
+
viewport: SnapshotViewport,
|
|
24
25
|
policy: SnapshotVisibilityFold.Policy
|
|
25
26
|
) throws -> ValidationStats {
|
|
26
27
|
var parentClipLookups = 0
|
|
27
|
-
var clipIncludingNodeByIndex: [Int: CGRect] = [:]
|
|
28
|
+
var clipIncludingNodeByIndex: [Int: CGRect?] = [:]
|
|
28
29
|
clipIncludingNodeByIndex.reserveCapacity(nodes.count)
|
|
30
|
+
let rootClip = viewport.rect
|
|
29
31
|
|
|
30
32
|
for node in nodes {
|
|
31
|
-
let ancestorClip: CGRect
|
|
33
|
+
let ancestorClip: CGRect?
|
|
32
34
|
if let parentIndex = node.raw.parentIndex {
|
|
33
35
|
parentClipLookups += 1
|
|
34
|
-
ancestorClip = clipIncludingNodeByIndex[parentIndex] ??
|
|
36
|
+
ancestorClip = clipIncludingNodeByIndex[parentIndex] ?? rootClip
|
|
35
37
|
} else {
|
|
36
|
-
ancestorClip =
|
|
38
|
+
ancestorClip = rootClip
|
|
37
39
|
}
|
|
38
40
|
|
|
39
41
|
let frame = node.effectiveRect.cgRect
|
|
40
|
-
let clipIncludingNode: CGRect
|
|
42
|
+
let clipIncludingNode: CGRect?
|
|
41
43
|
if policy == .cursorProjected,
|
|
42
44
|
SnapshotVisibilityFold.scrollContainerTypeNames.contains(node.raw.type),
|
|
43
45
|
!frame.isNull,
|
|
@@ -50,7 +52,7 @@ public enum SnapshotPresentationInvariant {
|
|
|
50
52
|
clipIncludingNodeByIndex[node.raw.index] = clipIncludingNode
|
|
51
53
|
|
|
52
54
|
guard !frame.isNull, !frame.isEmpty else {
|
|
53
|
-
if node.raw.hittable {
|
|
55
|
+
if node.raw.hittable == true {
|
|
54
56
|
throw SnapshotPresentationFailure.regularDegenerateNodeIsActionable(
|
|
55
57
|
index: node.raw.index,
|
|
56
58
|
frame: node.effectiveRect
|
|
@@ -59,11 +61,11 @@ public enum SnapshotPresentationInvariant {
|
|
|
59
61
|
continue
|
|
60
62
|
}
|
|
61
63
|
|
|
62
|
-
|
|
64
|
+
if let clip = ancestorClip, !contains(frame, in: clip) {
|
|
63
65
|
throw SnapshotPresentationFailure.regularNodeOutsideCumulativeClip(
|
|
64
66
|
index: node.raw.index,
|
|
65
67
|
frame: node.effectiveRect,
|
|
66
|
-
clip: SnapshotRect(
|
|
68
|
+
clip: SnapshotRect(clip)
|
|
67
69
|
)
|
|
68
70
|
}
|
|
69
71
|
}
|
|
@@ -100,7 +100,7 @@ public enum SnapshotVisibilityFold {
|
|
|
100
100
|
public static func traversalDecision(
|
|
101
101
|
for node: RawAXNode,
|
|
102
102
|
parent: TraversalState,
|
|
103
|
-
viewport:
|
|
103
|
+
viewport: SnapshotViewport,
|
|
104
104
|
interactiveOnly: Bool,
|
|
105
105
|
hasChildren: Bool,
|
|
106
106
|
policy: Policy
|
|
@@ -153,7 +153,7 @@ public enum SnapshotVisibilityFold {
|
|
|
153
153
|
|
|
154
154
|
public static func fold(
|
|
155
155
|
_ nodes: [RawAXNode],
|
|
156
|
-
viewport:
|
|
156
|
+
viewport: SnapshotViewport,
|
|
157
157
|
interactiveOnly: Bool,
|
|
158
158
|
policy: Policy
|
|
159
159
|
) -> [SnapshotPresentationNode] {
|
|
@@ -203,12 +203,17 @@ public enum SnapshotVisibilityFold {
|
|
|
203
203
|
enabled: node.enabled,
|
|
204
204
|
focused: node.focused,
|
|
205
205
|
selected: node.selected,
|
|
206
|
-
hittable: node.parentIndex
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
206
|
+
hittable: node.parentIndex == nil
|
|
207
|
+
? false
|
|
208
|
+
: node.hittable.flatMap { sourceHittable in
|
|
209
|
+
sourceHittable
|
|
210
|
+
? SnapshotGeometry.isGeometricallyActionable(
|
|
211
|
+
enabled: node.enabled,
|
|
212
|
+
frame: decision.effectiveFrame,
|
|
213
|
+
viewport: viewport
|
|
214
|
+
)
|
|
215
|
+
: false
|
|
216
|
+
},
|
|
212
217
|
depth: outDepth,
|
|
213
218
|
parentIndex: keptIndex,
|
|
214
219
|
hiddenContentAbove: node.hiddenContentAbove,
|