agent-device 0.20.4 → 0.20.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.4.apk → agent-device-android-ime-helper-0.20.6.apk} +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.6.apk.sha256 +1 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.4.manifest.json → agent-device-android-ime-helper-0.20.6.manifest.json} +4 -4
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.4.apk → agent-device-android-snapshot-helper-0.20.6.apk} +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.6.apk.sha256 +1 -0
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.4.manifest.json → agent-device-android-snapshot-helper-0.20.6.manifest.json} +6 -6
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +28 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +41 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +298 -92
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +126 -187
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +32 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +212 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +118 -8
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +134 -27
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +23 -5
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +10 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +0 -67
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +11 -1
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +2 -22
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +48 -11
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +0 -20
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +177 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +105 -448
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +4 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +587 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +18 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +72 -0
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +89 -0
- package/dist/src/agent-device-client.js +2 -2
- package/dist/src/android.js +1 -1
- package/dist/src/app-log-request-scope.js +1 -1
- package/dist/src/app-resolution.js +1 -1
- package/dist/src/apps.js +5 -5
- package/dist/src/cli-config.js +1 -1
- package/dist/src/cli-help.js +34 -24
- package/dist/src/cli-runner.js +1 -1
- package/dist/src/cli.js +15 -15
- package/dist/src/command-metadata.js +1 -1
- package/dist/src/command-schema.js +3 -3
- package/dist/src/context.js +1 -1
- package/dist/src/debug-symbols.js +1 -1
- package/dist/src/find.js +1 -1
- package/dist/src/finders.js +1 -1
- package/dist/src/generic-settle.js +1 -0
- package/dist/src/generic.js +1 -1
- package/dist/src/ime-lifecycle.js +3 -3
- package/dist/src/index.d.ts +27 -7
- package/dist/src/input-actions.js +4 -4
- package/dist/src/interaction-runtime.js +1 -0
- package/dist/src/interaction-snapshot.js +1 -1
- package/dist/src/interaction-touch-reference-frame.js +1 -0
- package/dist/src/interaction.js +1 -1
- package/dist/src/interactor.js +2 -2
- package/dist/src/interactor2.js +1 -1
- package/dist/src/internal/bin.js +2 -2
- package/dist/src/internal/daemon.js +15 -15
- package/dist/src/lease-context.js +1 -1
- package/dist/src/lease.js +1 -1
- package/dist/src/limrun.d.ts +23 -2
- package/dist/src/linux.js +1 -1
- package/dist/src/process-exit.js +1 -0
- package/dist/src/react-native.js +1 -1
- package/dist/src/record-trace.js +1 -1
- package/dist/src/register-builtins.js +1 -1
- package/dist/src/registry.js +25 -24
- package/dist/src/remote-config.js +1 -1
- package/dist/src/remote-config2.js +1 -1
- package/dist/src/reporting.js +2 -2
- package/dist/src/runner-client.js +1 -1
- package/dist/src/runner-disposal.js +1 -1
- package/dist/src/runner-lease.js +1 -1
- package/dist/src/runtime.js +3 -3
- package/dist/src/screenshot-geometry.js +1 -1
- package/dist/src/screenshot-result.js +4 -4
- package/dist/src/sdk-android-adb.d.ts +20 -2
- package/dist/src/sdk-android-adb.js +1 -1
- package/dist/src/sdk-batch-runner.js +3 -3
- package/dist/src/sdk-contracts.d.ts +17 -2
- package/dist/src/sdk-finders.d.ts +4 -20
- package/dist/src/sdk-finders.js +1 -1
- package/dist/src/{interaction.d.ts → sdk-selectors.d.ts} +98 -3
- package/dist/src/sdk-selectors.js +1 -1
- package/dist/src/selector-runtime.js +1 -1
- package/dist/src/selector-vocabulary.js +1 -1
- package/dist/src/selectors.d.ts +1 -46
- package/dist/src/selectors.js +1 -1
- package/dist/src/server.js +3 -3
- package/dist/src/session-target-evidence.js +1 -1
- package/dist/src/session.js +9 -8
- package/dist/src/snapshot2.js +1 -1
- package/dist/src/text-surface.js +1 -0
- package/dist/src/viewport-dimension.js +13 -12
- package/dist/src/web.js +1 -1
- package/package.json +42 -34
- package/server.json +3 -3
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.4.apk.sha256 +0 -1
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.4.apk.sha256 +0 -1
- package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -225
package/README.md
CHANGED
|
@@ -61,6 +61,8 @@ agent-device close
|
|
|
61
61
|
|
|
62
62
|
Use refs only from the latest output. Do not assume an earlier `@eN` still identifies the same element. After a command with `--settle`, use the refs in its diff. Take another snapshot only if the diff omits what you need.
|
|
63
63
|
|
|
64
|
+
`--settle` works the same way on `scroll` and `back`, so scroll-then-observe and back-then-observe are one call too.
|
|
65
|
+
|
|
64
66
|
Snapshots use the app's accessibility tree. Clear labels, roles, and test IDs make agent runs more reliable. Use screenshots and videos as evidence or when accessibility data is poor. Use refs and selectors for actions and assertions when you can.
|
|
65
67
|
|
|
66
68
|

|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
7f719c5e1698e827948a9de22bf31aab065d7f2cb7d17731793b586fd3f88f4f agent-device-android-ime-helper-0.20.6.apk
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "android-ime-helper",
|
|
3
|
-
"version": "0.20.
|
|
4
|
-
"assetName": "agent-device-android-ime-helper-0.20.
|
|
5
|
-
"sha256": "
|
|
3
|
+
"version": "0.20.6",
|
|
4
|
+
"assetName": "agent-device-android-ime-helper-0.20.6.apk",
|
|
5
|
+
"sha256": "7f719c5e1698e827948a9de22bf31aab065d7f2cb7d17731793b586fd3f88f4f",
|
|
6
6
|
"packageName": "com.callstack.agentdevice.imehelper",
|
|
7
|
-
"versionCode":
|
|
7
|
+
"versionCode": 20006,
|
|
8
8
|
"serviceComponent": "com.callstack.agentdevice.imehelper/.TestInputMethodService",
|
|
9
9
|
"broadcastProtocol": "android-ime-helper-v1"
|
|
10
10
|
}
|
|
Binary file
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
696f8e518bbc2ad66225cb80f2e9e70d2b19b75d382918edd8fce3009200e434 agent-device-android-snapshot-helper-0.20.6.apk
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "android-snapshot-helper",
|
|
3
|
-
"version": "0.20.
|
|
4
|
-
"releaseTag": "v0.20.
|
|
5
|
-
"assetName": "agent-device-android-snapshot-helper-0.20.
|
|
3
|
+
"version": "0.20.6",
|
|
4
|
+
"releaseTag": "v0.20.6",
|
|
5
|
+
"assetName": "agent-device-android-snapshot-helper-0.20.6.apk",
|
|
6
6
|
"apkUrl": null,
|
|
7
|
-
"sha256": "
|
|
8
|
-
"checksumName": "agent-device-android-snapshot-helper-0.20.
|
|
7
|
+
"sha256": "696f8e518bbc2ad66225cb80f2e9e70d2b19b75d382918edd8fce3009200e434",
|
|
8
|
+
"checksumName": "agent-device-android-snapshot-helper-0.20.6.apk.sha256",
|
|
9
9
|
"packageName": "com.callstack.agentdevice.snapshothelper",
|
|
10
|
-
"versionCode":
|
|
10
|
+
"versionCode": 20006,
|
|
11
11
|
"instrumentationRunner": "com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation",
|
|
12
12
|
"minSdk": 23,
|
|
13
13
|
"targetSdk": 36,
|
|
@@ -62,6 +62,13 @@ int main(int argc, const char *argv[]) {
|
|
|
62
62
|
|
|
63
63
|
@implementation AgentDeviceRunnerViewController
|
|
64
64
|
|
|
65
|
+
- (void)agentDeviceTextEntryDidChange:(UITextField *)textField {
|
|
66
|
+
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-disappear-after-input"] &&
|
|
67
|
+
textField.text.length > 0) {
|
|
68
|
+
[textField removeFromSuperview];
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
65
72
|
- (void)viewDidLoad {
|
|
66
73
|
[super viewDidLoad];
|
|
67
74
|
|
|
@@ -78,6 +85,27 @@ int main(int argc, const char *argv[]) {
|
|
|
78
85
|
[label.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
|
|
79
86
|
[label.centerYAnchor constraintEqualToAnchor:self.view.centerYAnchor],
|
|
80
87
|
]];
|
|
88
|
+
|
|
89
|
+
// Keep the fixture behind a launch argument so normal runner snapshots remain unchanged.
|
|
90
|
+
#if TARGET_OS_IOS
|
|
91
|
+
if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) {
|
|
92
|
+
UITextField *textField = [[UITextField alloc] init];
|
|
93
|
+
textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input";
|
|
94
|
+
textField.borderStyle = UITextBorderStyleRoundedRect;
|
|
95
|
+
textField.inputView = [[UIView alloc] initWithFrame:CGRectMake(0, 0, 1, 1)];
|
|
96
|
+
[textField addTarget:self
|
|
97
|
+
action:@selector(agentDeviceTextEntryDidChange:)
|
|
98
|
+
forControlEvents:UIControlEventEditingChanged];
|
|
99
|
+
textField.translatesAutoresizingMaskIntoConstraints = NO;
|
|
100
|
+
[self.view addSubview:textField];
|
|
101
|
+
[NSLayoutConstraint activateConstraints:@[
|
|
102
|
+
[textField.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
|
|
103
|
+
[textField.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:24],
|
|
104
|
+
[textField.widthAnchor constraintEqualToConstant:240],
|
|
105
|
+
[textField.heightAnchor constraintEqualToConstant:44],
|
|
106
|
+
]];
|
|
107
|
+
}
|
|
108
|
+
#endif
|
|
81
109
|
}
|
|
82
110
|
|
|
83
111
|
@end
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h
CHANGED
|
@@ -3,14 +3,54 @@
|
|
|
3
3
|
|
|
4
4
|
NS_ASSUME_NONNULL_BEGIN
|
|
5
5
|
|
|
6
|
+
/// Keys of the `deepExtension` dictionary in the snapshot response, shared
|
|
7
|
+
/// with the Swift consumer so the response shape has one spelling.
|
|
8
|
+
FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionKey;
|
|
9
|
+
FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionCallsKey;
|
|
10
|
+
FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionNodesAddedKey;
|
|
11
|
+
FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionPendingKey;
|
|
12
|
+
FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionMissedKey;
|
|
13
|
+
|
|
14
|
+
/// A depth-capped childless node awaiting an element-rooted follow-up request.
|
|
15
|
+
/// Public so the runner unit bundle can drive the extension's miss paths with
|
|
16
|
+
/// fabricated snapshots — the executed contract that missed frontiers are
|
|
17
|
+
/// counted, never silently dropped.
|
|
18
|
+
@interface RunnerAXSnapshotFrontier : NSObject
|
|
19
|
+
@property(nonatomic, strong, nullable) id snapshot;
|
|
20
|
+
@property(nonatomic, strong, nullable) NSMutableDictionary *node;
|
|
21
|
+
@property(nonatomic, assign) NSInteger depth;
|
|
22
|
+
@end
|
|
23
|
+
|
|
6
24
|
@interface RunnerAXSnapshotBridge : NSObject
|
|
7
25
|
|
|
26
|
+
/// Captures the app's accessibility tree, then chains element-rooted follow-up
|
|
27
|
+
/// requests from depth-capped frontier nodes (the AX depth limit is
|
|
28
|
+
/// per-request, so re-rooting reaches content the app-rooted request could
|
|
29
|
+
/// not). The response gains a `deepExtension` {calls, nodesAdded,
|
|
30
|
+
/// pendingFrontiers, missedFrontiers} dictionary when any frontier existed and
|
|
31
|
+
/// the call limit allowed extension. Pass 0 to disable extension entirely.
|
|
8
32
|
+ (NSDictionary<NSString *, id> *)snapshotTreeForApplication:(XCUIApplication *)application
|
|
9
33
|
maxDepth:(NSInteger)maxDepth
|
|
10
|
-
maxNodes:(NSInteger)maxNodes
|
|
34
|
+
maxNodes:(NSInteger)maxNodes
|
|
35
|
+
deepExtensionCallLimit:(NSInteger)deepExtensionCallLimit
|
|
36
|
+
deadline:(nullable NSDate *)deadline;
|
|
11
37
|
|
|
12
38
|
+ (NSInteger)processIdentifierForApplication:(XCUIApplication *)application;
|
|
13
39
|
|
|
40
|
+
/// The frontier-extension loop, exposed for the runner unit bundle: a frontier
|
|
41
|
+
/// whose element vanished or whose re-rooted request fails must count as
|
|
42
|
+
/// missed (see the deepExtension keys above) — an all-miss extension reporting
|
|
43
|
+
/// itself drained would present a capped capture as complete.
|
|
44
|
+
+ (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray<RunnerAXSnapshotFrontier *> *)frontiers
|
|
45
|
+
axClient:(id)axClient
|
|
46
|
+
attributes:(NSArray *)attributes
|
|
47
|
+
maxDepth:(NSInteger)maxDepth
|
|
48
|
+
maxNodes:(NSInteger)maxNodes
|
|
49
|
+
nodeCount:(NSInteger *)nodeCount
|
|
50
|
+
truncated:(BOOL *)truncated
|
|
51
|
+
callsAllowed:(NSInteger)callsAllowed
|
|
52
|
+
deadline:(nullable NSDate *)deadline;
|
|
53
|
+
|
|
14
54
|
@end
|
|
15
55
|
|
|
16
56
|
NS_ASSUME_NONNULL_END
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m
CHANGED
|
@@ -8,15 +8,32 @@ static NSString *const RunnerAXSnapshotErrorKey = @"error";
|
|
|
8
8
|
static NSString *const RunnerAXSnapshotRootKey = @"root";
|
|
9
9
|
static NSString *const RunnerAXSnapshotTruncatedKey = @"truncated";
|
|
10
10
|
|
|
11
|
+
NSString *const RunnerAXSnapshotDeepExtensionKey = @"deepExtension";
|
|
12
|
+
NSString *const RunnerAXSnapshotDeepExtensionCallsKey = @"calls";
|
|
13
|
+
NSString *const RunnerAXSnapshotDeepExtensionNodesAddedKey = @"nodesAdded";
|
|
14
|
+
NSString *const RunnerAXSnapshotDeepExtensionPendingKey = @"pendingFrontiers";
|
|
15
|
+
NSString *const RunnerAXSnapshotDeepExtensionMissedKey = @"missedFrontiers";
|
|
16
|
+
|
|
11
17
|
typedef id (*RunnerAXObjectMsgSend)(id, SEL);
|
|
12
18
|
typedef NSInteger (*RunnerAXIntegerMsgSend)(id, SEL);
|
|
13
19
|
typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
14
20
|
|
|
21
|
+
// A childless serialized node remembered with its depth (interface in the
|
|
22
|
+
// header for the unit bundle). Nodes at the deepest observed level of a
|
|
23
|
+
// depth-capped capture are the truncation frontier: the AX server withholds
|
|
24
|
+
// children uniformly at its per-request cap, so every capped branch ends at
|
|
25
|
+
// the same level (observed one BELOW the requested maxDepth — the server
|
|
26
|
+
// counts node levels, not edges).
|
|
27
|
+
@implementation RunnerAXSnapshotFrontier
|
|
28
|
+
@end
|
|
29
|
+
|
|
15
30
|
@implementation RunnerAXSnapshotBridge
|
|
16
31
|
|
|
17
32
|
+ (NSDictionary<NSString *, id> *)snapshotTreeForApplication:(XCUIApplication *)application
|
|
18
33
|
maxDepth:(NSInteger)maxDepth
|
|
19
34
|
maxNodes:(NSInteger)maxNodes
|
|
35
|
+
deepExtensionCallLimit:(NSInteger)deepExtensionCallLimit
|
|
36
|
+
deadline:(nullable NSDate *)deadline
|
|
20
37
|
{
|
|
21
38
|
@try {
|
|
22
39
|
id axClient = [self objectFrom:XCUIDevice.sharedDevice selectorName:@"accessibilityInterface"];
|
|
@@ -29,103 +46,277 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
29
46
|
return [self failure:@"Could not match active AX application for XCTest application"];
|
|
30
47
|
}
|
|
31
48
|
|
|
32
|
-
|
|
33
|
-
id defaults = [self objectFrom:axClient selectorName:@"defaultParameters"];
|
|
34
|
-
if ([defaults isKindOfClass:NSDictionary.class]) {
|
|
35
|
-
[parameters addEntriesFromDictionary:(NSDictionary *)defaults];
|
|
36
|
-
}
|
|
37
|
-
parameters[@"maxDepth"] = @(MAX(0, maxDepth));
|
|
38
|
-
parameters[@"maxChildren"] = @(MAX(1, maxNodes));
|
|
39
|
-
parameters[@"maxArrayCount"] = @(MAX(1, maxNodes));
|
|
40
|
-
parameters[@"traverseFromParentsToChildren"] = @YES;
|
|
41
|
-
|
|
42
|
-
SEL requestSelector = NSSelectorFromString(@"requestSnapshotForElement:attributes:parameters:error:");
|
|
43
|
-
if (![axClient respondsToSelector:requestSelector]) {
|
|
44
|
-
return [self failure:@"AX client does not support requestSnapshotForElement"];
|
|
45
|
-
}
|
|
46
|
-
|
|
49
|
+
NSArray *attributes = [self snapshotAttributes];
|
|
47
50
|
NSError *error = nil;
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
@"enabled",
|
|
55
|
-
@"selected",
|
|
56
|
-
@"hasFocus",
|
|
57
|
-
@"children",
|
|
58
|
-
];
|
|
59
|
-
// The AX server expects real accessibility attribute identifiers, not snapshot keypath
|
|
60
|
-
// strings; passing raw keypaths silently drops attributes it does not recognize (frame
|
|
61
|
-
// came back zeroed). XCElementSnapshot owns the keypath -> AX attribute mapping.
|
|
62
|
-
NSArray *attributes = keyPaths;
|
|
63
|
-
Class snapshotClass = NSClassFromString(@"XCElementSnapshot");
|
|
64
|
-
SEL mapSelector = NSSelectorFromString(@"axAttributesForElementSnapshotKeyPaths:isMacOS:");
|
|
65
|
-
if ([snapshotClass respondsToSelector:mapSelector]) {
|
|
66
|
-
typedef id (*RunnerAXMapMsgSend)(id, SEL, id, BOOL);
|
|
67
|
-
RunnerAXMapMsgSend mapSend = (RunnerAXMapMsgSend)objc_msgSend;
|
|
68
|
-
id mapped = mapSend(snapshotClass, mapSelector, keyPaths, NO);
|
|
69
|
-
if ([mapped isKindOfClass:NSSet.class]) {
|
|
70
|
-
mapped = [(NSSet *)mapped allObjects];
|
|
71
|
-
}
|
|
72
|
-
if ([mapped isKindOfClass:NSArray.class] && [(NSArray *)mapped count] > 0) {
|
|
73
|
-
// The mapper expands keypaths with extra attributes (automation type, window display
|
|
74
|
-
// id, base type) that are disproportionately expensive for the AX server to compute
|
|
75
|
-
// on large React Native trees. Keep only the attributes we actually consume.
|
|
76
|
-
NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"Frame",
|
|
77
|
-
@"Enabled", @"Selected", @"Focus" ];
|
|
78
|
-
NSMutableArray *filtered = [NSMutableArray array];
|
|
79
|
-
for (id attribute in (NSArray *)mapped) {
|
|
80
|
-
NSString *name = [attribute description];
|
|
81
|
-
for (NSString *suffix in needed) {
|
|
82
|
-
if ([name hasSuffix:suffix]) {
|
|
83
|
-
[filtered addObject:attribute];
|
|
84
|
-
break;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
attributes = filtered.count > 0 ? filtered : mapped;
|
|
89
|
-
}
|
|
90
|
-
}
|
|
91
|
-
RunnerAXSnapshotMsgSend send = (RunnerAXSnapshotMsgSend)objc_msgSend;
|
|
92
|
-
id result = send(axClient, requestSelector, target, attributes, parameters.copy, &error);
|
|
93
|
-
if (nil == result) {
|
|
94
|
-
return [self failure:error.localizedDescription ?: @"AX snapshot request returned nil"];
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
id root = nil;
|
|
98
|
-
@try {
|
|
99
|
-
root = [result valueForKey:@"_rootElementSnapshot"];
|
|
100
|
-
} @catch (NSException *exception) {
|
|
101
|
-
root = nil;
|
|
102
|
-
}
|
|
51
|
+
id root = [self requestSnapshotFromClient:axClient
|
|
52
|
+
target:target
|
|
53
|
+
attributes:attributes
|
|
54
|
+
maxDepth:maxDepth
|
|
55
|
+
maxNodes:maxNodes
|
|
56
|
+
error:&error];
|
|
103
57
|
if (nil == root) {
|
|
104
|
-
|
|
58
|
+
return [self failure:error.localizedDescription ?: @"AX snapshot request returned nil"];
|
|
105
59
|
}
|
|
106
60
|
|
|
107
61
|
BOOL truncated = NO;
|
|
108
62
|
NSInteger nodeCount = 0;
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
63
|
+
// nil disables candidate collection entirely — exact --depth captures pay
|
|
64
|
+
// zero extension bookkeeping.
|
|
65
|
+
NSMutableArray<RunnerAXSnapshotFrontier *> *candidates =
|
|
66
|
+
deepExtensionCallLimit > 0 ? [NSMutableArray array] : nil;
|
|
67
|
+
NSMutableDictionary *rootNode = [self dictionaryForSnapshot:root
|
|
68
|
+
depth:0
|
|
69
|
+
maxDepth:maxDepth
|
|
70
|
+
maxNodes:maxNodes
|
|
71
|
+
nodeCount:&nodeCount
|
|
72
|
+
truncated:&truncated
|
|
73
|
+
frontiers:candidates];
|
|
115
74
|
if (nil == rootNode) {
|
|
116
75
|
return [self failure:@"AX snapshot root could not be serialized"];
|
|
117
76
|
}
|
|
118
77
|
|
|
119
|
-
|
|
78
|
+
NSMutableArray<RunnerAXSnapshotFrontier *> *frontiers =
|
|
79
|
+
[self cappedFrontiersFromCandidates:candidates ?: @[] maxDepth:maxDepth];
|
|
80
|
+
NSDictionary *deepExtension = [self extendSnapshotFrontiers:frontiers
|
|
81
|
+
axClient:axClient
|
|
82
|
+
attributes:attributes
|
|
83
|
+
maxDepth:maxDepth
|
|
84
|
+
maxNodes:maxNodes
|
|
85
|
+
nodeCount:&nodeCount
|
|
86
|
+
truncated:&truncated
|
|
87
|
+
callsAllowed:deepExtensionCallLimit
|
|
88
|
+
deadline:deadline];
|
|
89
|
+
|
|
90
|
+
NSMutableDictionary *response = [NSMutableDictionary dictionaryWithDictionary:@{
|
|
120
91
|
RunnerAXSnapshotOkKey: @YES,
|
|
121
92
|
RunnerAXSnapshotRootKey: rootNode,
|
|
122
93
|
RunnerAXSnapshotTruncatedKey: @(truncated),
|
|
123
|
-
};
|
|
94
|
+
}];
|
|
95
|
+
if (nil != deepExtension) {
|
|
96
|
+
response[RunnerAXSnapshotDeepExtensionKey] = deepExtension;
|
|
97
|
+
}
|
|
98
|
+
return response.copy;
|
|
124
99
|
} @catch (NSException *exception) {
|
|
125
100
|
return [self failure:exception.reason ?: exception.name ?: @"AX snapshot bridge exception"];
|
|
126
101
|
}
|
|
127
102
|
}
|
|
128
103
|
|
|
104
|
+
/// Chains element-rooted snapshot requests from depth-capped frontier nodes.
|
|
105
|
+
/// The AX server's depth limit is per-request (kAXErrorIllegalArgument above a
|
|
106
|
+
/// tree-size-dependent value), so re-rooting the SAME request at a frontier
|
|
107
|
+
/// node's live accessibility element reaches content the app-rooted request
|
|
108
|
+
/// could not, without ever passing a larger depth parameter. Bounded by call
|
|
109
|
+
/// count, the shared node budget, and the capture-plan deadline.
|
|
110
|
+
+ (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray<RunnerAXSnapshotFrontier *> *)frontiers
|
|
111
|
+
axClient:(id)axClient
|
|
112
|
+
attributes:(NSArray *)attributes
|
|
113
|
+
maxDepth:(NSInteger)maxDepth
|
|
114
|
+
maxNodes:(NSInteger)maxNodes
|
|
115
|
+
nodeCount:(NSInteger *)nodeCount
|
|
116
|
+
truncated:(BOOL *)truncated
|
|
117
|
+
callsAllowed:(NSInteger)callsAllowed
|
|
118
|
+
deadline:(nullable NSDate *)deadline
|
|
119
|
+
{
|
|
120
|
+
if (callsAllowed <= 0 || frontiers.count == 0) {
|
|
121
|
+
return nil;
|
|
122
|
+
}
|
|
123
|
+
NSInteger callsUsed = 0;
|
|
124
|
+
NSInteger nodesAdded = 0;
|
|
125
|
+
// A frontier whose live element vanished (list churn between serialization
|
|
126
|
+
// and extension) or whose re-rooted request failed leaves its subtree
|
|
127
|
+
// unresolved — it must count as missed, never as drained, or an all-miss
|
|
128
|
+
// extension would present a capped capture as complete.
|
|
129
|
+
NSInteger missedFrontiers = 0;
|
|
130
|
+
while (frontiers.count > 0) {
|
|
131
|
+
if (callsUsed >= callsAllowed || *nodeCount >= maxNodes
|
|
132
|
+
|| (nil != deadline && deadline.timeIntervalSinceNow <= 0)) {
|
|
133
|
+
*truncated = YES;
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
RunnerAXSnapshotFrontier *frontier = frontiers.firstObject;
|
|
137
|
+
[frontiers removeObjectAtIndex:0];
|
|
138
|
+
id element = [self accessibilityElementForSnapshot:frontier.snapshot];
|
|
139
|
+
if (nil == element) {
|
|
140
|
+
missedFrontiers += 1;
|
|
141
|
+
NSLog(@"AGENT_DEVICE_RUNNER_PRIVATE_AX_DEEP_EXTENSION_MISS=element unavailable");
|
|
142
|
+
continue;
|
|
143
|
+
}
|
|
144
|
+
callsUsed += 1;
|
|
145
|
+
NSError *error = nil;
|
|
146
|
+
id subRoot = [self requestSnapshotFromClient:axClient
|
|
147
|
+
target:element
|
|
148
|
+
attributes:attributes
|
|
149
|
+
maxDepth:maxDepth
|
|
150
|
+
maxNodes:maxNodes - *nodeCount
|
|
151
|
+
error:&error];
|
|
152
|
+
if (nil == subRoot) {
|
|
153
|
+
missedFrontiers += 1;
|
|
154
|
+
NSLog(@"AGENT_DEVICE_RUNNER_PRIVATE_AX_DEEP_EXTENSION_MISS=%@",
|
|
155
|
+
error.localizedDescription ?: @"nil subtree");
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
// The re-rooted snapshot's root IS the frontier element captured again;
|
|
159
|
+
// splice only its children so the frontier node is not duplicated.
|
|
160
|
+
NSInteger before = *nodeCount;
|
|
161
|
+
NSMutableArray<RunnerAXSnapshotFrontier *> *subCandidates = [NSMutableArray array];
|
|
162
|
+
NSMutableArray *children = [NSMutableArray array];
|
|
163
|
+
for (id child in [self childrenForSnapshot:subRoot]) {
|
|
164
|
+
NSMutableDictionary *childNode = [self dictionaryForSnapshot:child
|
|
165
|
+
depth:1
|
|
166
|
+
maxDepth:maxDepth
|
|
167
|
+
maxNodes:maxNodes
|
|
168
|
+
nodeCount:nodeCount
|
|
169
|
+
truncated:truncated
|
|
170
|
+
frontiers:subCandidates];
|
|
171
|
+
if (nil != childNode) {
|
|
172
|
+
[children addObject:childNode];
|
|
173
|
+
}
|
|
174
|
+
if (*nodeCount >= maxNodes) {
|
|
175
|
+
*truncated = YES;
|
|
176
|
+
break;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
frontier.node[@"children"] = children;
|
|
180
|
+
nodesAdded += *nodeCount - before;
|
|
181
|
+
[frontiers addObjectsFromArray:[self cappedFrontiersFromCandidates:subCandidates
|
|
182
|
+
maxDepth:maxDepth]];
|
|
183
|
+
}
|
|
184
|
+
NSLog(@"AGENT_DEVICE_RUNNER_PRIVATE_AX_DEEP_EXTENSION calls=%ld nodes=%ld pending=%ld missed=%ld",
|
|
185
|
+
(long)callsUsed, (long)nodesAdded, (long)frontiers.count, (long)missedFrontiers);
|
|
186
|
+
return @{
|
|
187
|
+
RunnerAXSnapshotDeepExtensionCallsKey: @(callsUsed),
|
|
188
|
+
RunnerAXSnapshotDeepExtensionNodesAddedKey: @(nodesAdded),
|
|
189
|
+
RunnerAXSnapshotDeepExtensionPendingKey: @(frontiers.count),
|
|
190
|
+
RunnerAXSnapshotDeepExtensionMissedKey: @(missedFrontiers),
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/// Keeps only candidates at the deepest observed level, and only when that
|
|
195
|
+
/// level sits at the request's cap (the server emits maxDepth node LEVELS, so
|
|
196
|
+
/// the deepest possible level is maxDepth - 1). A tree that ended naturally
|
|
197
|
+
/// above the cap yields no frontiers and costs nothing.
|
|
198
|
+
+ (NSMutableArray<RunnerAXSnapshotFrontier *> *)cappedFrontiersFromCandidates:
|
|
199
|
+
(NSArray<RunnerAXSnapshotFrontier *> *)candidates
|
|
200
|
+
maxDepth:(NSInteger)maxDepth
|
|
201
|
+
{
|
|
202
|
+
NSMutableArray<RunnerAXSnapshotFrontier *> *frontiers = [NSMutableArray array];
|
|
203
|
+
NSInteger deepest = -1;
|
|
204
|
+
for (RunnerAXSnapshotFrontier *candidate in candidates) {
|
|
205
|
+
deepest = MAX(deepest, candidate.depth);
|
|
206
|
+
}
|
|
207
|
+
if (deepest < maxDepth - 1) {
|
|
208
|
+
return frontiers;
|
|
209
|
+
}
|
|
210
|
+
for (RunnerAXSnapshotFrontier *candidate in candidates) {
|
|
211
|
+
if (candidate.depth == deepest) {
|
|
212
|
+
[frontiers addObject:candidate];
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
return frontiers;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
+ (nullable id)requestSnapshotFromClient:(id)axClient
|
|
219
|
+
target:(id)target
|
|
220
|
+
attributes:(NSArray *)attributes
|
|
221
|
+
maxDepth:(NSInteger)maxDepth
|
|
222
|
+
maxNodes:(NSInteger)maxNodes
|
|
223
|
+
error:(NSError **)error
|
|
224
|
+
{
|
|
225
|
+
SEL requestSelector = NSSelectorFromString(@"requestSnapshotForElement:attributes:parameters:error:");
|
|
226
|
+
if (![axClient respondsToSelector:requestSelector]) {
|
|
227
|
+
if (NULL != error) {
|
|
228
|
+
*error = [NSError errorWithDomain:@"agent-device.runner"
|
|
229
|
+
code:1
|
|
230
|
+
userInfo:@{
|
|
231
|
+
NSLocalizedDescriptionKey:
|
|
232
|
+
@"AX client does not support requestSnapshotForElement"
|
|
233
|
+
}];
|
|
234
|
+
}
|
|
235
|
+
return nil;
|
|
236
|
+
}
|
|
237
|
+
NSMutableDictionary *parameters = [NSMutableDictionary dictionary];
|
|
238
|
+
id defaults = [self objectFrom:axClient selectorName:@"defaultParameters"];
|
|
239
|
+
if ([defaults isKindOfClass:NSDictionary.class]) {
|
|
240
|
+
[parameters addEntriesFromDictionary:(NSDictionary *)defaults];
|
|
241
|
+
}
|
|
242
|
+
parameters[@"maxDepth"] = @(MAX(0, maxDepth));
|
|
243
|
+
parameters[@"maxChildren"] = @(MAX(1, maxNodes));
|
|
244
|
+
parameters[@"maxArrayCount"] = @(MAX(1, maxNodes));
|
|
245
|
+
parameters[@"traverseFromParentsToChildren"] = @YES;
|
|
246
|
+
|
|
247
|
+
RunnerAXSnapshotMsgSend send = (RunnerAXSnapshotMsgSend)objc_msgSend;
|
|
248
|
+
id result = send(axClient, requestSelector, target, attributes, parameters.copy, error);
|
|
249
|
+
if (nil == result) {
|
|
250
|
+
return nil;
|
|
251
|
+
}
|
|
252
|
+
id root = nil;
|
|
253
|
+
@try {
|
|
254
|
+
root = [result valueForKey:@"_rootElementSnapshot"];
|
|
255
|
+
} @catch (NSException *exception) {
|
|
256
|
+
root = nil;
|
|
257
|
+
}
|
|
258
|
+
return root ?: result;
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
+ (NSArray *)snapshotAttributes
|
|
262
|
+
{
|
|
263
|
+
NSArray<NSString *> *keyPaths = @[
|
|
264
|
+
@"elementType",
|
|
265
|
+
@"identifier",
|
|
266
|
+
@"label",
|
|
267
|
+
@"value",
|
|
268
|
+
@"frame",
|
|
269
|
+
@"enabled",
|
|
270
|
+
@"selected",
|
|
271
|
+
@"hasFocus",
|
|
272
|
+
@"children",
|
|
273
|
+
];
|
|
274
|
+
// The AX server expects real accessibility attribute identifiers, not snapshot keypath
|
|
275
|
+
// strings; passing raw keypaths silently drops attributes it does not recognize (frame
|
|
276
|
+
// came back zeroed). XCElementSnapshot owns the keypath -> AX attribute mapping.
|
|
277
|
+
NSArray *attributes = keyPaths;
|
|
278
|
+
Class snapshotClass = NSClassFromString(@"XCElementSnapshot");
|
|
279
|
+
SEL mapSelector = NSSelectorFromString(@"axAttributesForElementSnapshotKeyPaths:isMacOS:");
|
|
280
|
+
if ([snapshotClass respondsToSelector:mapSelector]) {
|
|
281
|
+
typedef id (*RunnerAXMapMsgSend)(id, SEL, id, BOOL);
|
|
282
|
+
RunnerAXMapMsgSend mapSend = (RunnerAXMapMsgSend)objc_msgSend;
|
|
283
|
+
id mapped = mapSend(snapshotClass, mapSelector, keyPaths, NO);
|
|
284
|
+
if ([mapped isKindOfClass:NSSet.class]) {
|
|
285
|
+
mapped = [(NSSet *)mapped allObjects];
|
|
286
|
+
}
|
|
287
|
+
if ([mapped isKindOfClass:NSArray.class] && [(NSArray *)mapped count] > 0) {
|
|
288
|
+
// The mapper expands keypaths with extra attributes (automation type, window display
|
|
289
|
+
// id, base type) that are disproportionately expensive for the AX server to compute
|
|
290
|
+
// on large React Native trees. Keep only the attributes we actually consume.
|
|
291
|
+
NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"Frame",
|
|
292
|
+
@"Enabled", @"Selected", @"Focus" ];
|
|
293
|
+
NSMutableArray *filtered = [NSMutableArray array];
|
|
294
|
+
for (id attribute in (NSArray *)mapped) {
|
|
295
|
+
NSString *name = [attribute description];
|
|
296
|
+
for (NSString *suffix in needed) {
|
|
297
|
+
if ([name hasSuffix:suffix]) {
|
|
298
|
+
[filtered addObject:attribute];
|
|
299
|
+
break;
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
attributes = filtered.count > 0 ? filtered : mapped;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return attributes;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
+ (nullable id)accessibilityElementForSnapshot:(id)snapshot
|
|
310
|
+
{
|
|
311
|
+
id element = nil;
|
|
312
|
+
@try {
|
|
313
|
+
element = [snapshot valueForKey:@"accessibilityElement"];
|
|
314
|
+
} @catch (NSException *exception) {
|
|
315
|
+
element = nil;
|
|
316
|
+
}
|
|
317
|
+
return element;
|
|
318
|
+
}
|
|
319
|
+
|
|
129
320
|
+ (NSDictionary<NSString *, id> *)failure:(NSString *)message
|
|
130
321
|
{
|
|
131
322
|
return @{
|
|
@@ -186,12 +377,13 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
186
377
|
return nil;
|
|
187
378
|
}
|
|
188
379
|
|
|
189
|
-
+ (nullable
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
380
|
+
+ (nullable NSMutableDictionary *)dictionaryForSnapshot:(id)snapshot
|
|
381
|
+
depth:(NSInteger)depth
|
|
382
|
+
maxDepth:(NSInteger)maxDepth
|
|
383
|
+
maxNodes:(NSInteger)maxNodes
|
|
384
|
+
nodeCount:(NSInteger *)nodeCount
|
|
385
|
+
truncated:(BOOL *)truncated
|
|
386
|
+
frontiers:(NSMutableArray<RunnerAXSnapshotFrontier *> *)frontiers
|
|
195
387
|
{
|
|
196
388
|
if (nil == snapshot || *nodeCount >= maxNodes) {
|
|
197
389
|
*truncated = YES;
|
|
@@ -212,12 +404,13 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
212
404
|
NSMutableArray *children = [NSMutableArray array];
|
|
213
405
|
if (depth < maxDepth) {
|
|
214
406
|
for (id child in [self childrenForSnapshot:snapshot]) {
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
407
|
+
NSMutableDictionary *childNode = [self dictionaryForSnapshot:child
|
|
408
|
+
depth:depth + 1
|
|
409
|
+
maxDepth:maxDepth
|
|
410
|
+
maxNodes:maxNodes
|
|
411
|
+
nodeCount:nodeCount
|
|
412
|
+
truncated:truncated
|
|
413
|
+
frontiers:frontiers];
|
|
221
414
|
if (nil != childNode) {
|
|
222
415
|
[children addObject:childNode];
|
|
223
416
|
}
|
|
@@ -227,8 +420,21 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
227
420
|
}
|
|
228
421
|
}
|
|
229
422
|
}
|
|
423
|
+
if (children.count == 0 && nil != frontiers && depth >= maxDepth - 1) {
|
|
424
|
+
// Childless node at the cap boundary: either a real leaf or a branch whose
|
|
425
|
+
// children the AX server withheld. The two are indistinguishable here, so
|
|
426
|
+
// boundary childless nodes become candidates; the caller keeps only the
|
|
427
|
+
// deepest observed level, where every capped branch necessarily ends.
|
|
428
|
+
// Shallower childless nodes are provably real leaves and are never
|
|
429
|
+
// collected.
|
|
430
|
+
RunnerAXSnapshotFrontier *frontier = [[RunnerAXSnapshotFrontier alloc] init];
|
|
431
|
+
frontier.snapshot = snapshot;
|
|
432
|
+
frontier.node = result;
|
|
433
|
+
frontier.depth = depth;
|
|
434
|
+
[frontiers addObject:frontier];
|
|
435
|
+
}
|
|
230
436
|
result[@"children"] = children;
|
|
231
|
-
return result
|
|
437
|
+
return result;
|
|
232
438
|
}
|
|
233
439
|
|
|
234
440
|
+ (NSArray *)childrenForSnapshot:(id)snapshot
|