agent-device 0.20.5 → 0.20.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (100) hide show
  1. package/README.md +2 -0
  2. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.apk → agent-device-android-ime-helper-0.20.6.apk} +0 -0
  3. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.6.apk.sha256 +1 -0
  4. package/android/ime-helper/dist/{agent-device-android-ime-helper-0.20.5.manifest.json → agent-device-android-ime-helper-0.20.6.manifest.json} +4 -4
  5. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.apk → agent-device-android-snapshot-helper-0.20.6.apk} +0 -0
  6. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.6.apk.sha256 +1 -0
  7. package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.20.5.manifest.json → agent-device-android-snapshot-helper-0.20.6.manifest.json} +6 -6
  8. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunner/AgentDeviceRunnerApp.m +28 -0
  9. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/AgentDeviceRunnerUITests-Bridging-Header.h +1 -0
  10. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +41 -1
  11. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +298 -92
  12. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +126 -187
  13. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.h +32 -0
  14. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedTextEntry.m +212 -0
  15. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +118 -8
  16. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +134 -27
  17. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Keyboard.swift +23 -5
  18. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +10 -0
  19. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+LifecycleCacheTests.swift +0 -67
  20. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +11 -1
  21. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +2 -22
  22. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +48 -11
  23. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotTraversalIdentityTests.swift +0 -20
  24. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedTextEntry.swift +177 -0
  25. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntry.swift +105 -448
  26. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextEntryPolicyTests.swift +4 -0
  27. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TextTyping.swift +587 -0
  28. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +18 -0
  29. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +72 -0
  30. package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +89 -0
  31. package/dist/src/agent-device-client.js +2 -2
  32. package/dist/src/android.js +1 -1
  33. package/dist/src/app-log-request-scope.js +1 -1
  34. package/dist/src/app-resolution.js +1 -1
  35. package/dist/src/apps.js +5 -5
  36. package/dist/src/cli-config.js +1 -1
  37. package/dist/src/cli-help.js +34 -24
  38. package/dist/src/cli-runner.js +1 -1
  39. package/dist/src/cli.js +15 -15
  40. package/dist/src/command-metadata.js +1 -1
  41. package/dist/src/command-schema.js +3 -3
  42. package/dist/src/context.js +1 -1
  43. package/dist/src/debug-symbols.js +1 -1
  44. package/dist/src/find.js +1 -1
  45. package/dist/src/finders.js +1 -1
  46. package/dist/src/generic-settle.js +1 -0
  47. package/dist/src/generic.js +1 -1
  48. package/dist/src/ime-lifecycle.js +3 -3
  49. package/dist/src/index.d.ts +27 -7
  50. package/dist/src/input-actions.js +4 -4
  51. package/dist/src/interaction-runtime.js +1 -0
  52. package/dist/src/interaction-snapshot.js +1 -1
  53. package/dist/src/interaction-touch-reference-frame.js +1 -0
  54. package/dist/src/interaction.js +1 -1
  55. package/dist/src/interactor.js +2 -2
  56. package/dist/src/interactor2.js +1 -1
  57. package/dist/src/internal/bin.js +2 -2
  58. package/dist/src/internal/daemon.js +15 -15
  59. package/dist/src/lease-context.js +1 -1
  60. package/dist/src/lease.js +1 -1
  61. package/dist/src/limrun.d.ts +23 -2
  62. package/dist/src/linux.js +1 -1
  63. package/dist/src/process-exit.js +1 -0
  64. package/dist/src/react-native.js +1 -1
  65. package/dist/src/record-trace.js +1 -1
  66. package/dist/src/register-builtins.js +1 -1
  67. package/dist/src/registry.js +25 -24
  68. package/dist/src/remote-config.js +1 -1
  69. package/dist/src/remote-config2.js +1 -1
  70. package/dist/src/reporting.js +5 -5
  71. package/dist/src/runner-client.js +1 -1
  72. package/dist/src/runner-disposal.js +1 -1
  73. package/dist/src/runner-lease.js +1 -1
  74. package/dist/src/runtime.js +3 -3
  75. package/dist/src/screenshot-geometry.js +1 -1
  76. package/dist/src/screenshot-result.js +4 -4
  77. package/dist/src/sdk-android-adb.d.ts +20 -2
  78. package/dist/src/sdk-android-adb.js +1 -1
  79. package/dist/src/sdk-batch-runner.js +3 -3
  80. package/dist/src/sdk-contracts.d.ts +17 -2
  81. package/dist/src/sdk-finders.d.ts +4 -20
  82. package/dist/src/sdk-finders.js +1 -1
  83. package/dist/src/{interaction.d.ts → sdk-selectors.d.ts} +98 -3
  84. package/dist/src/sdk-selectors.js +1 -1
  85. package/dist/src/selector-runtime.js +1 -1
  86. package/dist/src/selector-vocabulary.js +1 -1
  87. package/dist/src/selectors.d.ts +1 -46
  88. package/dist/src/selectors.js +1 -1
  89. package/dist/src/server.js +3 -3
  90. package/dist/src/session-target-evidence.js +1 -1
  91. package/dist/src/session.js +9 -9
  92. package/dist/src/snapshot2.js +1 -1
  93. package/dist/src/text-surface.js +1 -0
  94. package/dist/src/viewport-dimension.js +13 -12
  95. package/dist/src/web.js +1 -1
  96. package/package.json +41 -35
  97. package/server.json +3 -3
  98. package/android/ime-helper/dist/agent-device-android-ime-helper-0.20.5.apk.sha256 +0 -1
  99. package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.20.5.apk.sha256 +0 -1
  100. package/dist/apple/runner/AgentDeviceRunner/RecordingScripts/recording-resize.swift +0 -225
package/README.md CHANGED
@@ -61,6 +61,8 @@ agent-device close
61
61
 
62
62
  Use refs only from the latest output. Do not assume an earlier `@eN` still identifies the same element. After a command with `--settle`, use the refs in its diff. Take another snapshot only if the diff omits what you need.
63
63
 
64
+ `--settle` works the same way on `scroll` and `back`, so scroll-then-observe and back-then-observe are one call too.
65
+
64
66
  Snapshots use the app's accessibility tree. Clear labels, roles, and test IDs make agent runs more reliable. Use screenshots and videos as evidence or when accessibility data is poor. Use refs and selectors for actions and assertions when you can.
65
67
 
66
68
  ![agent-device demo showing Codex using agent-device to create a new contact in the iOS Contacts app from a simple prompt](./website/docs/public/agent-device-contacts.gif)
@@ -0,0 +1 @@
1
+ 7f719c5e1698e827948a9de22bf31aab065d7f2cb7d17731793b586fd3f88f4f agent-device-android-ime-helper-0.20.6.apk
@@ -1,10 +1,10 @@
1
1
  {
2
2
  "name": "android-ime-helper",
3
- "version": "0.20.5",
4
- "assetName": "agent-device-android-ime-helper-0.20.5.apk",
5
- "sha256": "1f5a95fcf0c6d668a11453644b2e897147932b528c1aa6c74883eaf9d73a5e7f",
3
+ "version": "0.20.6",
4
+ "assetName": "agent-device-android-ime-helper-0.20.6.apk",
5
+ "sha256": "7f719c5e1698e827948a9de22bf31aab065d7f2cb7d17731793b586fd3f88f4f",
6
6
  "packageName": "com.callstack.agentdevice.imehelper",
7
- "versionCode": 20005,
7
+ "versionCode": 20006,
8
8
  "serviceComponent": "com.callstack.agentdevice.imehelper/.TestInputMethodService",
9
9
  "broadcastProtocol": "android-ime-helper-v1"
10
10
  }
@@ -0,0 +1 @@
1
+ 696f8e518bbc2ad66225cb80f2e9e70d2b19b75d382918edd8fce3009200e434 agent-device-android-snapshot-helper-0.20.6.apk
@@ -1,13 +1,13 @@
1
1
  {
2
2
  "name": "android-snapshot-helper",
3
- "version": "0.20.5",
4
- "releaseTag": "v0.20.5",
5
- "assetName": "agent-device-android-snapshot-helper-0.20.5.apk",
3
+ "version": "0.20.6",
4
+ "releaseTag": "v0.20.6",
5
+ "assetName": "agent-device-android-snapshot-helper-0.20.6.apk",
6
6
  "apkUrl": null,
7
- "sha256": "ce370af9f1c97e7c812cb93772f864596731c10f3d0b7cadc1c88b772a062970",
8
- "checksumName": "agent-device-android-snapshot-helper-0.20.5.apk.sha256",
7
+ "sha256": "696f8e518bbc2ad66225cb80f2e9e70d2b19b75d382918edd8fce3009200e434",
8
+ "checksumName": "agent-device-android-snapshot-helper-0.20.6.apk.sha256",
9
9
  "packageName": "com.callstack.agentdevice.snapshothelper",
10
- "versionCode": 20005,
10
+ "versionCode": 20006,
11
11
  "instrumentationRunner": "com.callstack.agentdevice.snapshothelper/.SnapshotInstrumentation",
12
12
  "minSdk": 23,
13
13
  "targetSdk": 36,
@@ -62,6 +62,13 @@ int main(int argc, const char *argv[]) {
62
62
 
63
63
  @implementation AgentDeviceRunnerViewController
64
64
 
65
+ - (void)agentDeviceTextEntryDidChange:(UITextField *)textField {
66
+ if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-disappear-after-input"] &&
67
+ textField.text.length > 0) {
68
+ [textField removeFromSuperview];
69
+ }
70
+ }
71
+
65
72
  - (void)viewDidLoad {
66
73
  [super viewDidLoad];
67
74
 
@@ -78,6 +85,27 @@ int main(int argc, const char *argv[]) {
78
85
  [label.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
79
86
  [label.centerYAnchor constraintEqualToAnchor:self.view.centerYAnchor],
80
87
  ]];
88
+
89
+ // Keep the fixture behind a launch argument so normal runner snapshots remain unchanged.
90
+ #if TARGET_OS_IOS
91
+ if ([NSProcessInfo.processInfo.arguments containsObject:@"--agent-device-text-entry-regression"]) {
92
+ UITextField *textField = [[UITextField alloc] init];
93
+ textField.accessibilityIdentifier = @"agent-device-hardware-keyboard-input";
94
+ textField.borderStyle = UITextBorderStyleRoundedRect;
95
+ textField.inputView = [[UIView alloc] initWithFrame:CGRectMake(0, 0, 1, 1)];
96
+ [textField addTarget:self
97
+ action:@selector(agentDeviceTextEntryDidChange:)
98
+ forControlEvents:UIControlEventEditingChanged];
99
+ textField.translatesAutoresizingMaskIntoConstraints = NO;
100
+ [self.view addSubview:textField];
101
+ [NSLayoutConstraint activateConstraints:@[
102
+ [textField.centerXAnchor constraintEqualToAnchor:self.view.centerXAnchor],
103
+ [textField.topAnchor constraintEqualToAnchor:label.bottomAnchor constant:24],
104
+ [textField.widthAnchor constraintEqualToConstant:240],
105
+ [textField.heightAnchor constraintEqualToConstant:44],
106
+ ]];
107
+ }
108
+ #endif
81
109
  }
82
110
 
83
111
  @end
@@ -1,3 +1,4 @@
1
1
  #import "RunnerObjCExceptionCatcher.h"
2
2
  #import "RunnerAXSnapshotBridge.h"
3
3
  #import "RunnerSynthesizedGesture.h"
4
+ #import "RunnerSynthesizedTextEntry.h"
@@ -3,14 +3,54 @@
3
3
 
4
4
  NS_ASSUME_NONNULL_BEGIN
5
5
 
6
+ /// Keys of the `deepExtension` dictionary in the snapshot response, shared
7
+ /// with the Swift consumer so the response shape has one spelling.
8
+ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionKey;
9
+ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionCallsKey;
10
+ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionNodesAddedKey;
11
+ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionPendingKey;
12
+ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionMissedKey;
13
+
14
+ /// A depth-capped childless node awaiting an element-rooted follow-up request.
15
+ /// Public so the runner unit bundle can drive the extension's miss paths with
16
+ /// fabricated snapshots — the executed contract that missed frontiers are
17
+ /// counted, never silently dropped.
18
+ @interface RunnerAXSnapshotFrontier : NSObject
19
+ @property(nonatomic, strong, nullable) id snapshot;
20
+ @property(nonatomic, strong, nullable) NSMutableDictionary *node;
21
+ @property(nonatomic, assign) NSInteger depth;
22
+ @end
23
+
6
24
  @interface RunnerAXSnapshotBridge : NSObject
7
25
 
26
+ /// Captures the app's accessibility tree, then chains element-rooted follow-up
27
+ /// requests from depth-capped frontier nodes (the AX depth limit is
28
+ /// per-request, so re-rooting reaches content the app-rooted request could
29
+ /// not). The response gains a `deepExtension` {calls, nodesAdded,
30
+ /// pendingFrontiers, missedFrontiers} dictionary when any frontier existed and
31
+ /// the call limit allowed extension. Pass 0 to disable extension entirely.
8
32
  + (NSDictionary<NSString *, id> *)snapshotTreeForApplication:(XCUIApplication *)application
9
33
  maxDepth:(NSInteger)maxDepth
10
- maxNodes:(NSInteger)maxNodes;
34
+ maxNodes:(NSInteger)maxNodes
35
+ deepExtensionCallLimit:(NSInteger)deepExtensionCallLimit
36
+ deadline:(nullable NSDate *)deadline;
11
37
 
12
38
  + (NSInteger)processIdentifierForApplication:(XCUIApplication *)application;
13
39
 
40
+ /// The frontier-extension loop, exposed for the runner unit bundle: a frontier
41
+ /// whose element vanished or whose re-rooted request fails must count as
42
+ /// missed (see the deepExtension keys above) — an all-miss extension reporting
43
+ /// itself drained would present a capped capture as complete.
44
+ + (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray<RunnerAXSnapshotFrontier *> *)frontiers
45
+ axClient:(id)axClient
46
+ attributes:(NSArray *)attributes
47
+ maxDepth:(NSInteger)maxDepth
48
+ maxNodes:(NSInteger)maxNodes
49
+ nodeCount:(NSInteger *)nodeCount
50
+ truncated:(BOOL *)truncated
51
+ callsAllowed:(NSInteger)callsAllowed
52
+ deadline:(nullable NSDate *)deadline;
53
+
14
54
  @end
15
55
 
16
56
  NS_ASSUME_NONNULL_END
@@ -8,15 +8,32 @@ static NSString *const RunnerAXSnapshotErrorKey = @"error";
8
8
  static NSString *const RunnerAXSnapshotRootKey = @"root";
9
9
  static NSString *const RunnerAXSnapshotTruncatedKey = @"truncated";
10
10
 
11
+ NSString *const RunnerAXSnapshotDeepExtensionKey = @"deepExtension";
12
+ NSString *const RunnerAXSnapshotDeepExtensionCallsKey = @"calls";
13
+ NSString *const RunnerAXSnapshotDeepExtensionNodesAddedKey = @"nodesAdded";
14
+ NSString *const RunnerAXSnapshotDeepExtensionPendingKey = @"pendingFrontiers";
15
+ NSString *const RunnerAXSnapshotDeepExtensionMissedKey = @"missedFrontiers";
16
+
11
17
  typedef id (*RunnerAXObjectMsgSend)(id, SEL);
12
18
  typedef NSInteger (*RunnerAXIntegerMsgSend)(id, SEL);
13
19
  typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
14
20
 
21
+ // A childless serialized node remembered with its depth (interface in the
22
+ // header for the unit bundle). Nodes at the deepest observed level of a
23
+ // depth-capped capture are the truncation frontier: the AX server withholds
24
+ // children uniformly at its per-request cap, so every capped branch ends at
25
+ // the same level (observed one BELOW the requested maxDepth — the server
26
+ // counts node levels, not edges).
27
+ @implementation RunnerAXSnapshotFrontier
28
+ @end
29
+
15
30
  @implementation RunnerAXSnapshotBridge
16
31
 
17
32
  + (NSDictionary<NSString *, id> *)snapshotTreeForApplication:(XCUIApplication *)application
18
33
  maxDepth:(NSInteger)maxDepth
19
34
  maxNodes:(NSInteger)maxNodes
35
+ deepExtensionCallLimit:(NSInteger)deepExtensionCallLimit
36
+ deadline:(nullable NSDate *)deadline
20
37
  {
21
38
  @try {
22
39
  id axClient = [self objectFrom:XCUIDevice.sharedDevice selectorName:@"accessibilityInterface"];
@@ -29,103 +46,277 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
29
46
  return [self failure:@"Could not match active AX application for XCTest application"];
30
47
  }
31
48
 
32
- NSMutableDictionary *parameters = [NSMutableDictionary dictionary];
33
- id defaults = [self objectFrom:axClient selectorName:@"defaultParameters"];
34
- if ([defaults isKindOfClass:NSDictionary.class]) {
35
- [parameters addEntriesFromDictionary:(NSDictionary *)defaults];
36
- }
37
- parameters[@"maxDepth"] = @(MAX(0, maxDepth));
38
- parameters[@"maxChildren"] = @(MAX(1, maxNodes));
39
- parameters[@"maxArrayCount"] = @(MAX(1, maxNodes));
40
- parameters[@"traverseFromParentsToChildren"] = @YES;
41
-
42
- SEL requestSelector = NSSelectorFromString(@"requestSnapshotForElement:attributes:parameters:error:");
43
- if (![axClient respondsToSelector:requestSelector]) {
44
- return [self failure:@"AX client does not support requestSnapshotForElement"];
45
- }
46
-
49
+ NSArray *attributes = [self snapshotAttributes];
47
50
  NSError *error = nil;
48
- NSArray<NSString *> *keyPaths = @[
49
- @"elementType",
50
- @"identifier",
51
- @"label",
52
- @"value",
53
- @"frame",
54
- @"enabled",
55
- @"selected",
56
- @"hasFocus",
57
- @"children",
58
- ];
59
- // The AX server expects real accessibility attribute identifiers, not snapshot keypath
60
- // strings; passing raw keypaths silently drops attributes it does not recognize (frame
61
- // came back zeroed). XCElementSnapshot owns the keypath -> AX attribute mapping.
62
- NSArray *attributes = keyPaths;
63
- Class snapshotClass = NSClassFromString(@"XCElementSnapshot");
64
- SEL mapSelector = NSSelectorFromString(@"axAttributesForElementSnapshotKeyPaths:isMacOS:");
65
- if ([snapshotClass respondsToSelector:mapSelector]) {
66
- typedef id (*RunnerAXMapMsgSend)(id, SEL, id, BOOL);
67
- RunnerAXMapMsgSend mapSend = (RunnerAXMapMsgSend)objc_msgSend;
68
- id mapped = mapSend(snapshotClass, mapSelector, keyPaths, NO);
69
- if ([mapped isKindOfClass:NSSet.class]) {
70
- mapped = [(NSSet *)mapped allObjects];
71
- }
72
- if ([mapped isKindOfClass:NSArray.class] && [(NSArray *)mapped count] > 0) {
73
- // The mapper expands keypaths with extra attributes (automation type, window display
74
- // id, base type) that are disproportionately expensive for the AX server to compute
75
- // on large React Native trees. Keep only the attributes we actually consume.
76
- NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"Frame",
77
- @"Enabled", @"Selected", @"Focus" ];
78
- NSMutableArray *filtered = [NSMutableArray array];
79
- for (id attribute in (NSArray *)mapped) {
80
- NSString *name = [attribute description];
81
- for (NSString *suffix in needed) {
82
- if ([name hasSuffix:suffix]) {
83
- [filtered addObject:attribute];
84
- break;
85
- }
86
- }
87
- }
88
- attributes = filtered.count > 0 ? filtered : mapped;
89
- }
90
- }
91
- RunnerAXSnapshotMsgSend send = (RunnerAXSnapshotMsgSend)objc_msgSend;
92
- id result = send(axClient, requestSelector, target, attributes, parameters.copy, &error);
93
- if (nil == result) {
94
- return [self failure:error.localizedDescription ?: @"AX snapshot request returned nil"];
95
- }
96
-
97
- id root = nil;
98
- @try {
99
- root = [result valueForKey:@"_rootElementSnapshot"];
100
- } @catch (NSException *exception) {
101
- root = nil;
102
- }
51
+ id root = [self requestSnapshotFromClient:axClient
52
+ target:target
53
+ attributes:attributes
54
+ maxDepth:maxDepth
55
+ maxNodes:maxNodes
56
+ error:&error];
103
57
  if (nil == root) {
104
- root = result;
58
+ return [self failure:error.localizedDescription ?: @"AX snapshot request returned nil"];
105
59
  }
106
60
 
107
61
  BOOL truncated = NO;
108
62
  NSInteger nodeCount = 0;
109
- NSDictionary *rootNode = [self dictionaryForSnapshot:root
110
- depth:0
111
- maxDepth:maxDepth
112
- maxNodes:maxNodes
113
- nodeCount:&nodeCount
114
- truncated:&truncated];
63
+ // nil disables candidate collection entirely — exact --depth captures pay
64
+ // zero extension bookkeeping.
65
+ NSMutableArray<RunnerAXSnapshotFrontier *> *candidates =
66
+ deepExtensionCallLimit > 0 ? [NSMutableArray array] : nil;
67
+ NSMutableDictionary *rootNode = [self dictionaryForSnapshot:root
68
+ depth:0
69
+ maxDepth:maxDepth
70
+ maxNodes:maxNodes
71
+ nodeCount:&nodeCount
72
+ truncated:&truncated
73
+ frontiers:candidates];
115
74
  if (nil == rootNode) {
116
75
  return [self failure:@"AX snapshot root could not be serialized"];
117
76
  }
118
77
 
119
- return @{
78
+ NSMutableArray<RunnerAXSnapshotFrontier *> *frontiers =
79
+ [self cappedFrontiersFromCandidates:candidates ?: @[] maxDepth:maxDepth];
80
+ NSDictionary *deepExtension = [self extendSnapshotFrontiers:frontiers
81
+ axClient:axClient
82
+ attributes:attributes
83
+ maxDepth:maxDepth
84
+ maxNodes:maxNodes
85
+ nodeCount:&nodeCount
86
+ truncated:&truncated
87
+ callsAllowed:deepExtensionCallLimit
88
+ deadline:deadline];
89
+
90
+ NSMutableDictionary *response = [NSMutableDictionary dictionaryWithDictionary:@{
120
91
  RunnerAXSnapshotOkKey: @YES,
121
92
  RunnerAXSnapshotRootKey: rootNode,
122
93
  RunnerAXSnapshotTruncatedKey: @(truncated),
123
- };
94
+ }];
95
+ if (nil != deepExtension) {
96
+ response[RunnerAXSnapshotDeepExtensionKey] = deepExtension;
97
+ }
98
+ return response.copy;
124
99
  } @catch (NSException *exception) {
125
100
  return [self failure:exception.reason ?: exception.name ?: @"AX snapshot bridge exception"];
126
101
  }
127
102
  }
128
103
 
104
+ /// Chains element-rooted snapshot requests from depth-capped frontier nodes.
105
+ /// The AX server's depth limit is per-request (kAXErrorIllegalArgument above a
106
+ /// tree-size-dependent value), so re-rooting the SAME request at a frontier
107
+ /// node's live accessibility element reaches content the app-rooted request
108
+ /// could not, without ever passing a larger depth parameter. Bounded by call
109
+ /// count, the shared node budget, and the capture-plan deadline.
110
+ + (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray<RunnerAXSnapshotFrontier *> *)frontiers
111
+ axClient:(id)axClient
112
+ attributes:(NSArray *)attributes
113
+ maxDepth:(NSInteger)maxDepth
114
+ maxNodes:(NSInteger)maxNodes
115
+ nodeCount:(NSInteger *)nodeCount
116
+ truncated:(BOOL *)truncated
117
+ callsAllowed:(NSInteger)callsAllowed
118
+ deadline:(nullable NSDate *)deadline
119
+ {
120
+ if (callsAllowed <= 0 || frontiers.count == 0) {
121
+ return nil;
122
+ }
123
+ NSInteger callsUsed = 0;
124
+ NSInteger nodesAdded = 0;
125
+ // A frontier whose live element vanished (list churn between serialization
126
+ // and extension) or whose re-rooted request failed leaves its subtree
127
+ // unresolved — it must count as missed, never as drained, or an all-miss
128
+ // extension would present a capped capture as complete.
129
+ NSInteger missedFrontiers = 0;
130
+ while (frontiers.count > 0) {
131
+ if (callsUsed >= callsAllowed || *nodeCount >= maxNodes
132
+ || (nil != deadline && deadline.timeIntervalSinceNow <= 0)) {
133
+ *truncated = YES;
134
+ break;
135
+ }
136
+ RunnerAXSnapshotFrontier *frontier = frontiers.firstObject;
137
+ [frontiers removeObjectAtIndex:0];
138
+ id element = [self accessibilityElementForSnapshot:frontier.snapshot];
139
+ if (nil == element) {
140
+ missedFrontiers += 1;
141
+ NSLog(@"AGENT_DEVICE_RUNNER_PRIVATE_AX_DEEP_EXTENSION_MISS=element unavailable");
142
+ continue;
143
+ }
144
+ callsUsed += 1;
145
+ NSError *error = nil;
146
+ id subRoot = [self requestSnapshotFromClient:axClient
147
+ target:element
148
+ attributes:attributes
149
+ maxDepth:maxDepth
150
+ maxNodes:maxNodes - *nodeCount
151
+ error:&error];
152
+ if (nil == subRoot) {
153
+ missedFrontiers += 1;
154
+ NSLog(@"AGENT_DEVICE_RUNNER_PRIVATE_AX_DEEP_EXTENSION_MISS=%@",
155
+ error.localizedDescription ?: @"nil subtree");
156
+ continue;
157
+ }
158
+ // The re-rooted snapshot's root IS the frontier element captured again;
159
+ // splice only its children so the frontier node is not duplicated.
160
+ NSInteger before = *nodeCount;
161
+ NSMutableArray<RunnerAXSnapshotFrontier *> *subCandidates = [NSMutableArray array];
162
+ NSMutableArray *children = [NSMutableArray array];
163
+ for (id child in [self childrenForSnapshot:subRoot]) {
164
+ NSMutableDictionary *childNode = [self dictionaryForSnapshot:child
165
+ depth:1
166
+ maxDepth:maxDepth
167
+ maxNodes:maxNodes
168
+ nodeCount:nodeCount
169
+ truncated:truncated
170
+ frontiers:subCandidates];
171
+ if (nil != childNode) {
172
+ [children addObject:childNode];
173
+ }
174
+ if (*nodeCount >= maxNodes) {
175
+ *truncated = YES;
176
+ break;
177
+ }
178
+ }
179
+ frontier.node[@"children"] = children;
180
+ nodesAdded += *nodeCount - before;
181
+ [frontiers addObjectsFromArray:[self cappedFrontiersFromCandidates:subCandidates
182
+ maxDepth:maxDepth]];
183
+ }
184
+ NSLog(@"AGENT_DEVICE_RUNNER_PRIVATE_AX_DEEP_EXTENSION calls=%ld nodes=%ld pending=%ld missed=%ld",
185
+ (long)callsUsed, (long)nodesAdded, (long)frontiers.count, (long)missedFrontiers);
186
+ return @{
187
+ RunnerAXSnapshotDeepExtensionCallsKey: @(callsUsed),
188
+ RunnerAXSnapshotDeepExtensionNodesAddedKey: @(nodesAdded),
189
+ RunnerAXSnapshotDeepExtensionPendingKey: @(frontiers.count),
190
+ RunnerAXSnapshotDeepExtensionMissedKey: @(missedFrontiers),
191
+ };
192
+ }
193
+
194
+ /// Keeps only candidates at the deepest observed level, and only when that
195
+ /// level sits at the request's cap (the server emits maxDepth node LEVELS, so
196
+ /// the deepest possible level is maxDepth - 1). A tree that ended naturally
197
+ /// above the cap yields no frontiers and costs nothing.
198
+ + (NSMutableArray<RunnerAXSnapshotFrontier *> *)cappedFrontiersFromCandidates:
199
+ (NSArray<RunnerAXSnapshotFrontier *> *)candidates
200
+ maxDepth:(NSInteger)maxDepth
201
+ {
202
+ NSMutableArray<RunnerAXSnapshotFrontier *> *frontiers = [NSMutableArray array];
203
+ NSInteger deepest = -1;
204
+ for (RunnerAXSnapshotFrontier *candidate in candidates) {
205
+ deepest = MAX(deepest, candidate.depth);
206
+ }
207
+ if (deepest < maxDepth - 1) {
208
+ return frontiers;
209
+ }
210
+ for (RunnerAXSnapshotFrontier *candidate in candidates) {
211
+ if (candidate.depth == deepest) {
212
+ [frontiers addObject:candidate];
213
+ }
214
+ }
215
+ return frontiers;
216
+ }
217
+
218
+ + (nullable id)requestSnapshotFromClient:(id)axClient
219
+ target:(id)target
220
+ attributes:(NSArray *)attributes
221
+ maxDepth:(NSInteger)maxDepth
222
+ maxNodes:(NSInteger)maxNodes
223
+ error:(NSError **)error
224
+ {
225
+ SEL requestSelector = NSSelectorFromString(@"requestSnapshotForElement:attributes:parameters:error:");
226
+ if (![axClient respondsToSelector:requestSelector]) {
227
+ if (NULL != error) {
228
+ *error = [NSError errorWithDomain:@"agent-device.runner"
229
+ code:1
230
+ userInfo:@{
231
+ NSLocalizedDescriptionKey:
232
+ @"AX client does not support requestSnapshotForElement"
233
+ }];
234
+ }
235
+ return nil;
236
+ }
237
+ NSMutableDictionary *parameters = [NSMutableDictionary dictionary];
238
+ id defaults = [self objectFrom:axClient selectorName:@"defaultParameters"];
239
+ if ([defaults isKindOfClass:NSDictionary.class]) {
240
+ [parameters addEntriesFromDictionary:(NSDictionary *)defaults];
241
+ }
242
+ parameters[@"maxDepth"] = @(MAX(0, maxDepth));
243
+ parameters[@"maxChildren"] = @(MAX(1, maxNodes));
244
+ parameters[@"maxArrayCount"] = @(MAX(1, maxNodes));
245
+ parameters[@"traverseFromParentsToChildren"] = @YES;
246
+
247
+ RunnerAXSnapshotMsgSend send = (RunnerAXSnapshotMsgSend)objc_msgSend;
248
+ id result = send(axClient, requestSelector, target, attributes, parameters.copy, error);
249
+ if (nil == result) {
250
+ return nil;
251
+ }
252
+ id root = nil;
253
+ @try {
254
+ root = [result valueForKey:@"_rootElementSnapshot"];
255
+ } @catch (NSException *exception) {
256
+ root = nil;
257
+ }
258
+ return root ?: result;
259
+ }
260
+
261
+ + (NSArray *)snapshotAttributes
262
+ {
263
+ NSArray<NSString *> *keyPaths = @[
264
+ @"elementType",
265
+ @"identifier",
266
+ @"label",
267
+ @"value",
268
+ @"frame",
269
+ @"enabled",
270
+ @"selected",
271
+ @"hasFocus",
272
+ @"children",
273
+ ];
274
+ // The AX server expects real accessibility attribute identifiers, not snapshot keypath
275
+ // strings; passing raw keypaths silently drops attributes it does not recognize (frame
276
+ // came back zeroed). XCElementSnapshot owns the keypath -> AX attribute mapping.
277
+ NSArray *attributes = keyPaths;
278
+ Class snapshotClass = NSClassFromString(@"XCElementSnapshot");
279
+ SEL mapSelector = NSSelectorFromString(@"axAttributesForElementSnapshotKeyPaths:isMacOS:");
280
+ if ([snapshotClass respondsToSelector:mapSelector]) {
281
+ typedef id (*RunnerAXMapMsgSend)(id, SEL, id, BOOL);
282
+ RunnerAXMapMsgSend mapSend = (RunnerAXMapMsgSend)objc_msgSend;
283
+ id mapped = mapSend(snapshotClass, mapSelector, keyPaths, NO);
284
+ if ([mapped isKindOfClass:NSSet.class]) {
285
+ mapped = [(NSSet *)mapped allObjects];
286
+ }
287
+ if ([mapped isKindOfClass:NSArray.class] && [(NSArray *)mapped count] > 0) {
288
+ // The mapper expands keypaths with extra attributes (automation type, window display
289
+ // id, base type) that are disproportionately expensive for the AX server to compute
290
+ // on large React Native trees. Keep only the attributes we actually consume.
291
+ NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"Frame",
292
+ @"Enabled", @"Selected", @"Focus" ];
293
+ NSMutableArray *filtered = [NSMutableArray array];
294
+ for (id attribute in (NSArray *)mapped) {
295
+ NSString *name = [attribute description];
296
+ for (NSString *suffix in needed) {
297
+ if ([name hasSuffix:suffix]) {
298
+ [filtered addObject:attribute];
299
+ break;
300
+ }
301
+ }
302
+ }
303
+ attributes = filtered.count > 0 ? filtered : mapped;
304
+ }
305
+ }
306
+ return attributes;
307
+ }
308
+
309
+ + (nullable id)accessibilityElementForSnapshot:(id)snapshot
310
+ {
311
+ id element = nil;
312
+ @try {
313
+ element = [snapshot valueForKey:@"accessibilityElement"];
314
+ } @catch (NSException *exception) {
315
+ element = nil;
316
+ }
317
+ return element;
318
+ }
319
+
129
320
  + (NSDictionary<NSString *, id> *)failure:(NSString *)message
130
321
  {
131
322
  return @{
@@ -186,12 +377,13 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
186
377
  return nil;
187
378
  }
188
379
 
189
- + (nullable NSDictionary *)dictionaryForSnapshot:(id)snapshot
190
- depth:(NSInteger)depth
191
- maxDepth:(NSInteger)maxDepth
192
- maxNodes:(NSInteger)maxNodes
193
- nodeCount:(NSInteger *)nodeCount
194
- truncated:(BOOL *)truncated
380
+ + (nullable NSMutableDictionary *)dictionaryForSnapshot:(id)snapshot
381
+ depth:(NSInteger)depth
382
+ maxDepth:(NSInteger)maxDepth
383
+ maxNodes:(NSInteger)maxNodes
384
+ nodeCount:(NSInteger *)nodeCount
385
+ truncated:(BOOL *)truncated
386
+ frontiers:(NSMutableArray<RunnerAXSnapshotFrontier *> *)frontiers
195
387
  {
196
388
  if (nil == snapshot || *nodeCount >= maxNodes) {
197
389
  *truncated = YES;
@@ -212,12 +404,13 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
212
404
  NSMutableArray *children = [NSMutableArray array];
213
405
  if (depth < maxDepth) {
214
406
  for (id child in [self childrenForSnapshot:snapshot]) {
215
- NSDictionary *childNode = [self dictionaryForSnapshot:child
216
- depth:depth + 1
217
- maxDepth:maxDepth
218
- maxNodes:maxNodes
219
- nodeCount:nodeCount
220
- truncated:truncated];
407
+ NSMutableDictionary *childNode = [self dictionaryForSnapshot:child
408
+ depth:depth + 1
409
+ maxDepth:maxDepth
410
+ maxNodes:maxNodes
411
+ nodeCount:nodeCount
412
+ truncated:truncated
413
+ frontiers:frontiers];
221
414
  if (nil != childNode) {
222
415
  [children addObject:childNode];
223
416
  }
@@ -227,8 +420,21 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
227
420
  }
228
421
  }
229
422
  }
423
+ if (children.count == 0 && nil != frontiers && depth >= maxDepth - 1) {
424
+ // Childless node at the cap boundary: either a real leaf or a branch whose
425
+ // children the AX server withheld. The two are indistinguishable here, so
426
+ // boundary childless nodes become candidates; the caller keeps only the
427
+ // deepest observed level, where every capped branch necessarily ends.
428
+ // Shallower childless nodes are provably real leaves and are never
429
+ // collected.
430
+ RunnerAXSnapshotFrontier *frontier = [[RunnerAXSnapshotFrontier alloc] init];
431
+ frontier.snapshot = snapshot;
432
+ frontier.node = result;
433
+ frontier.depth = depth;
434
+ [frontiers addObject:frontier];
435
+ }
230
436
  result[@"children"] = children;
231
- return result.copy;
437
+ return result;
232
438
  }
233
439
 
234
440
  + (NSArray *)childrenForSnapshot:(id)snapshot