agent-device 0.21.32 → 0.21.33
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.32.apk → agent-device-android-ime-helper-0.21.33.apk} +0 -0
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.33.apk.sha256 +1 -0
- package/android/ime-helper/dist/{agent-device-android-ime-helper-0.21.32.manifest.json → agent-device-android-ime-helper-0.21.33.manifest.json} +4 -4
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.32.apk → agent-device-android-snapshot-helper-0.21.33.apk} +0 -0
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.33.apk.sha256 +1 -0
- package/android/snapshot-helper/dist/{agent-device-android-snapshot-helper-0.21.32.manifest.json → agent-device-android-snapshot-helper-0.21.33.manifest.json} +7 -7
- package/apple/fold-helper/Fold.m +32 -11
- package/apple/snapshot-bridge/ForegroundApplicationGeometry.h +4 -16
- package/apple/snapshot-bridge/SnapshotBridgeRuntime.m +1 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +36 -58
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +66 -111
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerScrollViewportPolicy.swift +1 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h +2 -2
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m +5 -29
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +4 -17
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Navigation.swift +66 -117
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScrollDragExecution.swift +4 -4
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedGesturePolicy.swift +21 -75
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SynthesizedInteraction.swift +7 -61
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+TypeExecution.swift +1 -4
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.h +24 -53
- package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerXCTestEventBridge.m +13 -39
- package/dist/src/app-backend.js +1 -1
- package/dist/src/app-launch.js +1 -1
- package/dist/src/command-metadata.js +1 -1
- package/dist/src/config.js +1 -1
- package/dist/src/device-readiness-runtime.js +1 -1
- package/dist/src/display-inventory.js +1 -1
- package/dist/src/fold-helper.js +2 -2
- package/dist/src/fold-runtime.js +1 -0
- package/dist/src/interactor.js +5 -5
- package/dist/src/internal/bin.js +1 -1
- package/dist/src/mechanics.js +12 -12
- package/dist/src/physical-device-control.js +1 -1
- package/dist/src/runtime.js +3 -3
- package/dist/src/runtime3.js +5 -5
- package/dist/src/simulator.js +1 -1
- package/package.json +7 -2
- package/server.json +2 -2
- package/android/ime-helper/dist/agent-device-android-ime-helper-0.21.32.apk.sha256 +0 -1
- package/android/snapshot-helper/dist/agent-device-android-snapshot-helper-0.21.32.apk.sha256 +0 -1
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m
CHANGED
|
@@ -21,28 +21,19 @@ NSString *const RunnerAXSnapshotCustomActionsCandidatesKey = @"candidates";
|
|
|
21
21
|
NSString *const RunnerAXSnapshotCustomActionsTruncatedKey = @"truncated";
|
|
22
22
|
NSString *const RunnerAXSnapshotCustomActionsBlockedKey = @"blocked";
|
|
23
23
|
|
|
24
|
-
|
|
25
|
-
|
|
24
|
+
// The AX server's name for UIAccessibilityCustomActions (React Native's `accessibilityActions`
|
|
25
|
+
// prop lands here too).
|
|
26
26
|
static NSString *const RunnerAXCustomActionsAttribute = @"XC_kAXXCAttributeCustomActions";
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
/// capture's budget and starve every remaining candidate, turning a bounded
|
|
31
|
-
/// enrichment into a capture-length stall.
|
|
28
|
+
// Bounds one read. It exists for the unresponsive case: without it one wedged element would
|
|
29
|
+
// swallow the whole capture's budget and starve every remaining candidate.
|
|
32
30
|
static const NSTimeInterval RunnerAXCustomActionReadTimeout = 1.0;
|
|
33
31
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
/// joined by a second concurrent user of the shared XCAXClient;
|
|
40
|
-
/// 2. a single-flight guard (below) refuses to dispatch while an abandoned
|
|
41
|
-
/// read is still outstanding, so repeated captures cannot pile up work
|
|
42
|
-
/// behind it.
|
|
43
|
-
///
|
|
44
|
-
/// Reads resume by themselves once the hung call returns (or the runner
|
|
45
|
-
/// restarts); nothing has to be reset by hand.
|
|
32
|
+
// The AX call is a synchronous XPC round trip that cannot be cancelled once issued, so the
|
|
33
|
+
// timeout above only frees the caller — the call keeps running. Containment needs both parts:
|
|
34
|
+
// every read runs on this one serial queue because the shared XCAXClient has no concurrent user,
|
|
35
|
+
// and the single-flight guard refuses to dispatch while an abandoned read is outstanding. Reads
|
|
36
|
+
// resume by themselves once the hung call returns; nothing has to be reset by hand.
|
|
46
37
|
static dispatch_queue_t RunnerAXCustomActionReadQueue(void)
|
|
47
38
|
{
|
|
48
39
|
static dispatch_queue_t queue;
|
|
@@ -54,21 +45,19 @@ static dispatch_queue_t RunnerAXCustomActionReadQueue(void)
|
|
|
54
45
|
return queue;
|
|
55
46
|
}
|
|
56
47
|
|
|
57
|
-
|
|
58
|
-
|
|
48
|
+
// Reads dispatched but not yet returned. Non-zero after a read blew its deadline, which is the
|
|
49
|
+
// condition the single-flight guard refuses on.
|
|
59
50
|
static atomic_int RunnerAXCustomActionReadsInFlight = 0;
|
|
60
51
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
/// staying put, so it is recorded rather than inferred.
|
|
52
|
+
// Total dispatches. The invariant "a blocked capture adds no work" is observable only as this
|
|
53
|
+
// counter staying put, so it is recorded rather than inferred.
|
|
64
54
|
static atomic_long RunnerAXCustomActionReadDispatches = 0;
|
|
65
55
|
|
|
66
|
-
|
|
56
|
+
// Total single-flight admission refusals.
|
|
67
57
|
static atomic_long RunnerAXCustomActionReadBlocked = 0;
|
|
68
58
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
/// cannot spend the whole snapshot on one node's action list.
|
|
59
|
+
// Output caps. The element budget bounds how many elements are read; these bound ONE element's
|
|
60
|
+
// action list, so a pathological app cannot spend the whole snapshot on one node's list.
|
|
72
61
|
static const NSUInteger RunnerAXCustomActionsMaxPerElement = 8;
|
|
73
62
|
static const NSUInteger RunnerAXCustomActionNameMaxLength = 80;
|
|
74
63
|
|
|
@@ -76,12 +65,6 @@ typedef id (*RunnerAXObjectMsgSend)(id, SEL);
|
|
|
76
65
|
typedef NSInteger (*RunnerAXIntegerMsgSend)(id, SEL);
|
|
77
66
|
typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
78
67
|
|
|
79
|
-
// A childless serialized node remembered with its depth (interface in the
|
|
80
|
-
// header for the unit bundle). Nodes at the deepest observed level of a
|
|
81
|
-
// depth-capped capture are the truncation frontier: the AX server withholds
|
|
82
|
-
// children uniformly at its per-request cap, so every capped branch ends at
|
|
83
|
-
// the same level (observed one BELOW the requested maxDepth — the server
|
|
84
|
-
// counts node levels, not edges).
|
|
85
68
|
@implementation RunnerAXSnapshotFrontier
|
|
86
69
|
@end
|
|
87
70
|
|
|
@@ -139,8 +122,8 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
139
122
|
|
|
140
123
|
BOOL truncated = NO;
|
|
141
124
|
NSInteger nodeCount = 0;
|
|
142
|
-
// nil disables candidate collection entirely — exact --depth captures pay
|
|
143
|
-
//
|
|
125
|
+
// nil disables candidate collection entirely — exact --depth captures pay zero extension
|
|
126
|
+
// bookkeeping.
|
|
144
127
|
NSMutableArray<RunnerAXSnapshotFrontier *> *candidates =
|
|
145
128
|
deepExtensionCallLimit > 0 ? [NSMutableArray array] : nil;
|
|
146
129
|
NSMutableArray<RunnerAXSnapshotFrontier *> *merged =
|
|
@@ -199,12 +182,11 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
199
182
|
}
|
|
200
183
|
}
|
|
201
184
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
/// count, the shared node budget, and the capture-plan deadline.
|
|
185
|
+
// Chains element-rooted snapshot requests from depth-capped frontier nodes. The AX server's depth
|
|
186
|
+
// limit is per-request (kAXErrorIllegalArgument above a tree-size-dependent value), so re-rooting
|
|
187
|
+
// the same request at a frontier node's live element reaches content the app-rooted request could
|
|
188
|
+
// not, without passing a larger depth parameter. Bounded by call count, the shared node budget,
|
|
189
|
+
// and the capture deadline checked between requests.
|
|
208
190
|
+ (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray<RunnerAXSnapshotFrontier *> *)frontiers
|
|
209
191
|
axClient:(id)axClient
|
|
210
192
|
attributes:(NSArray *)attributes
|
|
@@ -221,10 +203,8 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
221
203
|
}
|
|
222
204
|
NSInteger callsUsed = 0;
|
|
223
205
|
NSInteger nodesAdded = 0;
|
|
224
|
-
// A frontier whose live element vanished
|
|
225
|
-
//
|
|
226
|
-
// unresolved — it must count as missed, never as drained, or an all-miss
|
|
227
|
-
// extension would present a capped capture as complete.
|
|
206
|
+
// A frontier whose live element vanished or whose re-rooted request failed leaves its subtree
|
|
207
|
+
// unresolved, so it must count as missed, never as drained.
|
|
228
208
|
NSInteger missedFrontiers = 0;
|
|
229
209
|
while (frontiers.count > 0) {
|
|
230
210
|
if (callsUsed >= callsAllowed || *nodeCount >= maxNodes
|
|
@@ -254,8 +234,8 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
254
234
|
error.localizedDescription ?: @"nil subtree");
|
|
255
235
|
continue;
|
|
256
236
|
}
|
|
257
|
-
// The re-rooted snapshot's root IS the frontier element captured again;
|
|
258
|
-
//
|
|
237
|
+
// The re-rooted snapshot's root IS the frontier element captured again; splice only its
|
|
238
|
+
// children so the frontier node is not duplicated.
|
|
259
239
|
NSInteger before = *nodeCount;
|
|
260
240
|
NSMutableArray<RunnerAXSnapshotFrontier *> *subCandidates = [NSMutableArray array];
|
|
261
241
|
NSMutableArray *children = [NSMutableArray array];
|
|
@@ -291,10 +271,9 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
291
271
|
};
|
|
292
272
|
}
|
|
293
273
|
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
/// above the cap yields no frontiers and costs nothing.
|
|
274
|
+
// Keeps only candidates at the deepest observed level, and only when that level sits at the
|
|
275
|
+
// request's cap: the server emits maxDepth node LEVELS, so the deepest possible level is
|
|
276
|
+
// maxDepth - 1. A tree that ended naturally above the cap yields no frontiers.
|
|
298
277
|
+ (NSMutableArray<RunnerAXSnapshotFrontier *> *)cappedFrontiersFromCandidates:
|
|
299
278
|
(NSArray<RunnerAXSnapshotFrontier *> *)candidates
|
|
300
279
|
maxDepth:(NSInteger)maxDepth
|
|
@@ -372,9 +351,9 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
372
351
|
@"hasFocus",
|
|
373
352
|
@"children",
|
|
374
353
|
];
|
|
375
|
-
// The AX server expects real accessibility attribute identifiers, not snapshot keypath
|
|
376
|
-
//
|
|
377
|
-
//
|
|
354
|
+
// The AX server expects real accessibility attribute identifiers, not snapshot keypath strings;
|
|
355
|
+
// raw keypaths silently drop attributes (frame came back zeroed). XCElementSnapshot owns the
|
|
356
|
+
// keypath -> AX attribute mapping.
|
|
378
357
|
NSArray *attributes = keyPaths;
|
|
379
358
|
Class snapshotClass = NSClassFromString(@"XCElementSnapshot");
|
|
380
359
|
SEL mapSelector = NSSelectorFromString(@"axAttributesForElementSnapshotKeyPaths:isMacOS:");
|
|
@@ -386,9 +365,8 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
386
365
|
mapped = [(NSSet *)mapped allObjects];
|
|
387
366
|
}
|
|
388
367
|
if ([mapped isKindOfClass:NSArray.class] && [(NSArray *)mapped count] > 0) {
|
|
389
|
-
// The mapper expands keypaths with extra attributes
|
|
390
|
-
//
|
|
391
|
-
// on large React Native trees. Keep only the attributes we actually consume.
|
|
368
|
+
// The mapper expands keypaths with extra attributes that are disproportionately expensive
|
|
369
|
+
// for the AX server to compute on large trees. Keep only what is consumed here.
|
|
392
370
|
NSArray *needed = @[ @"ElementType", @"Identifier", @"Label", @"Value", @"PlaceholderValue",
|
|
393
371
|
@"Frame", @"Enabled", @"Selected", @"Focus" ];
|
|
394
372
|
NSMutableArray *filtered = [NSMutableArray array];
|
|
@@ -407,29 +385,15 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
407
385
|
return attributes;
|
|
408
386
|
}
|
|
409
387
|
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
///
|
|
420
|
-
/// `candidates` counts only elements still eligible at read time: a leaf that
|
|
421
|
-
/// deep extension filled in since serialization is not a merged element, so
|
|
422
|
-
/// counting it would inflate the denominator with nodes that never needed a
|
|
423
|
-
/// read. A leaf whose live element vanished stays counted and unread, because
|
|
424
|
-
/// that one really is an element we failed to answer for.
|
|
425
|
-
///
|
|
426
|
-
/// On-screen candidates are read first. The budget is smaller than a long
|
|
427
|
-
/// feed's card count, so the order decides which elements an agent can act on
|
|
428
|
-
/// at all — and it also makes the disclosure's remedy true: scrolling changes
|
|
429
|
-
/// the on-screen set, so a re-run reads elements the previous pass could not.
|
|
430
|
-
/// Note that `--scope` deliberately is NOT the ordering key: scope is applied
|
|
431
|
-
/// when the Swift walk builds nodes, long after this pass, so it cannot narrow
|
|
432
|
-
/// the read budget without duplicating the scope predicate here.
|
|
388
|
+
// Reads custom actions for the merged leaves in traversal order until the limit or the deadline is
|
|
389
|
+
// spent; each read is its own AX round trip, so this is strictly opt-in and strictly bounded.
|
|
390
|
+
// The counts exist so the response can disclose an incomplete pass: an unread element is
|
|
391
|
+
// byte-identical to one with no actions. `candidates` counts only elements still eligible at read
|
|
392
|
+
// time; a leaf whose live element vanished stays counted and unread. On-screen candidates are read
|
|
393
|
+
// first: the budget is smaller than a long feed's card count, so the order decides which elements
|
|
394
|
+
// an agent can act on, and scrolling changes the on-screen set so a re-run reaches what the
|
|
395
|
+
// previous pass could not. `--scope` is deliberately not the ordering key: the Swift walk applies
|
|
396
|
+
// it long after this pass, so it cannot narrow the read budget without duplicating the predicate.
|
|
433
397
|
+ (NSDictionary<NSString *, NSNumber *> *)
|
|
434
398
|
annotateCustomActionsOnMergedLeaves:(nullable NSArray<RunnerAXSnapshotFrontier *> *)leaves
|
|
435
399
|
axClient:(id)axClient
|
|
@@ -460,10 +424,8 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
460
424
|
if (attempts >= limit || (nil != deadline && deadline.timeIntervalSinceNow <= 0)) {
|
|
461
425
|
break;
|
|
462
426
|
}
|
|
463
|
-
// An abandoned read from an earlier capture is still outstanding
|
|
464
|
-
//
|
|
465
|
-
// all be refused, and record why so the response does not present the
|
|
466
|
-
// unread elements as "has no actions".
|
|
427
|
+
// An abandoned read from an earlier capture is still outstanding: stop here rather than spend
|
|
428
|
+
// a deadline per element on reads that will all be refused, and record why.
|
|
467
429
|
if ([self customActionReadsInFlight] > 0) {
|
|
468
430
|
blocked = YES;
|
|
469
431
|
break;
|
|
@@ -472,9 +434,8 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
472
434
|
if (nil == element) {
|
|
473
435
|
continue;
|
|
474
436
|
}
|
|
475
|
-
// The
|
|
476
|
-
//
|
|
477
|
-
// of being reported as "read, and it has no actions".
|
|
437
|
+
// The budget buys attempts; only a completed read counts as read, so a timed-out element stays
|
|
438
|
+
// in the undisclosed remainder.
|
|
478
439
|
attempts += 1;
|
|
479
440
|
BOOL completed = NO;
|
|
480
441
|
NSArray<NSString *> *names = [self customActionNamesForElement:element
|
|
@@ -507,8 +468,8 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
507
468
|
};
|
|
508
469
|
}
|
|
509
470
|
|
|
510
|
-
|
|
511
|
-
|
|
471
|
+
// An empty root frame means the capture has no viewport to judge against, so every candidate
|
|
472
|
+
// counts as on-screen and the order stays document order.
|
|
512
473
|
+ (BOOL)isFrameOnScreen:(id)frameValue rootFrame:(CGRect)rootFrame
|
|
513
474
|
{
|
|
514
475
|
if (CGRectIsEmpty(rootFrame)) {
|
|
@@ -544,9 +505,9 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
544
505
|
atomic_fetch_add(&RunnerAXCustomActionReadBlocked, 1);
|
|
545
506
|
return nil;
|
|
546
507
|
}
|
|
547
|
-
// The AX call is a synchronous XPC round trip with no timeout of its own, so
|
|
548
|
-
//
|
|
549
|
-
//
|
|
508
|
+
// The AX call is a synchronous XPC round trip with no timeout of its own, so it runs off-thread
|
|
509
|
+
// behind a bounded wait. On timeout the result box is never read, so a late-returning wedged
|
|
510
|
+
// call cannot race this thread.
|
|
550
511
|
atomic_fetch_add(&RunnerAXCustomActionReadDispatches, 1);
|
|
551
512
|
NSMutableArray *box = [NSMutableArray array];
|
|
552
513
|
dispatch_semaphore_t finished = dispatch_semaphore_create(0);
|
|
@@ -563,8 +524,8 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
563
524
|
if (nil != result) {
|
|
564
525
|
[box addObject:result];
|
|
565
526
|
}
|
|
566
|
-
// Clear in-flight BEFORE signalling, so a waiter that wakes
|
|
567
|
-
//
|
|
527
|
+
// Clear in-flight BEFORE signalling, so a waiter that wakes here never observes its own
|
|
528
|
+
// completed read as still outstanding.
|
|
568
529
|
atomic_store(&RunnerAXCustomActionReadsInFlight, 0);
|
|
569
530
|
dispatch_semaphore_signal(finished);
|
|
570
531
|
});
|
|
@@ -599,11 +560,9 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
599
560
|
return names.count > 0 ? names.copy : nil;
|
|
600
561
|
}
|
|
601
562
|
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
/// as well as a hostile case. Truncation is reported, never silent: a clipped
|
|
606
|
-
/// list is exactly as misleading as an unread element if it looks complete.
|
|
563
|
+
// Bounds what one element can contribute to the response. The names come from the app, so their
|
|
564
|
+
// count and length are the app's choice; truncation is reported, never silent, because a clipped
|
|
565
|
+
// list is as misleading as an unread element if it looks complete.
|
|
607
566
|
+ (NSArray<NSString *> *)cappedActionNames:(NSArray<NSString *> *)names
|
|
608
567
|
truncated:(BOOL *)truncated
|
|
609
568
|
{
|
|
@@ -689,9 +648,8 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
689
648
|
if (![target respondsToSelector:selector]) {
|
|
690
649
|
return 0;
|
|
691
650
|
}
|
|
692
|
-
// processID/processIdentifier return pid_t (int32); reading them through an
|
|
693
|
-
//
|
|
694
|
-
// signature to pick the correctly sized call.
|
|
651
|
+
// processID/processIdentifier return pid_t (int32); reading them through an NSInteger-returning
|
|
652
|
+
// cast is not upper-32-bit safe on arm64. Use the method signature to pick the sized call.
|
|
695
653
|
NSMethodSignature *signature = [target methodSignatureForSelector:selector];
|
|
696
654
|
const char *returnType = signature.methodReturnType;
|
|
697
655
|
if (returnType != NULL && strcmp(returnType, @encode(int)) == 0) {
|
|
@@ -794,9 +752,9 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
794
752
|
if (children.count == 0 && nil != mergedLeaves
|
|
795
753
|
&& [result[@"label"] isKindOfClass:NSString.class]
|
|
796
754
|
&& [(NSString *)result[@"label"] length] > 0) {
|
|
797
|
-
// A labelled childless node is the shape a merged card takes: the container
|
|
798
|
-
//
|
|
799
|
-
//
|
|
755
|
+
// A labelled childless node is the shape a merged card takes: the container is marked
|
|
756
|
+
// accessible, so its real controls never appear as children. Only these are worth an action
|
|
757
|
+
// read — unlabelled leaves are decoration.
|
|
800
758
|
RunnerAXSnapshotFrontier *candidate = [[RunnerAXSnapshotFrontier alloc] init];
|
|
801
759
|
candidate.snapshot = snapshot;
|
|
802
760
|
candidate.node = result;
|
|
@@ -804,12 +762,9 @@ typedef id (*RunnerAXSnapshotMsgSend)(id, SEL, id, id, id, NSError **);
|
|
|
804
762
|
[mergedLeaves addObject:candidate];
|
|
805
763
|
}
|
|
806
764
|
if (children.count == 0 && nil != frontiers && depth >= maxDepth - 1) {
|
|
807
|
-
// Childless node at the cap boundary: either a real leaf or a branch whose
|
|
808
|
-
//
|
|
809
|
-
//
|
|
810
|
-
// deepest observed level, where every capped branch necessarily ends.
|
|
811
|
-
// Shallower childless nodes are provably real leaves and are never
|
|
812
|
-
// collected.
|
|
765
|
+
// Childless node at the cap boundary: either a real leaf or a branch whose children the AX
|
|
766
|
+
// server withheld. The two are indistinguishable here, so boundary childless nodes become
|
|
767
|
+
// candidates; shallower childless nodes are provably real leaves and are never collected.
|
|
813
768
|
RunnerAXSnapshotFrontier *frontier = [[RunnerAXSnapshotFrontier alloc] init];
|
|
814
769
|
frontier.snapshot = snapshot;
|
|
815
770
|
frontier.node = result;
|
|
@@ -239,9 +239,8 @@ extension RunnerTests {
|
|
|
239
239
|
context: SynthesizedCoordinateContext
|
|
240
240
|
) {
|
|
241
241
|
NSLog(
|
|
242
|
-
"AGENT_DEVICE_RUNNER_SCROLL_VIEWPORT kind=scroll axHealth=%@
|
|
242
|
+
"AGENT_DEVICE_RUNNER_SCROLL_VIEWPORT kind=scroll axHealth=%@ decision=%@ keyboardMinY=%@ swipeHeight=%.1f",
|
|
243
243
|
context.accessibilityHealth.rawValue,
|
|
244
|
-
context.keyboardPolicy.rawValue,
|
|
245
244
|
decision,
|
|
246
245
|
keyboardMinY.map { String(format: "%.1f", $0) } ?? "none",
|
|
247
246
|
swipeHeight
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.h
CHANGED
|
@@ -14,8 +14,8 @@ typedef NS_ENUM(NSInteger, RunnerTapSynthesisStatus) {
|
|
|
14
14
|
@interface RunnerSynthesizedGesture : NSObject
|
|
15
15
|
|
|
16
16
|
// `resolvedWindow` is the app window the caller already resolved for the gesture's reference
|
|
17
|
-
// frame
|
|
18
|
-
//
|
|
17
|
+
// frame, so the record's display ID is read from that same window. Pass nil for the sole-screen
|
|
18
|
+
// route: the app's tree could not name its window and the screen is proven instead.
|
|
19
19
|
+ (NSString * _Nullable)synthesizeSwipeWithApplication:(id)application
|
|
20
20
|
resolvedWindow:(id _Nullable)resolvedWindow
|
|
21
21
|
x:(double)x
|
package/dist/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerSynthesizedGesture.m
CHANGED
|
@@ -29,7 +29,6 @@ typedef id (*RunnerDragPointerPathFactory)(
|
|
|
29
29
|
CGPoint,
|
|
30
30
|
double
|
|
31
31
|
);
|
|
32
|
-
typedef double (*RunnerDragProgressFunction)(double);
|
|
33
32
|
|
|
34
33
|
static NSString * _Nullable RunnerResolveGestureEventBridge(
|
|
35
34
|
id application,
|
|
@@ -58,14 +57,6 @@ static id RunnerControlledScrollPointerPath(
|
|
|
58
57
|
CGPoint end,
|
|
59
58
|
double durationMs
|
|
60
59
|
);
|
|
61
|
-
static id RunnerSampledDragPointerPath(
|
|
62
|
-
const RunnerGestureEventBridge *bridge,
|
|
63
|
-
CGPoint start,
|
|
64
|
-
CGPoint end,
|
|
65
|
-
double durationMs,
|
|
66
|
-
RunnerDragProgressFunction progress,
|
|
67
|
-
NSInteger frameCount
|
|
68
|
-
);
|
|
69
60
|
static NSString * _Nullable RunnerTrySynthesizeDrag(
|
|
70
61
|
id application,
|
|
71
62
|
id _Nullable resolvedWindow,
|
|
@@ -332,8 +323,10 @@ static NSString * _Nullable RunnerCreateEventRecord(
|
|
|
332
323
|
|
|
333
324
|
NSUInteger displayID = 0;
|
|
334
325
|
// A nil window is the caller's sole-screen route: the app tree could not name its window.
|
|
326
|
+
// Re-asking here, at dispatch, is what keeps the record's display the one the app fills now.
|
|
327
|
+
id soleScreen = nil;
|
|
335
328
|
NSString *displayError = resolvedWindow == nil
|
|
336
|
-
?
|
|
329
|
+
? RunnerResolveSoleScreen(application, &soleScreen, &displayID)
|
|
337
330
|
: RunnerResolveWindowDisplayID(resolvedWindow, &displayID);
|
|
338
331
|
if (displayError != nil) return displayError;
|
|
339
332
|
NSInteger interfaceOrientation =
|
|
@@ -418,34 +411,17 @@ static id RunnerControlledScrollPointerPath(
|
|
|
418
411
|
// A stationary hold is not a reliable zero-velocity release in XCTest. Cubic ease-out decreases
|
|
419
412
|
// velocity from the first movement and reaches near zero before lift. Profiles that accelerate
|
|
420
413
|
// after pan recognition can still be classified as flings despite a slow final sample.
|
|
421
|
-
return RunnerSampledDragPointerPath(
|
|
422
|
-
bridge,
|
|
423
|
-
start,
|
|
424
|
-
end,
|
|
425
|
-
durationMs,
|
|
426
|
-
RunnerControlledScrollProgress,
|
|
427
|
-
RunnerControlledScrollFrameCount(durationMs)
|
|
428
|
-
);
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
static id RunnerSampledDragPointerPath(
|
|
432
|
-
const RunnerGestureEventBridge *bridge,
|
|
433
|
-
CGPoint start,
|
|
434
|
-
CGPoint end,
|
|
435
|
-
double durationMs,
|
|
436
|
-
RunnerDragProgressFunction progress,
|
|
437
|
-
NSInteger frameCount
|
|
438
|
-
) {
|
|
439
414
|
id path =
|
|
440
415
|
((RunnerMsgSendInitPath)objc_msgSend)([bridge->core.pathClass alloc], bridge->initPathSelector, start, 0.0);
|
|
441
416
|
if (path == nil) {
|
|
442
417
|
return nil;
|
|
443
418
|
}
|
|
444
419
|
|
|
420
|
+
NSInteger frameCount = RunnerControlledScrollFrameCount(durationMs);
|
|
445
421
|
NSTimeInterval durationSeconds = durationMs / 1000.0;
|
|
446
422
|
for (NSInteger index = 1; index <= frameCount; index += 1) {
|
|
447
423
|
double t = (double)index / (double)frameCount;
|
|
448
|
-
double easedT =
|
|
424
|
+
double easedT = RunnerControlledScrollProgress(t);
|
|
449
425
|
CGPoint point = CGPointMake(
|
|
450
426
|
start.x + (end.x - start.x) * easedT,
|
|
451
427
|
start.y + (end.y - start.y) * easedT
|
|
@@ -76,10 +76,7 @@ extension RunnerTests {
|
|
|
76
76
|
var fallback: GestureFallback?
|
|
77
77
|
if command.synthesized == true {
|
|
78
78
|
let policyKind = SynthesizedGesturePolicyKind.coordinateTap
|
|
79
|
-
let context = synthesizedCoordinateContext(
|
|
80
|
-
app: activeApp,
|
|
81
|
-
policy: synthesizedGesturePolicy(policyKind)
|
|
82
|
-
)
|
|
79
|
+
let context = synthesizedCoordinateContext(app: activeApp)
|
|
83
80
|
switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
|
|
84
81
|
synthesizedTapAt(
|
|
85
82
|
app: activeApp,
|
|
@@ -169,10 +166,7 @@ extension RunnerTests {
|
|
|
169
166
|
var fallback: GestureFallback?
|
|
170
167
|
if command.synthesized == true {
|
|
171
168
|
let policyKind = SynthesizedGesturePolicyKind.coordinateTap
|
|
172
|
-
let context = synthesizedCoordinateContext(
|
|
173
|
-
app: activeApp,
|
|
174
|
-
policy: synthesizedGesturePolicy(policyKind)
|
|
175
|
-
)
|
|
169
|
+
let context = synthesizedCoordinateContext(app: activeApp)
|
|
176
170
|
switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
|
|
177
171
|
synthesizedTapAt(app: activeApp, x: x, y: y, context: context)
|
|
178
172
|
}) {
|
|
@@ -233,10 +227,7 @@ extension RunnerTests {
|
|
|
233
227
|
let fallback: GestureFallback?
|
|
234
228
|
#if os(iOS)
|
|
235
229
|
let policyKind = SynthesizedGesturePolicyKind.coordinateLongPress
|
|
236
|
-
let context = synthesizedCoordinateContext(
|
|
237
|
-
app: activeApp,
|
|
238
|
-
policy: synthesizedGesturePolicy(policyKind)
|
|
239
|
-
)
|
|
230
|
+
let context = synthesizedCoordinateContext(app: activeApp)
|
|
240
231
|
switch performSynthesizedGesture(activeApp, kind: policyKind, context: context, synthesize: {
|
|
241
232
|
synthesizedLongPressAt(app: activeApp, x: x, y: y, durationMs: durationMs, context: context)
|
|
242
233
|
}) {
|
|
@@ -287,11 +278,7 @@ extension RunnerTests {
|
|
|
287
278
|
else {
|
|
288
279
|
return invalidScrollDirectionResponse(commandName: "scroll")
|
|
289
280
|
}
|
|
290
|
-
let
|
|
291
|
-
guard let scrollContext = synthesizedCoordinateContext(
|
|
292
|
-
app: activeApp,
|
|
293
|
-
policy: synthesizedGesturePolicy(scrollPolicyKind)
|
|
294
|
-
) else {
|
|
281
|
+
guard let scrollContext = synthesizedCoordinateContext(app: activeApp) else {
|
|
295
282
|
return Response(
|
|
296
283
|
ok: false,
|
|
297
284
|
error: ErrorPayload(message: "scroll could not resolve a usable interaction frame")
|